Skip to contentSkip to conversation

tools/resolve_btfids: Install subcmd headers

PATCH Applied Ian Rogers 4 participants 14 messages Jan 24, 2023 – Feb 1, 2023 BPF

lore
Share & tools
lore

Ian Rogers

author
#0 lore raw

PATCH v4 1/2 tools/resolve_btfids: Install subcmd headers

Previously tools/lib/subcmd was added to the include path, switch to installing the headers and then including from that directory. This avoids dependencies on headers internal to tools/lib/subcmd. Add the missing subcmd directory to the affected #include.

---
 tools/bpf/resolve_btfids/Makefile | 19 ++++++++++++++-----
 tools/bpf/resolve_btfids/main.c   |  2 +-
 2 files changed, 15 insertions(+), 6 deletions(-)
tools/bpf/resolve_btfids/Makefile +14 -5
index f7375a119f54..1fe0082b2ecc 100644
@@ -35,21 +35,29 @@ SUBCMD_SRC := $(srctree)/tools/lib/subcmd/
 BPFOBJ     := $(OUTPUT)/libbpf/libbpf.a
 LIBBPF_OUT := $(abspath $(dir $(BPFOBJ)))/
 SUBCMDOBJ  := $(OUTPUT)/libsubcmd/libsubcmd.a
+SUBCMD_OUT := $(abspath $(dir $(SUBCMDOBJ)))/
 
 LIBBPF_DESTDIR := $(LIBBPF_OUT)
 LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include
 
+SUBCMD_DESTDIR := $(SUBCMD_OUT)
+SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include
+
 BINARY     := $(OUTPUT)/resolve_btfids
 BINARY_IN  := $(BINARY)-in.o
 
 all: $(BINARY)
 
+prepare: $(BPFOBJ) $(SUBCMDOBJ)
+
 $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 	$(call msg,MKDIR,,$@)
 	$(Q)mkdir -p $(@)
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
-	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
+	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
+		    DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+		    $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
@@ -63,7 +71,7 @@ CFLAGS += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
           -I$(LIBBPF_INCLUDE) \
-          -I$(SUBCMD_SRC) \
+          -I$(SUBCMD_INCLUDE) \
           $(LIBELF_FLAGS)
 
 LIBS = $(LIBELF_LIBS) -lz
@@ -71,7 +79,7 @@ LIBS = $(LIBELF_LIBS) -lz
 export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
-$(BINARY_IN): $(BPFOBJ) fixdep FORCE | $(OUTPUT)
+$(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
 	$(Q)$(MAKE) $(build)=resolve_btfids
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
@@ -83,7 +91,8 @@ clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.d             \
                             $(LIBBPF_OUT)                \
                             $(LIBBPF_DESTDIR)            \
-                            $(OUTPUT)/libsubcmd          \
+                            $(SUBCMD_OUT)                \
+                            $(SUBCMD_DESTDIR)            \
                             $(OUTPUT)/resolve_btfids)
 
 ifneq ($(clean_objects),)
@@ -100,4 +109,4 @@ tags:
 
 FORCE:
 
-.PHONY: all FORCE clean tags
+.PHONY: all FORCE clean tags prepare
tools/bpf/resolve_btfids/main.c +1 -1
index 80cd7843c677..77058174082d 100644
@@ -75,7 +75,7 @@
 #include <linux/err.h>
 #include <bpf/btf.h>
 #include <bpf/libbpf.h>
-#include <parse-options.h>
+#include <subcmd/parse-options.h>
 
 #define BTF_IDS_SECTION	".BTF_ids"
 #define BTF_ID		"__BTF_ID__"
Signature · 1 line
2.39.0.246.g2a6d74b583-goog
author
#1 lore raw

PATCH v4 2/2 tools/resolve_btfids: Alter how HOSTCC is forced

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -10
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+		  EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
 	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-		    DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+		    DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-		    DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+		    DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-	$(Q)$(MAKE) $(build)=resolve_btfids
+	$(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
 	$(call msg,LINK,$@)
-	$(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+	$(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
Signature · 1 line
2.39.0.246.g2a6d74b583-goog
Acked-by on v4 2/2
#2 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 10 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Quoted 4 lines

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

jirka

Quoted 58 lines
---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -10
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+		  EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
 	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-		    DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+		    DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-		    DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+		    DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-	$(Q)$(MAKE) $(build)=resolve_btfids
+	$(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
 	$(call msg,LINK,$@)
-	$(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+	$(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
Signature · 1 line
2.39.0.246.g2a6d74b583-goog
#3 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

In reply to Ian Rogers

Hi Ian,

Quoted 72 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -10
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+		  EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
 	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-		    DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+		    DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-		    DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+		    DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
 		    $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-	$(Q)$(MAKE) $(build)=resolve_btfids
+	$(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
 	$(call msg,LINK,$@)
-	$(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+	$(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
Signature · 1 line
2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Cheers,
Nathan

author
#4 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 113 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

Quoted 5 lines

If there is any other information I can provide or patches I can test, please let me know.

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

author
#5 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 118 lines · Ian Rogers
Quoted 116 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Quoted 7 lines
Quoted 2 lines

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

```
diff --git a/tools/bpf/resolve_btfids/Makefile
b/tools/bpf/resolve_btfids/Makefile
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolve_btfids/Makefile
+++ b/tools/bpf/resolve_btfids/Makefile
@@ -19,10 +19,9 @@ endif

# always use the host compiler
HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)"
ARCH="$(HOSTARCH)" \
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
CROSS_COMPILE=""

RM      ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Thanks,
Ian

#6 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 122 lines · Ian Rogers
Quoted 121 lines · Ian Rogers
Quoted 118 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Cheers,
Nathan

Quoted 7 lines
Quoted 4 lines

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Quoted 15 lines

```

tools/bpf/resolve_btfids/Makefile +0 -0
index daed388aa5d7..a06966841df4 100644
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

Cheers,
Nathan

author
#7 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 129 lines · Nathan Chancellor
Quoted 127 lines · Ian Rogers
Quoted 125 lines · Ian Rogers
Quoted 113 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

Quoted 5 lines

If there is any other information I can provide or patches I can test, please let me know.

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Quoted 8 lines
Quoted 2 lines

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Thanks, do you have instructions on setting up the container?

Quoted 5 lines

tools/bpf/resolve_btfids/Makefile

Quoted 5 lines

```

tools/bpf/resolve_btfids/Makefile +0 -1
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolv

Ian
e_btfids/Makefile

Quoted 20 lines
Quoted 10 lines

+++ b/tools/bpf/resolve_btfids/Makefile

tools/bpf/resolve_btfids/Makefile +0 -0
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

Quoted 2 lines

Cheers,
Nathan

Hmm.. I can see the issue. We avoid this in places like libsubcmd because the Makefile there includes tools/scripts/Makefile.include, but here the Makefile.build inclusion is direct and we have CC set up for cross compilation. This is messy and it will take some playing around to come up with a simple fix.

Thanks,
Ian

#8 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 133 lines · Ian Rogers
Quoted 132 lines · Nathan Chancellor
Quoted 129 lines · Ian Rogers
Quoted 118 lines · Ian Rogers
Quoted 116 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Quoted 7 lines
Quoted 2 lines

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

Quoted 7 lines
Quoted 5 lines

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Thanks, do you have instructions on setting up the container?

Sure thing, this worked for me in the Linux source you would like to build (I marked it readonly to make sure any modifications were done in my host environment, remove the ':ro' if you would like to make edits to the source within the container). Docker should work as well but I did not test it.

$ podman run --rm -ti -v $PWD:/linux:ro docker.io/archlinux:base-devel
# pacman -Syyu --noconfirm bc clang lib32-glibc lld llvm python
...

# To turn on CONFIG_DEBUG_INFO_BTF

# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build defconfig menuconfig
# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build prepare
ld.lld: error: /build/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
Quoted 28 lines
Quoted 5 lines

tools/bpf/resolve_btfids/Makefile

Quoted 5 lines

```

tools/bpf/resolve_btfids/Makefile +0 -1
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolv

Ian
e_btfids/Makefile

Quoted 20 lines
Quoted 10 lines

+++ b/tools/bpf/resolve_btfids/Makefile

tools/bpf/resolve_btfids/Makefile +0 -0
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

Quoted 8 lines
Quoted 2 lines

Cheers,
Nathan

Hmm.. I can see the issue. We avoid this in places like libsubcmd because the Makefile there includes tools/scripts/Makefile.include, but here the Makefile.build inclusion is direct and we have CC set up for cross compilation. This is messy and it will take some playing around to come up with a simple fix.

Ack, if you have any issues with reproducing it, please let me know.

Cheers,
Nathan

#9 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 152 lines · Nathan Chancellor
Quoted 133 lines · Ian Rogers
Quoted 132 lines · Nathan Chancellor
Quoted 129 lines · Ian Rogers
Quoted 118 lines · Ian Rogers
Quoted 116 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Quoted 7 lines
Quoted 2 lines

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

Quoted 7 lines
Quoted 5 lines

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Thanks, do you have instructions on setting up the container?

Sure thing, this worked for me in the Linux source you would like to build (I marked it readonly to make sure any modifications were done in my host environment, remove the ':ro' if you would like to make edits to the source within the container). Docker should work as well but I did not test it.

$ podman run --rm -ti -v $PWD:/linux:ro docker.io/archlinux:base-devel
# pacman -Syyu --noconfirm bc clang lib32-glibc lld llvm python
...
Quoted 34 lines

# To turn on CONFIG_DEBUG_INFO_BTF

# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build defconfig menuconfig
# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build prepare
ld.lld: error: /build/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
Quoted 27 lines
Quoted 5 lines

tools/bpf/resolve_btfids/Makefile

Quoted 5 lines

```

tools/bpf/resolve_btfids/Makefile +0 -1
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolv

Ian
e_btfids/Makefile

Quoted 19 lines
Quoted 10 lines

+++ b/tools/bpf/resolve_btfids/Makefile

tools/bpf/resolve_btfids/Makefile +0 -0
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

hi,
I can reproduce that and I think the reason is that the patch removes CFLAGS setup

  CFLAGS  := $(KBUILD_HOSTCFLAGS)

so now the 'include ../../scripts/Makefile.include' will set CFLAGS with the --target=... and we fail

I can compile with the change below.. we could also set CROSS_COMPILE before including tools/scripts/Makefile.include ... not sure which on is better

also I need to check if we should keep LDFLAGS setup as well

jirka

---

tools/bpf/resolve_btfids/Makefile +1 -0
index daed388aa5d7..875e3fcce7cd 100644
@@ -64,6 +64,7 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OU
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
 LIBELF_LIBS  := $(shell $(HOSTPKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
 
+CFLAGS := $(KBUILD_HOSTCFLAGS)
 CFLAGS += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
author
#10 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 198 lines · Jiri Olsa
Quoted 152 lines · Nathan Chancellor
Quoted 133 lines · Ian Rogers
Quoted 132 lines · Nathan Chancellor
Quoted 129 lines · Ian Rogers
Quoted 118 lines · Ian Rogers
Quoted 116 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Quoted 7 lines
Quoted 2 lines

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

Quoted 7 lines
Quoted 5 lines

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Thanks, do you have instructions on setting up the container?

Sure thing, this worked for me in the Linux source you would like to build (I marked it readonly to make sure any modifications were done in my host environment, remove the ':ro' if you would like to make edits to the source within the container). Docker should work as well but I did not test it.

$ podman run --rm -ti -v $PWD:/linux:ro docker.io/archlinux:base-devel
# pacman -Syyu --noconfirm bc clang lib32-glibc lld llvm python
...
Quoted 34 lines

# To turn on CONFIG_DEBUG_INFO_BTF

# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build defconfig menuconfig
# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build prepare
ld.lld: error: /build/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
Quoted 27 lines
Quoted 5 lines

tools/bpf/resolve_btfids/Makefile

Quoted 5 lines

```

tools/bpf/resolve_btfids/Makefile +0 -1
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolv

Ian
e_btfids/Makefile

Quoted 19 lines
Quoted 10 lines

+++ b/tools/bpf/resolve_btfids/Makefile

tools/bpf/resolve_btfids/Makefile +0 -0
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

hi,
I can reproduce that and I think the reason is that the patch removes CFLAGS setup

  CFLAGS  := $(KBUILD_HOSTCFLAGS)

so now the 'include ../../scripts/Makefile.include' will set CFLAGS with the --target=... and we fail

Quoted 7 lines

I can compile with the change below.. we could also set CROSS_COMPILE before including tools/scripts/Makefile.include ... not sure which on is better

also I need to check if we should keep LDFLAGS setup as well

jirka

Right, but the code shouldn't be doing that - it was the purpose of the patch to remove it. If you include tools/scripts/Makefile.include, which is done on line 2, then a bunch of warnings, cflags, etc. are set based on CC. To then overwrite those values with various HOST values is at best confusing - CC no longer being CC but HOSTCC, but other values may or may not have changed. As we build using submakes, the goal here was to push the problem into the submake which will only ever see CC and other values being passed in.

So perhaps:
HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)" CROSS_COMPILE=""

should be:
HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)" CROSS_COMPILE=""

I'm still trying to get a reproduction.

Thanks,
Ian

Quoted 15 lines

---

tools/bpf/resolve_btfids/Makefile +1 -0
index daed388aa5d7..875e3fcce7cd 100644
@@ -64,6 +64,7 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OU
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
 LIBELF_LIBS  := $(shell $(HOSTPKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
 
+CFLAGS := $(KBUILD_HOSTCFLAGS)
 CFLAGS += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
author
#11 lore raw

Re: [PATCH v4 2/2] tools/resolve_btfids: Alter how HOSTCC is forced

Quoted 227 lines · Ian Rogers
Quoted 198 lines · Jiri Olsa
Quoted 152 lines · Nathan Chancellor
Quoted 133 lines · Ian Rogers
Quoted 132 lines · Nathan Chancellor
Quoted 129 lines · Ian Rogers
Quoted 118 lines · Ian Rogers
Quoted 116 lines · Nathan Chancellor

Hi Ian,

Quoted 70 lines · Ian Rogers

HOSTCC is always wanted when building. Setting CC to HOSTCC happens after tools/scripts/Makefile.include is included, meaning flags are set assuming say CC is gcc, but then it can be later set to HOSTCC which may be clang. tools/scripts/Makefile.include is needed for host set up and common macros in objtool's Makefile. Rather than override CC to HOSTCC, just pass CC as HOSTCC to Makefile.build, the libsubcmd builds and the linkage step. This means the Makefiles don't see things like CC changing and tool flag determination, and similar, work properly.

Also, clear the passed subdir as otherwise an outer build may break by inadvertently passing an inappropriate value.

---
 tools/bpf/resolve_btfids/Makefile | 17 +++++++----------
 1 file changed, 7 insertions(+), 10 deletions(-)
tools/bpf/resolve_btfids/Makefile +7 -11
index 1fe0082b2ecc..daed388aa5d7 100644
@@ -18,14 +18,11 @@ else
 endif
 
 # always use the host compiler
-AR       = $(HOSTAR)
-CC       = $(HOSTCC)
-LD       = $(HOSTLD)
-ARCH     = $(HOSTARCH)
+HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
+               EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+
 RM      ?= rm
 CROSS_COMPILE =
-CFLAGS  := $(KBUILD_HOSTCFLAGS)
-LDFLAGS := $(KBUILD_HOSTLDFLAGS)
 
 OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
 
@@ -56,12 +53,12 @@ $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
      $(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
-                 DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+                 DESTDIR=$(SUBCMD_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
      $(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
-                 DESTDIR=$(LIBBPF_DESTDIR) prefix= EXTRA_CFLAGS="$(CFLAGS)" \
+                 DESTDIR=$(LIBBPF_DESTDIR) $(HOST_OVERRIDES) prefix= subdir= \
                  $(abspath $@) install_headers
 
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
@@ -80,11 +77,11 @@ export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
 $(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
-     $(Q)$(MAKE) $(build)=resolve_btfids
+     $(Q)$(MAKE) $(build)=resolve_btfids $(HOST_OVERRIDES)
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
      $(call msg,LINK,$@)
-     $(Q)$(CC) $(BINARY_IN) $(LDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
+     $(Q)$(HOSTCC) $(BINARY_IN) $(KBUILD_HOSTLDFLAGS) -o $@ $(BPFOBJ) $(SUBCMDOBJ) $(LIBS)
 
 clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.cmd           \
--
Quoted 2 lines

2.39.0.246.g2a6d74b583-goog

I just bisected a linking failure when building resolve_btfids with clang to this change as commit 13e07691a16f ("tools/resolve_btfids:
Alter how HOSTCC is forced") in the bpf-next tree.

It appears to be related to whether or not CROSS_COMPILE is specified, which we have to do for certain architectures and configurations still. arm64 is not one of those but it helps demonstrate the issue.

  # Turn off CONFIG_DEBUG_INFO_REDUCED and turn on CONFIG_DEBUG_INFO_BTF
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 defconfig menuconfig
  $ make -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 prepare
  ld.lld: error: $LINUX_SRC/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
  clang-17: error: linker command failed with exit code 1 (use -v to see invocation)
  ...

Before your change, with V=1, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o -Wall -Wmissing-prototypes -Wstrict-prototypes -O2 -fomit-frame-pointer -std=gnu11 -Wdeclaration-after-statement -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

After, I see:

clang -Wp,-MD,$LINUX_SRC/tools/bpf/resolve_btfids/.main.o.d -Wp,-MT,$LINUX_SRC/tools/bpf/resolve_btfids/main.o --target=aarch64-linux-gnu -g -I$LINUX_SRC/tools/include -I$LINUX_SRC/tools/include/uapi -I$LINUX_SRC/tools/bpf/resolve_btfids/libbpf/include -I$LINUX_SRC/tools/bpf/resolve_btfids/libsubcmd/include -D"BUILD_STR(s)=#s" -c -o $LINUX_SRC/tools/bpf/resolve_btfids/main.o main.c

We seem to have taken on a '--target=aarch64-linux-gnu' (changing the target of resolve_btfids-in.o) and we dropped the warning flags.

I think this comes from the clang block in tools/scripts/Makefile.include, which is included into the resolve_btfids Makefile via tools/lib/bpf/Makefile.

I am not super familiar with the tools build system, otherwise I would try to provide a patch. I tried moving CROSS_COMPILE from a recursive to simple variable ('=' -> ':=') and moving it to HOST_OVERRIDES but those did not appear to resolve it for me.

If there is any other information I can provide or patches I can test, please let me know.

Quoted 7 lines
Quoted 2 lines

Cheers,
Nathan

Thanks Nathan, and thanks for all the details in the bug report. I'm looking into this.

Ian

Given the somewhat complicated cross compile I wasn't able to get a reproduction. Could you see if the following addresses the problem:

Quoted 7 lines
Quoted 5 lines

As long as you have an LLVM toolchain that targets AArch64 and your host, you should be able to reproduce this issue with those commands verbatim, as that command should not use any GNU binutils. I am pretty sure I tried it in a fresh container before reporting it but it is possible that I did not.

Thanks, do you have instructions on setting up the container?

Sure thing, this worked for me in the Linux source you would like to build (I marked it readonly to make sure any modifications were done in my host environment, remove the ':ro' if you would like to make edits to the source within the container). Docker should work as well but I did not test it.

$ podman run --rm -ti -v $PWD:/linux:ro docker.io/archlinux:base-devel
# pacman -Syyu --noconfirm bc clang lib32-glibc lld llvm python
...
Quoted 34 lines

# To turn on CONFIG_DEBUG_INFO_BTF

# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build defconfig menuconfig
# make -C /linux -skj"$(nproc)" ARCH=arm64 CROSS_COMPILE=aarch64-linux-gnu- HOSTLDFLAGS=-fuse-ld=lld LLVM=1 O=/build prepare
ld.lld: error: /build/tools/bpf/resolve_btfids//resolve_btfids-in.o is incompatible with elf64-x86-64
clang-15: error: linker command failed with exit code 1 (use -v to see invocation)
Quoted 27 lines
Quoted 5 lines

tools/bpf/resolve_btfids/Makefile

Quoted 5 lines

```

tools/bpf/resolve_btfids/Makefile +0 -1
index daed388aa5d7..a06966841df4 100644
--- a/tools/bpf/resolv

Ian
e_btfids/Makefile

Quoted 19 lines
Quoted 10 lines

+++ b/tools/bpf/resolve_btfids/Makefile

tools/bpf/resolve_btfids/Makefile +0 -0
@@ -19,10 +19,9 @@ endif
 
 # always use the host compiler

HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \

tools/bpf/resolve_btfids/Makefile +1 -1
-                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"
+                 EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)"

CROSS_COMPILE=""

Quoted 6 lines

RM ?= rm
-CROSS_COMPILE =

OUTPUT ?= $(srctree)/tools/bpf/resolve_btfids/
```

Unfortunately, it does not. I still see '--target=' end up in the CFLAGS of those files.

hi,
I can reproduce that and I think the reason is that the patch removes CFLAGS setup

  CFLAGS  := $(KBUILD_HOSTCFLAGS)

so now the 'include ../../scripts/Makefile.include' will set CFLAGS with the --target=... and we fail

Quoted 7 lines

I can compile with the change below.. we could also set CROSS_COMPILE before including tools/scripts/Makefile.include ... not sure which on is better

also I need to check if we should keep LDFLAGS setup as well

jirka

Right, but the code shouldn't be doing that - it was the purpose of the patch to remove it. If you include tools/scripts/Makefile.include, which is done on line 2, then a bunch of warnings, cflags, etc. are set based on CC. To then overwrite those values with various HOST values is at best confusing - CC no longer being CC but HOSTCC, but other values may or may not have changed. As we build using submakes, the goal here was to push the problem into the submake which will only ever see CC and other values being passed in.

So perhaps:
HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
EXTRA_CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)" CROSS_COMPILE=""

should be:
HOST_OVERRIDES := AR="$(HOSTAR)" CC="$(HOSTCC)" LD="$(HOSTLD)" ARCH="$(HOSTARCH)" \
CFLAGS="$(HOSTCFLAGS) $(KBUILD_HOSTCFLAGS)" CROSS_COMPILE=""

Quoted 4 lines

I'm still trying to get a reproduction.

Thanks,
Ian

So I got a reproduction from Nathan's steps and I believe:
https://lore.kernel.org/lkml/[email protected]/
Fixes the issue and tidies the Makefile a little. I was a little unclear on the target being built, and tested some with -C, etc. I'm always quite capable of making more errors though, so please help me to check.

Apologies again for the problem and hopefully this fixes it. Thanks, Ian

Quoted 15 lines
Quoted 15 lines

---

tools/bpf/resolve_btfids/Makefile +1 -0
index daed388aa5d7..875e3fcce7cd 100644
@@ -64,6 +64,7 @@ $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OU
 LIBELF_FLAGS := $(shell $(HOSTPKG_CONFIG) libelf --cflags 2>/dev/null)
 LIBELF_LIBS  := $(shell $(HOSTPKG_CONFIG) libelf --libs 2>/dev/null || echo -lelf)
 
+CFLAGS := $(KBUILD_HOSTCFLAGS)
 CFLAGS += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
Acked-by iri Olsa on v4 1/2
#12 lore raw
In reply to Ian Rogers
Quoted 6 lines · Ian Rogers

Previously tools/lib/subcmd was added to the include path, switch to installing the headers and then including from that directory. This avoids dependencies on headers internal to tools/lib/subcmd. Add the missing subcmd directory to the affected #include.

jirka

Quoted 90 lines
---
 tools/bpf/resolve_btfids/Makefile | 19 ++++++++++++++-----
 tools/bpf/resolve_btfids/main.c   |  2 +-
 2 files changed, 15 insertions(+), 6 deletions(-)
tools/bpf/resolve_btfids/Makefile +14 -5
index f7375a119f54..1fe0082b2ecc 100644
@@ -35,21 +35,29 @@ SUBCMD_SRC := $(srctree)/tools/lib/subcmd/
 BPFOBJ     := $(OUTPUT)/libbpf/libbpf.a
 LIBBPF_OUT := $(abspath $(dir $(BPFOBJ)))/
 SUBCMDOBJ  := $(OUTPUT)/libsubcmd/libsubcmd.a
+SUBCMD_OUT := $(abspath $(dir $(SUBCMDOBJ)))/
 
 LIBBPF_DESTDIR := $(LIBBPF_OUT)
 LIBBPF_INCLUDE := $(LIBBPF_DESTDIR)include
 
+SUBCMD_DESTDIR := $(SUBCMD_OUT)
+SUBCMD_INCLUDE := $(SUBCMD_DESTDIR)include
+
 BINARY     := $(OUTPUT)/resolve_btfids
 BINARY_IN  := $(BINARY)-in.o
 
 all: $(BINARY)
 
+prepare: $(BPFOBJ) $(SUBCMDOBJ)
+
 $(OUTPUT) $(OUTPUT)/libsubcmd $(LIBBPF_OUT):
 	$(call msg,MKDIR,,$@)
 	$(Q)mkdir -p $(@)
 
 $(SUBCMDOBJ): fixdep FORCE | $(OUTPUT)/libsubcmd
-	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(abspath $(dir $@))/ $(abspath $@)
+	$(Q)$(MAKE) -C $(SUBCMD_SRC) OUTPUT=$(SUBCMD_OUT) \
+		    DESTDIR=$(SUBCMD_DESTDIR) prefix= \
+		    $(abspath $@) install_headers
 
 $(BPFOBJ): $(wildcard $(LIBBPF_SRC)/*.[ch] $(LIBBPF_SRC)/Makefile) | $(LIBBPF_OUT)
 	$(Q)$(MAKE) $(submake_extras) -C $(LIBBPF_SRC) OUTPUT=$(LIBBPF_OUT)    \
@@ -63,7 +71,7 @@ CFLAGS += -g \
           -I$(srctree)/tools/include \
           -I$(srctree)/tools/include/uapi \
           -I$(LIBBPF_INCLUDE) \
-          -I$(SUBCMD_SRC) \
+          -I$(SUBCMD_INCLUDE) \
           $(LIBELF_FLAGS)
 
 LIBS = $(LIBELF_LIBS) -lz
@@ -71,7 +79,7 @@ LIBS = $(LIBELF_LIBS) -lz
 export srctree OUTPUT CFLAGS Q
 include $(srctree)/tools/build/Makefile.include
 
-$(BINARY_IN): $(BPFOBJ) fixdep FORCE | $(OUTPUT)
+$(BINARY_IN): fixdep FORCE prepare | $(OUTPUT)
 	$(Q)$(MAKE) $(build)=resolve_btfids
 
 $(BINARY): $(BPFOBJ) $(SUBCMDOBJ) $(BINARY_IN)
@@ -83,7 +91,8 @@ clean_objects := $(wildcard $(OUTPUT)/*.o                \
                             $(OUTPUT)/.*.o.d             \
                             $(LIBBPF_OUT)                \
                             $(LIBBPF_DESTDIR)            \
-                            $(OUTPUT)/libsubcmd          \
+                            $(SUBCMD_OUT)                \
+                            $(SUBCMD_DESTDIR)            \
                             $(OUTPUT)/resolve_btfids)
 
 ifneq ($(clean_objects),)
@@ -100,4 +109,4 @@ tags:
 
 FORCE:
 
-.PHONY: all FORCE clean tags
+.PHONY: all FORCE clean tags prepare
tools/bpf/resolve_btfids/main.c +1 -1
index 80cd7843c677..77058174082d 100644
@@ -75,7 +75,7 @@
 #include <linux/err.h>
 #include <bpf/btf.h>
 #include <bpf/libbpf.h>
-#include <parse-options.h>
+#include <subcmd/parse-options.h>
 
 #define BTF_IDS_SECTION	".BTF_ids"
 #define BTF_ID		"__BTF_ID__"
Signature · 1 line
2.39.0.246.g2a6d74b583-goog
#13 lore raw
In reply to Ian Rogers
This series was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:

Hello:

This series was applied to bpf/bpf-next.git (master) by Daniel Borkmann <[email protected]>:

Quoted 8 lines · On Mon, 23 Jan 2023 22:43:23 -0800 you wrote:

Previously tools/lib/subcmd was added to the include path, switch to installing the headers and then including from that directory. This avoids dependencies on headers internal to tools/lib/subcmd. Add the missing subcmd directory to the affected #include.

[...]

Here is the summary with links:
- [v4,1/2] tools/resolve_btfids: Install subcmd headers

    https://git.kernel.org/bpf/bpf-next/c/af03299d8536
  - [v4,2/2] tools/resolve_btfids: Alter how HOSTCC is forced
    https://git.kernel.org/bpf/bpf-next/c/13e07691a16f

You are awesome, thank you!

Signature · 2 lines
Deet-doot-dot, I am a bot.
https://korg.docs.kernel.org/patchwork/pwbot.html