Skip to contentSkip to conversation

xen: rework compat headers generation

PATCH Superseded Anthony PERARD 5 participants 23 messages Jun 1, 2022 – Jun 13, 2022

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

cover xen: rework compat headers generation

Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1

Hi,

This patch series is about 2 improvement. First one is to use $(if_changed, ) in "include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.

Thanks.

Anthony PERARD (4):
build: xen/include: use if_changed
build: set PERL
build: replace get-fields.sh by a perl script
build: remove auto.conf prerequisite from compat/xlat.h target

 xen/Makefile                 |   1 +
 xen/include/Makefile         | 106 ++++---
 xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++
 xen/tools/get-fields.sh      | 528 ----------------------------------
 4 files changed, 614 insertions(+), 560 deletions(-)
 create mode 100755 xen/tools/compat-xlat-header
 delete mode 100644 xen/tools/get-fields.sh
Signature · 1 line
Anthony PERARD
#1 lore raw

PATCH 1/4 build: xen/include: use if_changed

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

---
 xen/include/Makefile | 108 ++++++++++++++++++++++++++++++-------------
 1 file changed, 76 insertions(+), 32 deletions(-)
xen/include/Makefile +76 -32
index 03baf10efb..6d9bcc19b0 100644
@@ -45,38 +45,65 @@ public-$(CONFIG_ARM) := $(wildcard $(srcdir)/public/arch-arm/*.h $(srcdir)/publi
 .PHONY: all
 all: $(addprefix $(obj)/,$(headers-y))
 
-$(obj)/compat/%.h: $(obj)/compat/%.i $(srcdir)/Makefile $(srctree)/tools/compat-build-header.py
-	$(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \
-	mv -f [email protected] $@
-
-$(obj)/compat/%.i: $(obj)/compat/%.c $(srcdir)/Makefile
-	$(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $<
-
-$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srcdir)/Makefile $(srctree)/tools/compat-build-source.py
-	mkdir -p $(@D)
-	$(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]
-	mv -f [email protected] $@
-
-$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh $(srcdir)/Makefile
-	export PYTHON=$(PYTHON); \
-	while read what name; do \
-		$(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \
-	done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]
-	mv -f [email protected] $@
+quiet_cmd_compat_h = GEN     $@
+cmd_compat_h = \
+    $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \
+    mv -f [email protected] $@
+
+quiet_cmd_compat_i = CPP     $@
+cmd_compat_i = $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $<
+
+quiet_cmd_compat_c = GEN     $@
+cmd_compat_c = \
+   $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]; \
+   mv -f [email protected] $@
+
+quiet_cmd_xlat_headers = GEN     $@
+cmd_xlat_headers = \
+    while read what name; do \
+        $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \
+    done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \
+    mv -f [email protected] $@
+
+targets += $(headers-y)
+$(obj)/compat/%.h: $(obj)/compat/%.i $(srctree)/tools/compat-build-header.py FORCE
+	$(call if_changed,compat_h)
+
+.PRECIOUS: $(obj)/compat/%.i
+targets += $(patsubst %.h, %.i, $(headers-y))
+$(obj)/compat/%.i: $(obj)/compat/%.c FORCE
+	$(call if_changed,compat_i)
+
+.PRECIOUS: $(obj)/compat/%.c
+targets += $(patsubst %.h, %.c, $(headers-y))
+$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat-build-source.py FORCE
+	$(call if_changed,compat_c)
+
+targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y))
+$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE
+	$(call if_changed,xlat_headers)
+
+quiet_cmd_xlat_lst = GEN     $@
+cmd_xlat_lst = \
+	grep -v '^[[:blank:]]*$(pound)' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]; \
+	$(call move-if-changed,[email protected],$@)
 
 .PRECIOUS: $(obj)/compat/.xlat/%.lst
-$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst $(srcdir)/Makefile
-	mkdir -p $(@D)
-	grep -v '^[[:blank:]]*#' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]
-	$(call move-if-changed,[email protected],$@)
+targets += $(patsubst compat/%.h, compat/.xlat/%.lst, $(headers-y))
+$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst FORCE
+	$(call if_changed,xlat_lst)
 
 xlat-y := $(shell sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,^[?!][[:blank:]]+[^[:blank:]]+[[:blank:]]+,,p' $(srcdir)/xlat.lst | uniq)
 xlat-y := $(filter $(patsubst compat/%,%,$(headers-y)),$(xlat-y))
 
-$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf $(srcdir)/Makefile
-	cat $(filter %.h,$^) >[email protected]
+quiet_cmd_xlat_h = GEN     $@
+cmd_xlat_h = \
+	cat $(filter %.h,$^) >[email protected]; \
 	mv -f [email protected] $@
 
+$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE
+	$(call if_changed,xlat_h)
+
 ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH))
 
 all: $(obj)/headers.chk $(obj)/headers99.chk $(obj)/headers++.chk
@@ -102,27 +129,31 @@ PUBLIC_C99_HEADERS := $(call public-filter-headers,public-c99-headers)
 $(src)/public/io/9pfs.h-prereq := string
 $(src)/public/io/pvcalls.h-prereq := string
 
-$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) $(srcdir)/Makefile
+quiet_cmd_header_chk = CHK     $@
+cmd_header_chk = \
 	for i in $(filter %.h,$^); do \
 	    $(CC) -x c -ansi -Wall -Werror -include stdint.h \
 	          -S -o /dev/null $$i || exit 1; \
 	    echo $$i; \
-	done >[email protected]
+	done >[email protected]; \
 	mv [email protected] $@
 
-$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) $(srcdir)/Makefile
-	rm -f [email protected]
+quiet_cmd_headers99_chk = CHK     $@
+define cmd_headers99_chk
+	rm -f [email protected]; \
 	$(foreach i, $(filter %.h,$^),                                        \
 	    echo "#include "\"$(i)\"                                          \
 	    | $(CC) -x c -std=c99 -Wall -Werror                               \
 	      -include stdint.h                                               \
 	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include $(j).h) \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];)
+	    || exit $$?; echo $(i) >> [email protected];) \
 	mv [email protected] $@
+endef
 
-$(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile
-	rm -f [email protected]
+quiet_cmd_headerscxx_chk = CHK     $@
+define cmd_headerscxx_chk
+	rm -f [email protected]; \
 	if ! $(CXX) -v >/dev/null 2>&1; then                                  \
 	    touch [email protected];                                                     \
 	    exit 0;                                                           \
@@ -133,8 +164,21 @@ $(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile
 	      -include stdint.h -include $(srcdir)/public/xen.h               \
 	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];)
+	    || exit $$?; echo $(i) >> [email protected];) \
 	mv [email protected] $@
+endef
+
+targets += headers.chk
+$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) FORCE
+	$(call if_changed,header_chk)
+
+targets += headers99.chk
+$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) FORCE
+	$(call if_changed,headers99_chk)
+
+targets += headers++.chk
+$(obj)/headers++.chk: $(PUBLIC_HEADERS) FORCE
+	$(call if_changed,headerscxx_chk)
 
 endif
 
Signature · 1 line
Anthony PERARD
#2 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Quoted 3 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

In cmd_xlat_lst you use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.

Jan

#3 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Quoted 8 lines · Jan Beulich
Quoted 3 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

In cmd_xlat_lst you use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.

I think I just forgot about $(pound) when I tried to debug why the command didn't work in some environment (that is when not using define).

I think the second thing that make me not replacing '#' by "$(pound)" is that reading "#include" in the Makefile is probably better that reading "$(pound)include".

I guess we should add something to the justification like:
That allow us to keep writing "#include" in the Makefile without having to replace that by "$(pound)include" which would be a bit less obvious about the command line purpose.

Thanks,

Signature · 1 line
Anthony PERARD
Acked-by
#4 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Quoted 13 lines · Anthony PERARD
Quoted 8 lines · Jan Beulich
Quoted 3 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

In cmd_xlat_lst you use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.

I think I just forgot about $(pound) when I tried to debug why the command didn't work in some environment (that is when not using define).

Quoted 8 lines

I think the second thing that make me not replacing '#' by "$(pound)" is that reading "#include" in the Makefile is probably better that reading "$(pound)include".

I guess we should add something to the justification like:
That allow us to keep writing "#include" in the Makefile without having to replace that by "$(pound)include" which would be a bit less obvious about the command line purpose.

I'd be okay with this, and I'd also be okay with adding this while committing. With it added

Jan

#5 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

In reply to Anthony PERARD

Hi Anthony,

Quoted 11 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Quoted 5 lines

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Cheers
Bertrand

Quoted 166 lines

---
xen/include/Makefile | 108 ++++++++++++++++++++++++++++++-------------

1 file changed, 76 insertions(+), 32 deletions(-)
xen/include/Makefile +0 -0
index 03baf10efb..6d9bcc19b0 100644
@@ -45,38 +45,65 @@ public-$(CONFIG_ARM) := $(wildcard $(srcdir)/public/arch-arm/*.h $(srcdir)/publi

.PHONY: all
all: $(addprefix $(obj)/,$(headers-y))

-$(obj)/compat/%.h: $(obj)/compat/%.i $(srcdir)/Makefile $(srctree)/tools/compat-build-header.py

xen/include/Makefile +42 -17
-	$(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \
-	mv -f [email protected] $@
-
-$(obj)/compat/%.i: $(obj)/compat/%.c $(srcdir)/Makefile
-	$(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $<
-
-$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srcdir)/Makefile $(srctree)/tools/compat-build-source.py
-	mkdir -p $(@D)
-	$(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]
-	mv -f [email protected] $@
-
-$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh $(srcdir)/Makefile
-	export PYTHON=$(PYTHON); \
-	while read what name; do \
-		$(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \
-	done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]
-	mv -f [email protected] $@
+quiet_cmd_compat_h = GEN     $@
+cmd_compat_h = \
+    $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \
+    mv -f [email protected] $@
+
+quiet_cmd_compat_i = CPP     $@
+cmd_compat_i = $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $<
+
+quiet_cmd_compat_c = GEN     $@
+cmd_compat_c = \
+   $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]; \
+   mv -f [email protected] $@
+
+quiet_cmd_xlat_headers = GEN     $@
+cmd_xlat_headers = \
+    while read what name; do \
+        $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \
+    done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \
+    mv -f [email protected] $@
+
+targets += $(headers-y)
+$(obj)/compat/%.h: $(obj)/compat/%.i $(srctree)/tools/compat-build-header.py FORCE
+	$(call if_changed,compat_h)
+
+.PRECIOUS: $(obj)/compat/%.i
+targets += $(patsubst %.h, %.i, $(headers-y))
+$(obj)/compat/%.i: $(obj)/compat/%.c FORCE
+	$(call if_changed,compat_i)
+
+.PRECIOUS: $(obj)/compat/%.c
+targets += $(patsubst %.h, %.c, $(headers-y))
+$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat-build-source.py FORCE
+	$(call if_changed,compat_c)
+
+targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y))
+$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE
+	$(call if_changed,xlat_headers)
+
+quiet_cmd_xlat_lst = GEN     $@
+cmd_xlat_lst = \
+	grep -v '^[[:blank:]]*$(pound)' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]; \
+	$(call move-if-changed,[email protected],$@)

.PRECIOUS: $(obj)/compat/.xlat/%.lst -$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst $(srcdir)/Makefile

xen/include/Makefile +3 -3
-	mkdir -p $(@D)
-	grep -v '^[[:blank:]]*#' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]
-	$(call move-if-changed,[email protected],$@)
+targets += $(patsubst compat/%.h, compat/.xlat/%.lst, $(headers-y))
+$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst FORCE
+	$(call if_changed,xlat_lst)

xlat-y := $(shell sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,^[?!][[:blank:]]+[^[:blank:]]+[[:blank:]]+,,p' $(srcdir)/xlat.lst | uniq) xlat-y := $(filter $(patsubst compat/%,%,$(headers-y)),$(xlat-y))

-$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf $(srcdir)/Makefile

xen/include/Makefile +3 -1
-	cat $(filter %.h,$^) >[email protected]
+quiet_cmd_xlat_h = GEN     $@
+cmd_xlat_h = \
+	cat $(filter %.h,$^) >[email protected]; \
	mv -f [email protected] $@

+$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE

xen/include/Makefile +2 -0
+	$(call if_changed,xlat_h)
+

ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH))

all: $(obj)/headers.chk $(obj)/headers99.chk $(obj)/headers++.chk

xen/include/Makefile +0 -0
@@ -102,27 +129,31 @@ PUBLIC_C99_HEADERS := $(call public-filter-headers,public-c99-headers)

$(src)/public/io/9pfs.h-prereq := string
$(src)/public/io/pvcalls.h-prereq := string

-$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) $(srcdir)/Makefile

xen/include/Makefile +2 -0
+quiet_cmd_header_chk = CHK     $@
+cmd_header_chk = \
	for i in $(filter %.h,$^); do \
	    $(CC) -x c -ansi -Wall -Werror -include stdint.h \
	          -S -o /dev/null $$i || exit 1; \
	    echo $$i; \
xen/include/Makefile +1 -1
-	done >[email protected]
+	done >[email protected]; \
	mv [email protected] $@

-$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) $(srcdir)/Makefile

xen/include/Makefile +3 -1
-	rm -f [email protected]
+quiet_cmd_headers99_chk = CHK     $@
+define cmd_headers99_chk
+	rm -f [email protected]; \
	$(foreach i, $(filter %.h,$^),                                        \
	    echo "#include "\"$(i)\"                                          \
	    | $(CC) -x c -std=c99 -Wall -Werror                               \
	      -include stdint.h                                               \
	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include $(j).h) \
	      -S -o /dev/null -                                               \
xen/include/Makefile +1 -1
-	    || exit $$?; echo $(i) >> [email protected];)
+	    || exit $$?; echo $(i) >> [email protected];) \
	mv [email protected] $@
xen/include/Makefile +4 -2
+endef
 
-$(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile
-	rm -f [email protected]
+quiet_cmd_headerscxx_chk = CHK     $@
+define cmd_headerscxx_chk
+	rm -f [email protected]; \
	if ! $(CXX) -v >/dev/null 2>&1; then                                  \
	    touch [email protected];                                                     \
	    exit 0;                                                           \
xen/include/Makefile +14 -1
@@ -133,8 +164,21 @@ $(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile
 	      -include stdint.h -include $(srcdir)/public/xen.h               \
 	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];)
+	    || exit $$?; echo $(i) >> [email protected];) \
 	mv [email protected] $@
+endef
+
+targets += headers.chk
+$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) FORCE
+	$(call if_changed,header_chk)
+
+targets += headers99.chk
+$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) FORCE
+	$(call if_changed,headers99_chk)
+
+targets += headers++.chk
+$(obj)/headers++.chk: $(PUBLIC_HEADERS) FORCE
+	$(call if_changed,headerscxx_chk)
 

endif

Signature · 1 line
Anthony PERARD
#6 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Quoted 17 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Quoted 5 lines

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

Quoted 8 lines

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

Also please remember to trim your replies.

Jan

#7 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Hi,

Quoted 25 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Quoted 5 lines

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Quoted 8 lines
Quoted 3 lines

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Doing a verbose compilation I have this (sorry for the long lines):

for i in include/public/vcpu.h include/public/errno.h include/public/kexec.h include/public/argo.h include/public/xen.h include/public/nmi.h include/public/xencomm.h include/public/xenoprof.h include/public/device_tree_defs.h include/public/version.h include/public/memory.h include/public/features.h include/public/sched.h include/public/xen-compat.h include/public/callback.h include/public/vm_event.h include/public/grant_table.h include/public/physdev.h include/public/tmem.h include/public/hypfs.h include/public/platform.h include/public/pmu.h include/public/elfnote.h include/public/trace.h include/public/event_channel.h include/public/io/vscsiif.h include/public/io/kbdif.h include/public/io/protocols.h include/public/io/ring.h include/public/io/displif.h include/public/io/fsif.h include/public/io/blkif.h include/public/io/console.h include/public/io/sndif.h include/public/io/fbif.h include/public/io/libxenvchan.h include/public/io/netif.h include/public/io/usbif.h include/public/io/pciif.h include/public/io/tpmif.h include/public/io/xs_wire.h include/public/io/xenbus.h include/public/io/cameraif.h include/public/hvm/pvdrivers.h include/public/hvm/e820.h include/public/hvm/hvm_xs_strings.h include/public/hvm/dm_op.h include/public/hvm/ioreq.h include/public/hvm/hvm_info_table.h include/public/hvm/hvm_vcpu.h include/public/hvm/hvm_op.h include/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -include stdint.h -S -o /dev/null $i || exit 1; echo $i; done >include/headers.chk.new; mv include/headers.chk.new include/headers.chk

|       rm -f include/headers99.chk.new;  echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new;  echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk
| make[9]: execvp: /bin/sh: Argument list too long
| make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127
| make[9]: *** Waiting for unfinished jobs....

So the command passed to the sub shell by make is quite long.

No idea why this comes out only when building in Yocto but I will dig a bit.

Quoted 3 lines

Also please remember to trim your replies.

Will do.

Bertrand

#8 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Quoted 39 lines · Bertrand Marquis
Quoted 24 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Quoted 5 lines

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Quoted 8 lines
Quoted 3 lines

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Doing a verbose compilation I have this (sorry for the long lines):

for i in include/public/vcpu.h include/public/errno.h include/public/kexec.h include/public/argo.h include/public/xen.h include/public/nmi.h include/public/xencomm.h include/public/xenoprof.h include/public/device_tree_defs.h include/public/version.h include/public/memory.h include/public/features.h include/public/sched.h include/public/xen-compat.h include/public/callback.h include/public/vm_event.h include/public/grant_table.h include/public/physdev.h include/public/tmem.h include/public/hypfs.h include/public/platform.h include/public/pmu.h include/public/elfnote.h include/public/trace.h include/public/event_channel.h include/public/io/vscsiif.h include/public/io/kbdif.h include/public/io/protocols.h include/public/io/ring.h include/public/io/displif.h include/public/io/fsif.h include/public/io/blkif.h include/public/io/console.h include/public/io/sndif.h include/public/io/fbif.h include/public/io/libxenvchan.h include/public/io/netif.h include/public/io/usbif.h include/public/io/pciif.h include/public/io/tpmif.h include/public/io/xs_wire.h include/public/io/xenbus.h include/public/io/cameraif.h include/public/hvm/pvdrivers.h include/public/hvm/e820.h include/public/hvm/hvm_xs_strings.h include/public/hvm/dm_op.h include/public/hvm/ioreq.h include/public/hvm/hvm_info_table.h include/public/hvm/hvm_vcpu.h include/public/hvm/hvm_op.h include/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -include stdint.h -S -o /dev/null $i || exit 1; echo $i; done >include/headers.chk.new; mv include/headers.chk.new include/headers.chk

Quoted 8 lines
|       rm -f include/headers99.chk.new;  echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new;  echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk
| make[9]: execvp: /bin/sh: Argument list too long
| make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127
| make[9]: *** Waiting for unfinished jobs....

So the command passed to the sub shell by make is quite long.

No idea why this comes out only when building in Yocto but I will dig a bit.

Maybe Yocto has an unusually low limit on command arguments' total size? The whole thing is just over 2500 chars, which doesn't look to be unusually long for Unix-like environments.

Jan

#9 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Hi,

Quoted 47 lines · Jan Beulich
Quoted 44 lines · Bertrand Marquis
Quoted 24 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD

Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).

Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.

Quoted 5 lines

Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).

Remove "export PYTHON" as it is already exported.

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Quoted 8 lines
Quoted 3 lines

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Doing a verbose compilation I have this (sorry for the long lines):

for i in include/public/vcpu.h include/public/errno.h include/public/kexec.h include/public/argo.h include/public/xen.h include/public/nmi.h include/public/xencomm.h include/public/xenoprof.h include/public/device_tree_defs.h include/public/version.h include/public/memory.h include/public/features.h include/public/sched.h include/public/xen-compat.h include/public/callback.h include/public/vm_event.h include/public/grant_table.h include/public/physdev.h include/public/tmem.h include/public/hypfs.h include/public/platform.h include/public/pmu.h include/public/elfnote.h include/public/trace.h include/public/event_channel.h include/public/io/vscsiif.h include/public/io/kbdif.h include/public/io/protocols.h include/public/io/ring.h include/public/io/displif.h include/public/io/fsif.h include/public/io/blkif.h include/public/io/console.h include/public/io/sndif.h include/public/io/fbif.h include/public/io/libxenvchan.h include/public/io/netif.h include/public/io/usbif.h include/public/io/pciif.h include/public/io/tpmif.h include/public/io/xs_wire.h include/public/io/xenbus.h include/public/io/cameraif.h include/public/hvm/pvdrivers.h include/public/hvm/e820.h include/public/hvm/hvm_xs_strings.h include/public/hvm/dm_op.h include/public/hvm/ioreq.h include/public/hvm/hvm_info_table.h include/public/hvm/hvm_vcpu.h include/public/hvm/hvm_op.h include/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -include stdint.h -S -o /dev/null $i || exit 1; echo $i; done >include/headers.chk.new; mv include/headers.chk.new include/headers.chk

|       rm -f include/headers99.chk.new;  echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new;  echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc  --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot  -x c -std=c99 -Wall -Werror -include stdint.h  -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk
| make[9]: execvp: /bin/sh: Argument list too long
| make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127
| make[9]: *** Waiting for unfinished jobs....
Quoted 8 lines
Quoted 3 lines

So the command passed to the sub shell by make is quite long.

No idea why this comes out only when building in Yocto but I will dig a bit.

Maybe Yocto has an unusually low limit on command arguments' total size? The whole thing is just over 2500 chars, which doesn't look to be unusually long for Unix-like environments.

Actually the command to generate headers++.chk is 15294 characters when I build xen normally. In Yocto it becomes a lot bigger as CC includes a sysroot parameter with a path.

Bertrand

#10 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

In reply to Bertrand Marquis
Quoted 14 lines · Bertrand Marquis

Hi,

Quoted 12 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Quoted 8 lines
Quoted 4 lines

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Will the following patch help?

From 0f32f749304b233c0d5574dc6b14f66e8709feba Mon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chk

The command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.

Error messages:
make[9]: execvp: /bin/sh: Argument list too long make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".

Fixes 28e13c7f43 ("build: xen/include: use if_changed")
---
 xen/include/Makefile | 17 +++++++++++++----
 1 file changed, 13 insertions(+), 4 deletions(-)
xen/include/Makefile +13 -4
index 6d9bcc19b0..ca5e868f38 100644
@@ -158,13 +158,22 @@ define cmd_headerscxx_chk
 	    touch [email protected];                                                     \
 	    exit 0;                                                           \
 	fi;                                                                   \
-	$(foreach i, $(filter %.h,$^),                                        \
-	    echo "#include "\"$(i)\"                                          \
+	get_prereq() {                                                        \
+	    case $$1 in                                                       \
+	    $(foreach i, $(filter %.h,$^),                                    \
+	    $(if $($(patsubst $(srctree)/%,%,$i)-prereq),                     \
+		$(patsubst $(srctree)/%,%,$i)$(close)                         \
+		echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq),   \
+			-include c$(j))";;))                                  \
+	    esac;                                                             \
+	};                                                                    \
+	for i in $(filter %.h,$^); do                                         \
+	    echo "#include "\"$$i\"                                           \
 	    | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__        \
 	      -include stdint.h -include $(srcdir)/public/xen.h               \
-	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \
+	      `get_prereq $$i`                                                \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];) \
+	    || exit $$?; echo $$i >> [email protected]; done;                            \
 	mv [email protected] $@
 endef
 
Signature · 1 line
Anthony PERARD
Reviewed-by
#11 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

Hi Antony,

Quoted 22 lines · Anthony PERARD
Quoted 19 lines · Bertrand Marquis

Hi,

Quoted 16 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

Quoted 5 lines
Quoted 3 lines

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Will the following patch help?

Yes it does, thanks a lot.

You can add my:

Cheers
Bertrand

Quoted 65 lines

From 0f32f749304b233c0d5574dc6b14f66e8709feba Mon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chk

The command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.

Error messages:
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".

Fixes 28e13c7f43 ("build: xen/include: use if_changed")

---
xen/include/Makefile | 17 +++++++++++++----

1 file changed, 13 insertions(+), 4 deletions(-)
xen/include/Makefile +13 -4
index 6d9bcc19b0..ca5e868f38 100644
@@ -158,13 +158,22 @@ define cmd_headerscxx_chk
 	    touch [email protected];                                                     \
 	    exit 0;                                                           \
 	fi;                                                                   \
-	$(foreach i, $(filter %.h,$^),                                        \
-	    echo "#include "\"$(i)\"                                          \
+	get_prereq() {                                                        \
+	    case $$1 in                                                       \
+	    $(foreach i, $(filter %.h,$^),                                    \
+	    $(if $($(patsubst $(srctree)/%,%,$i)-prereq),                     \
+		$(patsubst $(srctree)/%,%,$i)$(close)                         \
+		echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq),   \
+			-include c$(j))";;))                                  \
+	    esac;                                                             \
+	};                                                                    \
+	for i in $(filter %.h,$^); do                                         \
+	    echo "#include "\"$$i\"                                           \
 	    | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__        \
 	      -include stdint.h -include $(srcdir)/public/xen.h               \
-	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \
+	      `get_prereq $$i`                                                \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];) \
+	    || exit $$?; echo $$i >> [email protected]; done;                            \
 	mv [email protected] $@

endef

Signature · 1 line
Anthony PERARD
#12 lore raw

Re: [XEN PATCH 1/4] build: xen/include: use if_changed

In reply to Anthony PERARD

Hi Anthony,

Quoted 85 lines · Anthony PERARD
Quoted 14 lines · Bertrand Marquis

Hi,

Quoted 12 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis

With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Not quite sure yet why but I wanted to signal it early as other might be impacted.

Arm and arm64 builds are not impacted.

Quoted 8 lines
Quoted 4 lines

Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.

I have the problem only when building using Yocto, I did a normal build and the issue is not coming.

Will the following patch help?

From 0f32f749304b233c0d5574dc6b14f66e8709feba Mon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chk

The command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.

Error messages:
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127

Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".

Fixes 28e13c7f43 ("build: xen/include: use if_changed")

---
xen/include/Makefile | 17 +++++++++++++----

1 file changed, 13 insertions(+), 4 deletions(-)
xen/include/Makefile +13 -4
index 6d9bcc19b0..ca5e868f38 100644
@@ -158,13 +158,22 @@ define cmd_headerscxx_chk
 	    touch [email protected];                                                     \
 	    exit 0;                                                           \
 	fi;                                                                   \
-	$(foreach i, $(filter %.h,$^),                                        \
-	    echo "#include "\"$(i)\"                                          \
+	get_prereq() {                                                        \
+	    case $$1 in                                                       \
+	    $(foreach i, $(filter %.h,$^),                                    \
+	    $(if $($(patsubst $(srctree)/%,%,$i)-prereq),                     \
+		$(patsubst $(srctree)/%,%,$i)$(close)                         \
+		echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq),   \
+			-include c$(j))";;))                                  \
+	    esac;                                                             \
+	};                                                                    \
+	for i in $(filter %.h,$^); do                                         \
+	    echo "#include "\"$$i\"                                           \
 	    | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__        \
 	      -include stdint.h -include $(srcdir)/public/xen.h               \
-	      $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \
+	      `get_prereq $$i`                                                \
 	      -S -o /dev/null -                                               \
-	    || exit $$?; echo $(i) >> [email protected];) \
+	    || exit $$?; echo $$i >> [email protected]; done;                            \
 	mv [email protected] $@
Quoted 2 lines

endef

Just a small reminder that you need to push this patch officially :-)

Cheers
Bertrand

#13 lore raw

PATCH 2/4 build: set PERL

In reply to Anthony PERARD

We are going to use it in a moment.

---
 xen/Makefile | 1 +
 1 file changed, 1 insertion(+)
xen/Makefile +1 -0
index 82f5310b12..a6650a2acc 100644
@@ -22,6 +22,7 @@ PYTHON_INTERPRETER	:= $(word 1,$(shell which python3 python python2 2>/dev/null)
 export PYTHON		?= $(PYTHON_INTERPRETER)
 
 export CHECKPOLICY	?= checkpolicy
+export PERL		?= perl
 
 $(if $(filter __%, $(MAKECMDGOALS)), \
     $(error targets prefixed with '__' are only for internal use))
Signature · 1 line
Anthony PERARD
#14 lore raw

Re: [XEN PATCH 2/4] build: set PERL

xen/Makefile PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) · line 25

Quoted 7 lines · Anthony PERARD
xen/Makefile +1 -0
@@ -22,6 +22,7 @@ PYTHON_INTERPRETER	:= $(word 1,$(shell which python3 python python2 2>/dev/null)
 export PYTHON		?= $(PYTHON_INTERPRETER)
 
 export CHECKPOLICY	?= checkpolicy
+export PERL		?= perl

For the intended use, is there a minimum version requirement? If so, it needs documenting in ./README (and it preferably wouldn't be any newer than from around the times our other dependencies are). And even when the uses are fully backwards compatible, I think the need for the tool wants mentioning there.

Jan

#15 lore raw

Re: [XEN PATCH 2/4] build: set PERL

Quoted 6 lines · Jan Beulich
Quoted 5 lines · Anthony PERARD
xen/Makefile +0 -0
@@ -22,6 +22,7 @@ PYTHON_INTERPRETER	:= $(word 1,$(shell which python3 python python2 2>/dev/null)
 export PYTHON		?= $(PYTHON_INTERPRETER)
 
Quoted 8 lines

xen/Makefile PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) · line 25

Quoted 2 lines
xen/Makefile +1 -0
 export CHECKPOLICY	?= checkpolicy
+export PERL		?= perl

For the intended use, is there a minimum version requirement? If so, it needs documenting in ./README (and it preferably wouldn't be any newer than from around the times our other dependencies are). And even when the uses are fully backwards compatible, I think the need for the tool wants mentioning there.

I don't think there's a minimum version. The script works in our Gitlab CI, or at least the builds don't break.

Yes, it would be better to document the tool, I'll add it to the README. (We already use it in the toolstack, at least for libxl, so it was at least partially needed before.)

Thanks,

Signature · 1 line
Anthony PERARD
#16 lore raw

PATCH 3/4 build: replace get-fields.sh by a perl script

In reply to Anthony PERARD

The get-fields.sh which generate all the include/compat/.xlat/*.h headers is quite slow. It takes for example nearly 3 seconds to generate platform.h on a recent machine, or 2.3 seconds for memory.h.

Since it's only text processing, rewriting the mix of shell/sed/python into a single perl script make the generation of those file a lot faster.

I tried to keep a similar look for the code, to keep the code similar between the shell and perl, and to ease review. So some code in perl might look weird or could be written better.

---
 xen/include/Makefile         |   6 +-
 xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++
 xen/tools/get-fields.sh      | 528 ----------------------------------
 3 files changed, 541 insertions(+), 532 deletions(-)
 create mode 100755 xen/tools/compat-xlat-header
 delete mode 100644 xen/tools/get-fields.sh
xen/include/Makefile +2 -4
index 6d9bcc19b0..b7e7148665 100644
@@ -60,9 +60,7 @@ cmd_compat_c = \
 
 quiet_cmd_xlat_headers = GEN     $@
 cmd_xlat_headers = \
-    while read what name; do \
-        $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \
-    done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \
+    $(PERL) $(srctree)/tools/compat-xlat-header $< $(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst > [email protected]; \
     mv -f [email protected] $@
 
 targets += $(headers-y)
@@ -80,7 +78,7 @@ $(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat-
 	$(call if_changed,compat_c)
 
 targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y))
-$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE
+$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/compat-xlat-header FORCE
 	$(call if_changed,xlat_headers)
 
 quiet_cmd_xlat_lst = GEN     $@
xen/tools/compat-xlat-header new file +539 -0

539 lines not on this page · raw message

xen/tools/get-fields.sh deleted +0 -528

528 lines not on this page · raw message

Signature · 1 line
Anthony PERARD
#17 lore raw

Re: [XEN PATCH 3/4] build: replace get-fields.sh by a perl script

xen/tools/compat-xlat-header line 4

Quoted 10 lines · Anthony PERARD
xen/tools/compat-xlat-header new file +4 -0
new file mode 100755
index 0000000000..f1f42a9dde
@@ -0,0 +1,539 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;

I hope to take more of a look at this, but one thing I immediately
notice: -w is redundant with "use warnings;". I strongly prefer "use warnings;", but others may have different preferences.

Signature · 4 lines
(\___(\___(\______          --=> 8-) EHM <=--          ______/)___/)___/)
 \BS (    |         [email protected]  PGP 87145445         |    )   /
  \_CS\   |  _____  -O #include <stddisclaimer.h> O-   _____  |   /  _/
8A19\___\_|_/58D2 7E3D DDF4 7BA6 <-PGP-> 41D1 B375 37D0 8714\_|_/___/5445
#18 lore raw

Re: [XEN PATCH 3/4] build: replace get-fields.sh by a perl script

Quoted 15 lines · Elliott Mitchell

xen/tools/compat-xlat-header line 4

Quoted 10 lines · Anthony PERARD
xen/tools/compat-xlat-header new file +4 -0
new file mode 100755
index 0000000000..f1f42a9dde
@@ -0,0 +1,539 @@
+#!/usr/bin/perl -w
+
+use strict;
+use warnings;

I hope to take more of a look at this, but one thing I immediately
notice: -w is redundant with "use warnings;". I strongly prefer "use warnings;", but others may have different preferences.

Sounds good, I might have copy the shebang and the "use*" from an other script in our repo, without checking what the -w stand for.

Thanks,

Signature · 1 line
Anthony PERARD
#19 lore raw

PATCH 4/4 build: remove auto.conf prerequisite from compat/xlat.h target

In reply to Anthony PERARD

Now that the command line generating "xlat.h" is check on rebuild, the header will be regenerated whenever the list of xlat headers changes due to change in ".config". We don't need to force a regeneration for every changes in ".config".

---
 xen/include/Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)
xen/include/Makefile +1 -1
index b7e7148665..937a8bc884 100644
@@ -99,7 +99,7 @@ cmd_xlat_h = \
 	cat $(filter %.h,$^) >[email protected]; \
 	mv -f [email protected] $@
 
-$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE
+$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) FORCE
 	$(call if_changed,xlat_h)
 
 ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH))
Signature · 1 line
Anthony PERARD
Acked-by on v1 4/4
#20 lore raw

Re: [XEN PATCH 4/4] build: remove auto.conf prerequisite from compat/xlat.h target

Quoted 4 lines · Anthony PERARD

Now that the command line generating "xlat.h" is check on rebuild, the header will be regenerated whenever the list of xlat headers changes due to change in ".config". We don't need to force a regeneration for every changes in ".config".

This looks to be dependent on only patch 1; I wonder if it shouldn't be viewed as an integral part of that adjustment. Anyway - if you want to keep it on its own:

Jan

#21 lore raw
In reply to Anthony PERARD
Quoted 19 lines · Anthony PERARD

Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1

Hi,

This patch series is about 2 improvement. First one is to use $(if_changed, ) in "include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.

Thanks.

Anthony PERARD (4):
build: xen/include: use if_changed
build: set PERL
build: replace get-fields.sh by a perl script
build: remove auto.conf prerequisite from compat/xlat.h target

Quoted 4 lines
 xen/Makefile                 |   1 +
 xen/include/Makefile         | 106 ++++---
 xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++
 xen/tools/get-fields.sh      | 528 ----------------------------------

Excellent.  I was planning to ask you about this.  (I also need to refreshing my half series cleaning up other bits of the build.)

One trivial observation is that it would probably be nicer to name the script with a .pl extension.

Any numbers on what the speedup in patch 3 is?

Are the generated compat headers identical before and after this series?  If yes, I'm very tempted to ack and commit it straight away.

~Andrew

#22 lore raw
Quoted 25 lines · Andrew Cooper
Quoted 19 lines · Anthony PERARD

Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1

Hi,

This patch series is about 2 improvement. First one is to use $(if_changed, ) in "include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.

Thanks.

Anthony PERARD (4):
build: xen/include: use if_changed
build: set PERL
build: replace get-fields.sh by a perl script
build: remove auto.conf prerequisite from compat/xlat.h target

Quoted 4 lines
 xen/Makefile                 |   1 +
 xen/include/Makefile         | 106 ++++---
 xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++
 xen/tools/get-fields.sh      | 528 ----------------------------------
Quoted 5 lines

Excellent.  I was planning to ask you about this.  (I also need to refreshing my half series cleaning up other bits of the build.)

One trivial observation is that it would probably be nicer to name the script with a .pl extension.

Sound fine, I don't think it matter much here.

Quoted 1 line

Any numbers on what the speedup in patch 3 is?

Yes, on my machine when doing a full build, with ccache enabled, it saves about 1.17 seconds (out of ~17s), and without ccache, it saves about 2.0 seconds (out of ~37s).

Without ccache:

before:

    $ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null >/dev/null; rm -r build; done
    make --no-print-directory -j$(nproc) -s O=build > /dev/null  244.98s user 29.24s system 683% cpu 40.146 total
    make --no-print-directory -j$(nproc) -s O=build > /dev/null  47.05s user 11.50s system 332% cpu 17.610 total
    make --no-print-directory -j$(nproc) -s O=build > /dev/null  47.35s user 11.22s system 330% cpu 17.733 total
    make --no-print-directory -j$(nproc) -s O=build > /dev/null  47.31s user 11.23s system 333% cpu 17.577 total

after:

    $ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  237.28s user 25.97s system 667% cpu 39.456 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  37.60s user 8.20s system 282% cpu 16.214 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  37.95s user 8.67s system 279% cpu 16.651 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  38.02s user 8.40s system 280% cpu 16.545 total

And without ccache:

before:

    $ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  206.37s user 22.19s system 640% cpu 35.695 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  221.45s user 22.26s system 667% cpu 36.537 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  233.95s user 23.80s system 686% cpu 37.518 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  234.27s user 23.83s system 680% cpu 37.923 total

after:

    $ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  198.62s user 18.64s system 642% cpu 33.841 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  202.91s user 19.46s system 655% cpu 33.912 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  224.42s user 20.89s system 680% cpu 36.025 total
    make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null  222.93s user 21.29s system 683% cpu 35.708 total
Quoted 2 lines

Are the generated compat headers identical before and after this series?  If yes, I'm very tempted to ack and commit it straight away.

Yes, the headers are identical. Hopefully, I've managed to check with all compat headers enabled, since that depends on .config.

Cheers,

Signature · 1 line
Anthony PERARD