xen: rework compat headers generation
cover xen: rework compat headers generation
Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1
Hi,
This patch series is about 2 improvement. First one is to use $(if_changed, ) in "include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.
Thanks.
Anthony PERARD (4):
build: xen/include: use if_changed
build: set PERL
build: replace get-fields.sh by a perl script
build: remove auto.conf prerequisite from compat/xlat.h target
xen/Makefile | 1 + xen/include/Makefile | 106 ++++--- xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++ xen/tools/get-fields.sh | 528 ---------------------------------- 4 files changed, 614 insertions(+), 560 deletions(-) create mode 100755 xen/tools/compat-xlat-header delete mode 100644 xen/tools/get-fields.sh
Signature · 1 line
Anthony PERARD
PATCH 1/4 build: xen/include: use if_changed
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them make deletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.
Removing the call to mkdir from the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).
Remove "export PYTHON" as it is already exported.
--- xen/include/Makefile | 108 ++++++++++++++++++++++++++++++------------- 1 file changed, 76 insertions(+), 32 deletions(-)
xen/include/Makefile +76 -32
@@ -45,38 +45,65 @@ public-$(CONFIG_ARM) := $(wildcard $(srcdir)/public/arch-arm/*.h $(srcdir)/publi .PHONY: all all: $(addprefix $(obj)/,$(headers-y)) -$(obj)/compat/%.h: $(obj)/compat/%.i $(srcdir)/Makefile $(srctree)/tools/compat-build-header.py - $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \ - mv -f [email protected] $@ - -$(obj)/compat/%.i: $(obj)/compat/%.c $(srcdir)/Makefile - $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $< - -$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srcdir)/Makefile $(srctree)/tools/compat-build-source.py - mkdir -p $(@D) - $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected] - mv -f [email protected] $@ - -$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh $(srcdir)/Makefile - export PYTHON=$(PYTHON); \ - while read what name; do \ - $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \ - done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected] - mv -f [email protected] $@ +quiet_cmd_compat_h = GEN $@ +cmd_compat_h = \ + $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \ + mv -f [email protected] $@ + +quiet_cmd_compat_i = CPP $@ +cmd_compat_i = $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $< + +quiet_cmd_compat_c = GEN $@ +cmd_compat_c = \ + $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]; \ + mv -f [email protected] $@ + +quiet_cmd_xlat_headers = GEN $@ +cmd_xlat_headers = \ + while read what name; do \ + $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \ + done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \ + mv -f [email protected] $@ + +targets += $(headers-y) +$(obj)/compat/%.h: $(obj)/compat/%.i $(srctree)/tools/compat-build-header.py FORCE + $(call if_changed,compat_h) + +.PRECIOUS: $(obj)/compat/%.i +targets += $(patsubst %.h, %.i, $(headers-y)) +$(obj)/compat/%.i: $(obj)/compat/%.c FORCE + $(call if_changed,compat_i) + +.PRECIOUS: $(obj)/compat/%.c +targets += $(patsubst %.h, %.c, $(headers-y)) +$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat-build-source.py FORCE + $(call if_changed,compat_c) + +targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y)) +$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE + $(call if_changed,xlat_headers) + +quiet_cmd_xlat_lst = GEN $@ +cmd_xlat_lst = \ + grep -v '^[[:blank:]]*$(pound)' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]; \ + $(call move-if-changed,[email protected],$@) .PRECIOUS: $(obj)/compat/.xlat/%.lst -$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst $(srcdir)/Makefile - mkdir -p $(@D) - grep -v '^[[:blank:]]*#' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected] - $(call move-if-changed,[email protected],$@) +targets += $(patsubst compat/%.h, compat/.xlat/%.lst, $(headers-y)) +$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst FORCE + $(call if_changed,xlat_lst) xlat-y := $(shell sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,^[?!][[:blank:]]+[^[:blank:]]+[[:blank:]]+,,p' $(srcdir)/xlat.lst | uniq) xlat-y := $(filter $(patsubst compat/%,%,$(headers-y)),$(xlat-y)) -$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf $(srcdir)/Makefile - cat $(filter %.h,$^) >[email protected] +quiet_cmd_xlat_h = GEN $@ +cmd_xlat_h = \ + cat $(filter %.h,$^) >[email protected]; \ mv -f [email protected] $@ +$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE + $(call if_changed,xlat_h) + ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH)) all: $(obj)/headers.chk $(obj)/headers99.chk $(obj)/headers++.chk
@@ -102,27 +129,31 @@ PUBLIC_C99_HEADERS := $(call public-filter-headers,public-c99-headers) $(src)/public/io/9pfs.h-prereq := string $(src)/public/io/pvcalls.h-prereq := string -$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) $(srcdir)/Makefile +quiet_cmd_header_chk = CHK $@ +cmd_header_chk = \ for i in $(filter %.h,$^); do \ $(CC) -x c -ansi -Wall -Werror -include stdint.h \ -S -o /dev/null $$i || exit 1; \ echo $$i; \ - done >[email protected] + done >[email protected]; \ mv [email protected] $@ -$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) $(srcdir)/Makefile - rm -f [email protected] +quiet_cmd_headers99_chk = CHK $@ +define cmd_headers99_chk + rm -f [email protected]; \ $(foreach i, $(filter %.h,$^), \ echo "#include "\"$(i)\" \ | $(CC) -x c -std=c99 -Wall -Werror \ -include stdint.h \ $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include $(j).h) \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) + || exit $$?; echo $(i) >> [email protected];) \ mv [email protected] $@ +endef -$(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile - rm -f [email protected] +quiet_cmd_headerscxx_chk = CHK $@ +define cmd_headerscxx_chk + rm -f [email protected]; \ if ! $(CXX) -v >/dev/null 2>&1; then \ touch [email protected]; \ exit 0; \
@@ -133,8 +164,21 @@ $(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile -include stdint.h -include $(srcdir)/public/xen.h \ $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) + || exit $$?; echo $(i) >> [email protected];) \ mv [email protected] $@ +endef + +targets += headers.chk +$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) FORCE + $(call if_changed,header_chk) + +targets += headers99.chk +$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) FORCE + $(call if_changed,headers99_chk) + +targets += headers++.chk +$(obj)/headers++.chk: $(PUBLIC_HEADERS) FORCE + $(call if_changed,headerscxx_chk) endif
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Quoted 3 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
In cmd_xlat_lst you use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.
Jan
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Quoted 8 lines · Jan Beulich
Quoted 3 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
In
cmd_xlat_lstyou use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.
I think I just forgot about $(pound) when I tried to debug why the command didn't work in some environment (that is when not using define).
I think the second thing that make me not replacing '#' by "$(pound)" is that reading "#include" in the Makefile is probably better that reading "$(pound)include".
I guess we should add something to the justification like:
That allow us to keep writing "#include" in the Makefile without having to replace that by "$(pound)include" which would be a bit less obvious about the command line purpose.
Thanks,
Signature · 1 line
Anthony PERARD
Jan Beulich
Acked-byRe: [XEN PATCH 1/4] build: xen/include: use if_changed
Quoted 13 lines · Anthony PERARD
Quoted 8 lines · Jan Beulich
Quoted 3 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
In
cmd_xlat_lstyou use $(pound) - any reason this doesn't work in these rules? Note that I don't mind the use of "define", just that I'm puzzled by the justification.I think I just forgot about $(pound) when I tried to debug why the command didn't work in some environment (that is when not using define).
Quoted 8 lines
I think the second thing that make me not replacing '#' by "$(pound)" is that reading "#include" in the
Makefileis probably better that reading "$(pound)include".I guess we should add something to the justification like:
That allow us to keep writing "#include" in theMakefilewithout having to replace that by "$(pound)include" which would be a bit less obvious about the command line purpose.
I'd be okay with this, and I'd also be okay with adding this while committing. With it added
Jan
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
In reply to Anthony PERARDHi Anthony,
Quoted 11 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them
makedeletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.
Quoted 5 lines
Removing the call to
mkdirfrom the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).Remove "export PYTHON" as it is already exported.
With this change, compiling for x86 is now ending up in:
CHK include/headers99.chk
make[9]: execvp: /bin/sh: Argument list too long
make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127
Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Cheers
Bertrand
Quoted 166 lines
Signed-off-by Anthony PERARD <[email protected]>---
xen/include/Makefile | 108 ++++++++++++++++++++++++++++++-------------1 file changed, 76 insertions(+), 32 deletions(-)xen/include/Makefile +0
-0@@ -45,38 +45,65 @@ public-$(CONFIG_ARM) := $(wildcard $(srcdir)/public/arch-arm/*.h $(srcdir)/publi.PHONY: all
all: $(addprefix $(obj)/,$(headers-y))-$(obj)/compat/%.h: $(obj)/compat/%.i $(srcdir)/Makefile $(srctree)/tools/compat-build-header.py
xen/include/Makefile +42
-17- $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \ - mv -f [email protected] $@ - -$(obj)/compat/%.i: $(obj)/compat/%.c $(srcdir)/Makefile - $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $< - -$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srcdir)/Makefile $(srctree)/tools/compat-build-source.py - mkdir -p $(@D) - $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected] - mv -f [email protected] $@ - -$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh $(srcdir)/Makefile - export PYTHON=$(PYTHON); \ - while read what name; do \ - $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \ - done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected] - mv -f [email protected] $@ +quiet_cmd_compat_h = GEN $@ +cmd_compat_h = \ + $(PYTHON) $(srctree)/tools/compat-build-header.py <$< $(patsubst $(obj)/%,%,$@) >>[email protected]; \ + mv -f [email protected] $@ + +quiet_cmd_compat_i = CPP $@ +cmd_compat_i = $(CPP) $(filter-out -Wa$(comma)% -include %/include/xen/config.h,$(XEN_CFLAGS)) $(cppflags-y) -o $@ $< + +quiet_cmd_compat_c = GEN $@ +cmd_compat_c = \ + $(PYTHON) $(srctree)/tools/compat-build-source.py $(srcdir)/xlat.lst <$< >[email protected]; \ + mv -f [email protected] $@ + +quiet_cmd_xlat_headers = GEN $@ +cmd_xlat_headers = \ + while read what name; do \ + $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \ + done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \ + mv -f [email protected] $@ + +targets += $(headers-y) +$(obj)/compat/%.h: $(obj)/compat/%.i $(srctree)/tools/compat-build-header.py FORCE + $(call if_changed,compat_h) + +.PRECIOUS: $(obj)/compat/%.i +targets += $(patsubst %.h, %.i, $(headers-y)) +$(obj)/compat/%.i: $(obj)/compat/%.c FORCE + $(call if_changed,compat_i) + +.PRECIOUS: $(obj)/compat/%.c +targets += $(patsubst %.h, %.c, $(headers-y)) +$(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat-build-source.py FORCE + $(call if_changed,compat_c) + +targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y)) +$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE + $(call if_changed,xlat_headers) + +quiet_cmd_xlat_lst = GEN $@ +cmd_xlat_lst = \ + grep -v '^[[:blank:]]*$(pound)' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected]; \ + $(call move-if-changed,[email protected],$@).PRECIOUS: $(obj)/compat/.xlat/%.lst -$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst $(srcdir)/Makefile
xen/include/Makefile +3
-3- mkdir -p $(@D) - grep -v '^[[:blank:]]*#' $< | sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,[[:blank:]]+$*\.h[[:blank:]]*$$,,p' >[email protected] - $(call move-if-changed,[email protected],$@) +targets += $(patsubst compat/%.h, compat/.xlat/%.lst, $(headers-y)) +$(obj)/compat/.xlat/%.lst: $(srcdir)/xlat.lst FORCE + $(call if_changed,xlat_lst)xlat-y := $(shell sed -ne 's,@arch@,$(compat-arch-y),g' -re 's,^[?!][[:blank:]]+[^[:blank:]]+[[:blank:]]+,,p' $(srcdir)/xlat.lst | uniq) xlat-y := $(filter $(patsubst compat/%,%,$(headers-y)),$(xlat-y))
-$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf $(srcdir)/Makefile
xen/include/Makefile +3
-1- cat $(filter %.h,$^) >[email protected] +quiet_cmd_xlat_h = GEN $@ +cmd_xlat_h = \ + cat $(filter %.h,$^) >[email protected]; \mv -f [email protected] $@+$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE
xen/include/Makefile +2
-0+ $(call if_changed,xlat_h) +ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH))
all: $(obj)/headers.chk $(obj)/headers99.chk $(obj)/headers++.chk
xen/include/Makefile +0
-0@@ -102,27 +129,31 @@ PUBLIC_C99_HEADERS := $(call public-filter-headers,public-c99-headers)$(src)/public/io/9pfs.h-prereq := string
$(src)/public/io/pvcalls.h-prereq := string-$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) $(srcdir)/Makefile
xen/include/Makefile +2
-0+quiet_cmd_header_chk = CHK $@ +cmd_header_chk = \for i in $(filter %.h,$^); do \ $(CC) -x c -ansi -Wall -Werror -include stdint.h \ -S -o /dev/null $$i || exit 1; \ echo $$i; \xen/include/Makefile +1
-1- done >[email protected] + done >[email protected]; \mv [email protected] $@-$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) $(srcdir)/Makefile
xen/include/Makefile +3
-1- rm -f [email protected] +quiet_cmd_headers99_chk = CHK $@ +define cmd_headers99_chk + rm -f [email protected]; \$(foreach i, $(filter %.h,$^), \ echo "#include "\"$(i)\" \ | $(CC) -x c -std=c99 -Wall -Werror \ -include stdint.h \ $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include $(j).h) \ -S -o /dev/null - \xen/include/Makefile +1
-1- || exit $$?; echo $(i) >> [email protected];) + || exit $$?; echo $(i) >> [email protected];) \mv [email protected] $@xen/include/Makefile +4
-2+endef -$(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile - rm -f [email protected] +quiet_cmd_headerscxx_chk = CHK $@ +define cmd_headerscxx_chk + rm -f [email protected]; \if ! $(CXX) -v >/dev/null 2>&1; then \ touch [email protected]; \ exit 0; \xen/include/Makefile +14
-1@@ -133,8 +164,21 @@ $(obj)/headers++.chk: $(PUBLIC_HEADERS) $(srcdir)/Makefile -include stdint.h -include $(srcdir)/public/xen.h \ $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) + || exit $$?; echo $(i) >> [email protected];) \ mv [email protected] $@ +endef + +targets += headers.chk +$(obj)/headers.chk: $(PUBLIC_ANSI_HEADERS) FORCE + $(call if_changed,header_chk) + +targets += headers99.chk +$(obj)/headers99.chk: $(PUBLIC_C99_HEADERS) FORCE + $(call if_changed,headers99_chk) + +targets += headers++.chk +$(obj)/headers++.chk: $(PUBLIC_HEADERS) FORCE + $(call if_changed,headerscxx_chk)endif
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Quoted 17 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them
makedeletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.Quoted 5 lines
Removing the call to
mkdirfrom the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).Remove "export PYTHON" as it is already exported.
Quoted 8 lines
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
Also please remember to trim your replies.
Jan
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Hi,
Quoted 25 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them
makedeletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.Quoted 5 lines
Removing the call to
mkdirfrom the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).Remove "export PYTHON" as it is already exported.
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127
Quoted 8 lines
Quoted 3 lines
Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Doing a verbose compilation I have this (sorry for the long lines):
for i in include/public/vcpu.h include/public/errno.h include/public/kexec.h include/public/argo.h include/public/xen.h include/public/nmi.h include/public/xencomm.h include/public/xenoprof.h include/public/device_tree_defs.h include/public/version.h include/public/memory.h include/public/features.h include/public/sched.h include/public/xen-compat.h include/public/callback.h include/public/vm_event.h include/public/grant_table.h include/public/physdev.h include/public/tmem.h include/public/hypfs.h include/public/platform.h include/public/pmu.h include/public/elfnote.h include/public/trace.h include/public/event_channel.h include/public/io/vscsiif.h include/public/io/kbdif.h include/public/io/protocols.h include/public/io/ring.h include/public/io/displif.h include/public/io/fsif.h include/public/io/blkif.h include/public/io/console.h include/public/io/sndif.h include/public/io/fbif.h include/public/io/libxenvchan.h include/public/io/netif.h include/public/io/usbif.h include/public/io/pciif.h include/public/io/tpmif.h include/public/io/xs_wire.h include/public/io/xenbus.h include/public/io/cameraif.h include/public/hvm/pvdrivers.h include/public/hvm/e820.h include/public/hvm/hvm_xs_strings.h include/public/hvm/dm_op.h include/public/hvm/ioreq.h include/public/hvm/hvm_info_table.h include/public/hvm/hvm_vcpu.h include/public/hvm/hvm_op.h include/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -include stdint.h -S -o /dev/null $i || exit 1; echo $i; done >include/headers.chk.new; mv include/headers.chk.new include/headers.chk
| rm -f include/headers99.chk.new; echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new; echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk | make[9]: execvp: /bin/sh: Argument list too long | make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127 | make[9]: *** Waiting for unfinished jobs....
So the command passed to the sub shell by make is quite long.
No idea why this comes out only when building in Yocto but I will dig a bit.
Quoted 3 lines
Also please remember to trim your replies.
Will do.
Bertrand
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Quoted 39 lines · Bertrand Marquis
Quoted 24 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them
makedeletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.Quoted 5 lines
Removing the call to
mkdirfrom the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).Remove "export PYTHON" as it is already exported.
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Quoted 8 lines
Quoted 3 lines
Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Doing a verbose compilation I have this (sorry for the long lines):
for i in
include/public/vcpu.hinclude/public/errno.hinclude/public/kexec.hinclude/public/argo.hinclude/public/xen.hinclude/public/nmi.hinclude/public/xencomm.hinclude/public/xenoprof.hinclude/public/device_tree_defs.hinclude/public/version.hinclude/public/memory.hinclude/public/features.hinclude/public/sched.hinclude/public/xen-compat.hinclude/public/callback.hinclude/public/vm_event.hinclude/public/grant_table.hinclude/public/physdev.hinclude/public/tmem.hinclude/public/hypfs.hinclude/public/platform.hinclude/public/pmu.hinclude/public/elfnote.hinclude/public/trace.hinclude/public/event_channel.hinclude/public/io/vscsiif.hinclude/public/io/kbdif.hinclude/public/io/protocols.hinclude/public/io/ring.hinclude/public/io/displif.hinclude/public/io/fsif.hinclude/public/io/blkif.hinclude/public/io/console.hinclude/public/io/sndif.hinclude/public/io/fbif.hinclude/public/io/libxenvchan.hinclude/public/io/netif.hinclude/public/io/usbif.hinclude/public/io/pciif.hinclude/public/io/tpmif.hinclude/public/io/xs_wire.hinclude/public/io/xenbus.hinclude/public/io/cameraif.hinclude/public/hvm/pvdrivers.hinclude/public/hvm/e820.hinclude/public/hvm/hvm_xs_strings.hinclude/public/hvm/dm_op.hinclude/public/hvm/ioreq.hinclude/public/hvm/hvm_info_table.hinclude/public/hvm/hvm_vcpu.hinclude/public/hvm/hvm_op.hinclude/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -includestdint.h-S -o/dev/null$i || exit 1; echo $i; done >include/headers.chk.new; mvinclude/headers.chk.newinclude/headers.chk
Quoted 8 lines
| rm -f include/headers99.chk.new; echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new; echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk | make[9]: execvp: /bin/sh: Argument list too long | make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127 | make[9]: *** Waiting for unfinished jobs....So the command passed to the sub shell by make is quite long.
No idea why this comes out only when building in Yocto but I will dig a bit.
Maybe Yocto has an unusually low limit on command arguments' total size? The whole thing is just over 2500 chars, which doesn't look to be unusually long for Unix-like environments.
Jan
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
Hi,
Quoted 47 lines · Jan Beulich
Quoted 44 lines · Bertrand Marquis
Quoted 24 lines · Jan Beulich
Quoted 22 lines · Bertrand Marquis
Quoted 11 lines · Anthony PERARD
Use "define" for the headers*_chk commands as otherwise the "#" is interpreted as a comment and make can't find the end of $(foreach,).
Adding several .PRECIOUS as without them
makedeletes the intermediate targets. This is an issue because the macro $(if_changed,) check if the target exist in order to decide whether to recreate the target.Quoted 5 lines
Removing the call to
mkdirfrom the commands. Those aren't needed anymore because a rune in Rules.mk creates the directory for each $(targets).Remove "export PYTHON" as it is already exported.
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Quoted 8 lines
Quoted 3 lines
Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Doing a verbose compilation I have this (sorry for the long lines):
for i in
include/public/vcpu.hinclude/public/errno.hinclude/public/kexec.hinclude/public/argo.hinclude/public/xen.hinclude/public/nmi.hinclude/public/xencomm.hinclude/public/xenoprof.hinclude/public/device_tree_defs.hinclude/public/version.hinclude/public/memory.hinclude/public/features.hinclude/public/sched.hinclude/public/xen-compat.hinclude/public/callback.hinclude/public/vm_event.hinclude/public/grant_table.hinclude/public/physdev.hinclude/public/tmem.hinclude/public/hypfs.hinclude/public/platform.hinclude/public/pmu.hinclude/public/elfnote.hinclude/public/trace.hinclude/public/event_channel.hinclude/public/io/vscsiif.hinclude/public/io/kbdif.hinclude/public/io/protocols.hinclude/public/io/ring.hinclude/public/io/displif.hinclude/public/io/fsif.hinclude/public/io/blkif.hinclude/public/io/console.hinclude/public/io/sndif.hinclude/public/io/fbif.hinclude/public/io/libxenvchan.hinclude/public/io/netif.hinclude/public/io/usbif.hinclude/public/io/pciif.hinclude/public/io/tpmif.hinclude/public/io/xs_wire.hinclude/public/io/xenbus.hinclude/public/io/cameraif.hinclude/public/hvm/pvdrivers.hinclude/public/hvm/e820.hinclude/public/hvm/hvm_xs_strings.hinclude/public/hvm/dm_op.hinclude/public/hvm/ioreq.hinclude/public/hvm/hvm_info_table.hinclude/public/hvm/hvm_vcpu.hinclude/public/hvm/hvm_op.hinclude/public/hvm/params.h; do x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -ansi -Wall -Werror -includestdint.h-S -o/dev/null$i || exit 1; echo $i; done >include/headers.chk.new; mvinclude/headers.chk.newinclude/headers.chk| rm -f include/headers99.chk.new; echo "#include "\"include/public/io/9pfs.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/9pfs.h >> include/headers99.chk.new; echo "#include "\"include/public/io/pvcalls.h\" | x86_64-poky-linux-gcc --sysroot=/home/bermar01/Development/xen-dev/build/profile-qemu-x86_64.prj/tmp/work/core2-64-poky-linux/xen/4.17+git1-r0/recipe-sysroot -x c -std=c99 -Wall -Werror -include stdint.h -include string.h -S -o /dev/null - || exit $?; echo include/public/io/pvcalls.h >> include/headers99.chk.new; mv include/headers99.chk.new include/headers99.chk | make[9]: execvp: /bin/sh: Argument list too long | make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127 | make[9]: *** Waiting for unfinished jobs....
Quoted 8 lines
Quoted 3 lines
So the command passed to the sub shell by make is quite long.
No idea why this comes out only when building in Yocto but I will dig a bit.
Maybe Yocto has an unusually low limit on command arguments' total size? The whole thing is just over 2500 chars, which doesn't look to be unusually long for Unix-like environments.
Actually the command to generate headers++.chk is 15294 characters when I build xen normally. In Yocto it becomes a lot bigger as CC includes a sysroot parameter with a path.
Bertrand
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
In reply to Bertrand MarquisQuoted 14 lines · Bertrand Marquis
Hi,
Quoted 12 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Quoted 8 lines
Quoted 4 lines
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Will the following patch help?
From 0f32f749304b233c0d5574dc6b14f66e8709feba Mon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chk
The command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.
Error messages:
make[9]: execvp: /bin/sh: Argument list too long make[9]: *** [include/Makefile:181: include/headers++.chk] Error 127
Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".
--- xen/include/Makefile | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-)
xen/include/Makefile +13 -4
@@ -158,13 +158,22 @@ define cmd_headerscxx_chk touch [email protected]; \ exit 0; \ fi; \ - $(foreach i, $(filter %.h,$^), \ - echo "#include "\"$(i)\" \ + get_prereq() { \ + case $$1 in \ + $(foreach i, $(filter %.h,$^), \ + $(if $($(patsubst $(srctree)/%,%,$i)-prereq), \ + $(patsubst $(srctree)/%,%,$i)$(close) \ + echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), \ + -include c$(j))";;)) \ + esac; \ + }; \ + for i in $(filter %.h,$^); do \ + echo "#include "\"$$i\" \ | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__ \ -include stdint.h -include $(srcdir)/public/xen.h \ - $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \ + `get_prereq $$i` \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) \ + || exit $$?; echo $$i >> [email protected]; done; \ mv [email protected] $@ endef
Signature · 1 line
Anthony PERARD
Bertrand Marquis
Reviewed-byRe: [XEN PATCH 1/4] build: xen/include: use if_changed
Hi Antony,
Quoted 22 lines · Anthony PERARD
Quoted 19 lines · Bertrand Marquis
Hi,
Quoted 16 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
Quoted 5 lines
Quoted 3 lines
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Will the following patch help?
Yes it does, thanks a lot.
You can add my:
Cheers
Bertrand
Quoted 65 lines
From
0f32f749304b233c0d5574dc6b14f66e8709febaMon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chkThe command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.
Error messages:
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".
Reported-by Bertrand Marquis <[email protected]>Fixes 28e13c7f43 ("build: xen/include: use if_changed")Signed-off-by Anthony PERARD <[email protected]>---
xen/include/Makefile | 17 +++++++++++++----1 file changed, 13 insertions(+), 4 deletions(-)xen/include/Makefile +13
-4@@ -158,13 +158,22 @@ define cmd_headerscxx_chk touch [email protected]; \ exit 0; \ fi; \ - $(foreach i, $(filter %.h,$^), \ - echo "#include "\"$(i)\" \ + get_prereq() { \ + case $$1 in \ + $(foreach i, $(filter %.h,$^), \ + $(if $($(patsubst $(srctree)/%,%,$i)-prereq), \ + $(patsubst $(srctree)/%,%,$i)$(close) \ + echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), \ + -include c$(j))";;)) \ + esac; \ + }; \ + for i in $(filter %.h,$^); do \ + echo "#include "\"$$i\" \ | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__ \ -include stdint.h -include $(srcdir)/public/xen.h \ - $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \ + `get_prereq $$i` \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) \ + || exit $$?; echo $$i >> [email protected]; done; \ mv [email protected] $@endef
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 1/4] build: xen/include: use if_changed
In reply to Anthony PERARDHi Anthony,
Quoted 85 lines · Anthony PERARD
Quoted 14 lines · Bertrand Marquis
Hi,
Quoted 12 lines · Jan Beulich
Quoted 8 lines · Bertrand Marquis
With this change, compiling for x86 is now ending up in:
CHKinclude/headers99.chk
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Not quite sure yet why but I wanted to signal it early as other might be impacted.
Arm and arm64 builds are not impacted.
Quoted 8 lines
Quoted 4 lines
Hmm, that patch has passed the smoke push gate already, so there likely is more to it than there being an unconditional issue. I did build-test this before pushing, and I've just re-tested on a 2nd system without seeing an issue.
I have the problem only when building using Yocto, I did a normal build and the issue is not coming.
Will the following patch help?
From
0f32f749304b233c0d5574dc6b14f66e8709febaMon Sep 17 00:00:00 2001
From: Anthony PERARD <[email protected]>
Date: Thu, 9 Jun 2022 13:42:52 +0100
Subject: [XEN PATCH] build,include: rework shell script for headers++.chkThe command line generated for headers++.chk by make is quite long, and in some environment it is too long. This issue have been seen in Yocto build environment.
Error messages:
make[9]: execvp:/bin/sh: Argument list too long
make[9]: *** [include/Makefile:181:include/headers++.chk] Error 127Rework to that we do the foreach loop in shell rather that make to reduce the command line size by a lot. We also need a way to get headers prerequisite for some public headers so we use a switch "case" in shell to be able to do some simple pattern matching. Variables alone in POSIX shell don't allow to work with associative array or variables with "/".
Reported-by Bertrand Marquis <[email protected]>Fixes 28e13c7f43 ("build: xen/include: use if_changed")Signed-off-by Anthony PERARD <[email protected]>---
xen/include/Makefile | 17 +++++++++++++----1 file changed, 13 insertions(+), 4 deletions(-)xen/include/Makefile +13
-4@@ -158,13 +158,22 @@ define cmd_headerscxx_chk touch [email protected]; \ exit 0; \ fi; \ - $(foreach i, $(filter %.h,$^), \ - echo "#include "\"$(i)\" \ + get_prereq() { \ + case $$1 in \ + $(foreach i, $(filter %.h,$^), \ + $(if $($(patsubst $(srctree)/%,%,$i)-prereq), \ + $(patsubst $(srctree)/%,%,$i)$(close) \ + echo "$(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), \ + -include c$(j))";;)) \ + esac; \ + }; \ + for i in $(filter %.h,$^); do \ + echo "#include "\"$$i\" \ | $(CXX) -x c++ -std=gnu++98 -Wall -Werror -D__XEN_TOOLS__ \ -include stdint.h -include $(srcdir)/public/xen.h \ - $(foreach j, $($(patsubst $(srctree)/%,%,$i)-prereq), -include c$(j)) \ + `get_prereq $$i` \ -S -o /dev/null - \ - || exit $$?; echo $(i) >> [email protected];) \ + || exit $$?; echo $$i >> [email protected]; done; \ mv [email protected] $@
Quoted 2 lines
endef
Just a small reminder that you need to push this patch officially :-)
Cheers
Bertrand
PATCH 2/4 build: set PERL
In reply to Anthony PERARDWe are going to use it in a moment.
--- xen/Makefile | 1 + 1 file changed, 1 insertion(+)
xen/Makefile +1 -0
@@ -22,6 +22,7 @@ PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) export PYTHON ?= $(PYTHON_INTERPRETER) export CHECKPOLICY ?= checkpolicy +export PERL ?= perl $(if $(filter __%, $(MAKECMDGOALS)), \ $(error targets prefixed with '__' are only for internal use))
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 2/4] build: set PERL
xen/Makefile PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) · line 25
Quoted 7 lines · Anthony PERARD
xen/Makefile +1
-0@@ -22,6 +22,7 @@ PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) export PYTHON ?= $(PYTHON_INTERPRETER) export CHECKPOLICY ?= checkpolicy +export PERL ?= perl
For the intended use, is there a minimum version requirement? If so, it needs documenting in ./README (and it preferably wouldn't be any newer than from around the times our other dependencies are). And even when the uses are fully backwards compatible, I think the need for the tool wants mentioning there.
Jan
Re: [XEN PATCH 2/4] build: set PERL
Quoted 6 lines · Jan Beulich
Quoted 5 lines · Anthony PERARD
xen/Makefile +0
-0@@ -22,6 +22,7 @@ PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) export PYTHON ?= $(PYTHON_INTERPRETER)
Quoted 8 lines
xen/Makefile PYTHON_INTERPRETER := $(word 1,$(shell which python3 python python2 2>/dev/null) · line 25
Quoted 2 lines
xen/Makefile +1
-0export CHECKPOLICY ?= checkpolicy +export PERL ?= perlFor the intended use, is there a minimum version requirement? If so, it needs documenting in ./README (and it preferably wouldn't be any newer than from around the times our other dependencies are). And even when the uses are fully backwards compatible, I think the need for the tool wants mentioning there.
I don't think there's a minimum version. The script works in our Gitlab CI, or at least the builds don't break.
Yes, it would be better to document the tool, I'll add it to the README. (We already use it in the toolstack, at least for libxl, so it was at least partially needed before.)
Thanks,
Signature · 1 line
Anthony PERARD
PATCH 3/4 build: replace get-fields.sh by a perl script
In reply to Anthony PERARDThe get-fields.sh which generate all the include/compat/.xlat/*.h headers is quite slow. It takes for example nearly 3 seconds to generate platform.h on a recent machine, or 2.3 seconds for memory.h.
Since it's only text processing, rewriting the mix of shell/sed/python into a single perl script make the generation of those file a lot faster.
I tried to keep a similar look for the code, to keep the code similar between the shell and perl, and to ease review. So some code in perl might look weird or could be written better.
--- xen/include/Makefile | 6 +- xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++ xen/tools/get-fields.sh | 528 ---------------------------------- 3 files changed, 541 insertions(+), 532 deletions(-) create mode 100755 xen/tools/compat-xlat-header delete mode 100644 xen/tools/get-fields.sh
xen/include/Makefile +2 -4
@@ -60,9 +60,7 @@ cmd_compat_c = \ quiet_cmd_xlat_headers = GEN $@ cmd_xlat_headers = \ - while read what name; do \ - $(SHELL) $(srctree)/tools/get-fields.sh "$$what" compat_$$name $< || exit $$?; \ - done <$(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst >[email protected]; \ + $(PERL) $(srctree)/tools/compat-xlat-header $< $(patsubst $(obj)/compat/%,$(obj)/compat/.xlat/%,$(basename $<)).lst > [email protected]; \ mv -f [email protected] $@ targets += $(headers-y)
@@ -80,7 +78,7 @@ $(obj)/compat/%.c: $(src)/public/%.h $(srcdir)/xlat.lst $(srctree)/tools/compat- $(call if_changed,compat_c) targets += $(patsubst compat/%, compat/.xlat/%, $(headers-y)) -$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/get-fields.sh FORCE +$(obj)/compat/.xlat/%.h: $(obj)/compat/%.h $(obj)/compat/.xlat/%.lst $(srctree)/tools/compat-xlat-header FORCE $(call if_changed,xlat_headers) quiet_cmd_xlat_lst = GEN $@
xen/tools/compat-xlat-header new file +539 -0
539 lines not on this page · raw message
xen/tools/get-fields.sh deleted +0 -528
528 lines not on this page · raw message
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 3/4] build: replace get-fields.sh by a perl script
xen/tools/compat-xlat-header line 4
Quoted 10 lines · Anthony PERARD
xen/tools/compat-xlat-header new file +4
-0@@ -0,0 +1,539 @@ +#!/usr/bin/perl -w + +use strict; +use warnings;
I hope to take more of a look at this, but one thing I immediately
notice: -w is redundant with "use warnings;". I strongly prefer "use warnings;", but others may have different preferences.
Signature · 4 lines
(\___(\___(\______ --=> 8-) EHM <=-- ______/)___/)___/) \BS ( | [email protected] PGP 87145445 | ) / \_CS\ | _____ -O #include <stddisclaimer.h> O- _____ | / _/ 8A19\___\_|_/58D2 7E3D DDF4 7BA6 <-PGP-> 41D1 B375 37D0 8714\_|_/___/5445
Re: [XEN PATCH 3/4] build: replace get-fields.sh by a perl script
Quoted 15 lines · Elliott Mitchell
xen/tools/compat-xlat-header line 4
Quoted 10 lines · Anthony PERARD
xen/tools/compat-xlat-header new file +4
-0@@ -0,0 +1,539 @@ +#!/usr/bin/perl -w + +use strict; +use warnings;I hope to take more of a look at this, but one thing I immediately
notice: -w is redundant with "use warnings;". I strongly prefer "use warnings;", but others may have different preferences.
Sounds good, I might have copy the shebang and the "use*" from an other script in our repo, without checking what the -w stand for.
Thanks,
Signature · 1 line
Anthony PERARD
PATCH 4/4 build: remove auto.conf prerequisite from compat/xlat.h target
In reply to Anthony PERARDNow that the command line generating "xlat.h" is check on rebuild, the header will be regenerated whenever the list of xlat headers changes due to change in ".config". We don't need to force a regeneration for every changes in ".config".
--- xen/include/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
xen/include/Makefile +1 -1
@@ -99,7 +99,7 @@ cmd_xlat_h = \ cat $(filter %.h,$^) >[email protected]; \ mv -f [email protected] $@ -$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) $(obj)/config/auto.conf FORCE +$(obj)/compat/xlat.h: $(addprefix $(obj)/compat/.xlat/,$(xlat-y)) FORCE $(call if_changed,xlat_h) ifeq ($(XEN_TARGET_ARCH),$(XEN_COMPILE_ARCH))
Signature · 1 line
Anthony PERARD
Re: [XEN PATCH 4/4] build: remove auto.conf prerequisite from compat/xlat.h target
Quoted 4 lines · Anthony PERARD
Now that the command line generating "
xlat.h" is check on rebuild, the header will be regenerated whenever the list of xlat headers changes due to change in ".config". We don't need to force a regeneration for every changes in ".config".
This looks to be dependent on only patch 1; I wonder if it shouldn't be viewed as an integral part of that adjustment. Anyway - if you want to keep it on its own:
Jan
Quoted 19 lines · Anthony PERARD
Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1Hi,
This patch series is about 2 improvement. First one is to use $(if_changed, ) in "
include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.Thanks.
Anthony PERARD (4):
build: xen/include: useif_changed
build: set PERL
build: replaceget-fields.shby a perl script
build: remove auto.conf prerequisite fromcompat/xlat.htarget
Quoted 4 lines
xen/Makefile | 1 + xen/include/Makefile | 106 ++++--- xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++ xen/tools/get-fields.sh | 528 ----------------------------------
Excellent. I was planning to ask you about this. (I also need to refreshing my half series cleaning up other bits of the build.)
One trivial observation is that it would probably be nicer to name the script with a .pl extension.
Any numbers on what the speedup in patch 3 is?
Are the generated compat headers identical before and after this series? If yes, I'm very tempted to ack and commit it straight away.
~Andrew
Quoted 25 lines · Andrew Cooper
Quoted 19 lines · Anthony PERARD
Patch series available in this git branch:
https://xenbits.xen.org/git-http/people/aperard/xen-unstable.git br.build-system-xen-include-rework-v1Hi,
This patch series is about 2 improvement. First one is to use $(if_changed, ) in "
include/Makefile" to make the generation of the compat headers less verbose and to have the command line part of the decision to rebuild the headers. Second one is to replace one slow script by a much faster one, and save time when generating the headers.Thanks.
Anthony PERARD (4):
build: xen/include: useif_changed
build: set PERL
build: replaceget-fields.shby a perl script
build: remove auto.conf prerequisite fromcompat/xlat.htargetQuoted 4 lines
xen/Makefile | 1 + xen/include/Makefile | 106 ++++--- xen/tools/compat-xlat-header | 539 +++++++++++++++++++++++++++++++++++ xen/tools/get-fields.sh | 528 ----------------------------------
Quoted 5 lines
Excellent. I was planning to ask you about this. (I also need to refreshing my half series cleaning up other bits of the build.)
One trivial observation is that it would probably be nicer to name the script with a .pl extension.
Sound fine, I don't think it matter much here.
Quoted 1 line
Any numbers on what the speedup in patch 3 is?
Yes, on my machine when doing a full build, with ccache enabled, it saves about 1.17 seconds (out of ~17s), and without ccache, it saves about 2.0 seconds (out of ~37s).
Without ccache:
before:
$ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null >/dev/null; rm -r build; done
make --no-print-directory -j$(nproc) -s O=build > /dev/null 244.98s user 29.24s system 683% cpu 40.146 total
make --no-print-directory -j$(nproc) -s O=build > /dev/null 47.05s user 11.50s system 332% cpu 17.610 total
make --no-print-directory -j$(nproc) -s O=build > /dev/null 47.35s user 11.22s system 330% cpu 17.733 total
make --no-print-directory -j$(nproc) -s O=build > /dev/null 47.31s user 11.23s system 333% cpu 17.577 totalafter:
$ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 237.28s user 25.97s system 667% cpu 39.456 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 37.60s user 8.20s system 282% cpu 16.214 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 37.95s user 8.67s system 279% cpu 16.651 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 38.02s user 8.40s system 280% cpu 16.545 totalAnd without ccache:
before:
$ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 206.37s user 22.19s system 640% cpu 35.695 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 221.45s user 22.26s system 667% cpu 36.537 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 233.95s user 23.80s system 686% cpu 37.518 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 234.27s user 23.83s system 680% cpu 37.923 totalafter:
$ for i in `seq 4`; do time make -j$(nproc) -s O=build 2>/dev/null>/dev/null; rm -r build; done
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 198.62s user 18.64s system 642% cpu 33.841 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 202.91s user 19.46s system 655% cpu 33.912 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 224.42s user 20.89s system 680% cpu 36.025 total
make --no-print-directory -j$(nproc) -s O=build 2> /dev/null > /dev/null 222.93s user 21.29s system 683% cpu 35.708 totalQuoted 2 lines
Are the generated compat headers identical before and after this series? If yes, I'm very tempted to ack and commit it straight away.
Yes, the headers are identical. Hopefully, I've managed to check with all compat headers enabled, since that depends on .config.
Cheers,
Signature · 1 line
Anthony PERARD