Skip to contentSkip to conversation

kbuild: merge cmd_archive_linux and cmd_archive_perf

PATCH In review Masahiro Yamada 5 participants 12 messages Apr 7, 2023 – Apr 20, 2023 Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH 1/3 kbuild: merge cmd_archive_linux and cmd_archive_perf

The two commands, cmd_archive_linux and cmd_archive_perf, are similar. Merge them to make it easier to add more changes to the git-archive command.

---

 scripts/Makefile.package | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
scripts/Makefile.package +9 -10
index 61f72eb8d9be..a205617730c6 100644
@@ -57,16 +57,17 @@ check-git:
 		false; \
 	fi
 
+quiet_cmd_archive = ARCHIVE $@
+      cmd_archive = git -C $(srctree) archive \
+                    --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args)
+
 # Linux source tarball
 # ---------------------------------------------------------------------------
 
-quiet_cmd_archive_linux = ARCHIVE $@
-      cmd_archive_linux = \
-	git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ $$(cat $<)
-
 targets += linux.tar
+linux.tar: archive-args = $$(cat $<)
 linux.tar: .tmp_HEAD FORCE
-	$(call if_changed,archive_linux)
+	$(call if_changed,archive)
 
 # rpm-pkg
 # ---------------------------------------------------------------------------
@@ -180,16 +181,14 @@ quiet_cmd_perf_version_file = GEN     $@
 .tmp_perf/PERF-VERSION-FILE: .tmp_HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN | .tmp_perf
 	$(call cmd,perf_version_file)
 
-quiet_cmd_archive_perf = ARCHIVE $@
-      cmd_archive_perf = \
-	git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ \
-	--add-file=$$(realpath $(word 2, $^)) \
+perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \
 	--add-file=$$(realpath $(word 3, $^)) \
 	$$(cat $(word 2, $^))^{tree} $$(cat $<)
 
 targets += perf-$(KERNELVERSION).tar
+perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args)
 perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
-	$(call if_changed,archive_perf)
+	$(call if_changed,archive)
 
 PHONY += perf-tar-src-pkg
 perf-tar-src-pkg: perf-$(KERNELVERSION).tar
Signature · 1 line
2.37.2
#1 lore raw

PATCH 2/3 kbuild: do not create intermediate *.tar for source tarballs

Since commit 05e96e96a315 ("kbuild: use git-archive for source package creation"), source tarballs are created in two steps; create *.tar file then compress it. I split the compression as a separate rule because I just thought 'git archive' supported only gzip for compression. I admit the unneeded *.tar file is annoying.

For other compression algorithms, I could pipe the two commands:

  $ git archive HEAD | xz > linux.tar.xz

I read git-archive(1) carefully, and I realized GIT had provided a more elegant way:

  $ git -c tar.tar.xz.command=xz archive -o linux.tar.xz HEAD

This commit uses 'tar.tar.*.command' configuration to specify the compression backend so we can create a compressed tarball directly.

GIT commit 767cf4579f0e ("archive: implement configurable tar filters") is more than a decade old, so it should be available on almost all build environments.

---

 scripts/Makefile.package | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)
scripts/Makefile.package +17 -7
index a205617730c6..7707975f729b 100644
@@ -57,16 +57,23 @@ check-git:
 		false; \
 	fi
 
+archive-config-tar.gz  = -c tar.tar.gz.command="$(KGZIP)"
+archive-config-tar.bz2 = -c tar.tar.bz2.command="$(KBZIP2)"
+archive-config-tar.xz  = -c tar.tar.xz.command="$(XZ)"
+archive-config-tar.zst = -c tar.tar.zst.command="$(ZSTD)"
+
 quiet_cmd_archive = ARCHIVE $@
-      cmd_archive = git -C $(srctree) archive \
+      cmd_archive = git -C $(srctree) $(archive-config-tar$(suffix $@)) archive \
                     --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args)
 
 # Linux source tarball
 # ---------------------------------------------------------------------------
 
-targets += linux.tar
-linux.tar: archive-args = $$(cat $<)
-linux.tar: .tmp_HEAD FORCE
+linux-tarballs := $(addprefix linux, .tar.gz)
+
+targets += $(linux-tarballs)
+$(linux-tarballs): archive-args = $$(cat $<)
+$(linux-tarballs): .tmp_HEAD FORCE
 	$(call if_changed,archive)
 
 # rpm-pkg
@@ -185,9 +192,12 @@ perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \
 	--add-file=$$(realpath $(word 3, $^)) \
 	$$(cat $(word 2, $^))^{tree} $$(cat $<)
 
-targets += perf-$(KERNELVERSION).tar
-perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args)
-perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
+
+perf-tarballs := $(addprefix perf-$(KERNELVERSION), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst)
+
+targets += $(perf-tarballs)
+$(perf-tarballs): archive-args = $(perf-archive-args)
+$(perf-tarballs): tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
 	$(call if_changed,archive)
 
 PHONY += perf-tar-src-pkg
Signature · 1 line
2.37.2
maintainer Reviewed-by on v1 2/3
#2 lore raw

Re: [PATCH 2/3] kbuild: do not create intermediate *.tar for source tarballs

Quoted 6 lines · Masahiro Yamada

Since commit 05e96e96a315 ("kbuild: use git-archive for source package creation"), source tarballs are created in two steps; create *.tar file then compress it. I split the compression as a separate rule because I just thought 'git archive' supported only gzip for compression. I admit the unneeded *.tar file is annoying.

Quoted 6 lines

For other compression algorithms, I could pipe the two commands:

  $ git archive HEAD | xz > linux.tar.xz

I read git-archive(1) carefully, and I realized GIT had provided a more elegant way:

Hooray for documentation :)

Quoted 8 lines
  $ git -c tar.tar.xz.command=xz archive -o linux.tar.xz HEAD

This commit uses 'tar.tar.*.command' configuration to specify the compression backend so we can create a compressed tarball directly.

GIT commit 767cf4579f0e ("archive: implement configurable tar filters") is more than a decade old, so it should be available on almost all build environments.

git 1.7.7 it seems, certainly ancientware in my opinion. If people have issues with this, they can just upgrade git; even RHEL7 has git 1.8.x.

Quoted 1 line
Quoted 20 lines

---

 scripts/Makefile.package | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)
scripts/Makefile.package +5 -1
index a205617730c6..7707975f729b 100644
@@ -57,16 +57,23 @@ check-git:
 		false; \
 	fi
 
+archive-config-tar.gz  = -c tar.tar.gz.command="$(KGZIP)"
+archive-config-tar.bz2 = -c tar.tar.bz2.command="$(KBZIP2)"
+archive-config-tar.xz  = -c tar.tar.xz.command="$(XZ)"
+archive-config-tar.zst = -c tar.tar.zst.command="$(ZSTD)"
+
 quiet_cmd_archive = ARCHIVE $@
-      cmd_archive = git -C $(srctree) archive \

scripts/Makefile.package check-git: · line 72

Quoted 10 lines
scripts/Makefile.package +2 -3
+      cmd_archive = git -C $(srctree) $(archive-config-tar$(suffix $@)) archive \
                     --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args)
 
 # Linux source tarball
 # ---------------------------------------------------------------------------
 
-targets += linux.tar
-linux.tar: archive-args = $$(cat $<)
-linux.tar: .tmp_HEAD FORCE
+linux-tarballs := $(addprefix linux, .tar.gz)

Is there any reason not to allow other compression formats for linux like you do for perf?

Quoted 26 lines
scripts/Makefile.package +10 -3
+
+targets += $(linux-tarballs)
+$(linux-tarballs): archive-args = $$(cat $<)
+$(linux-tarballs): .tmp_HEAD FORCE
 	$(call if_changed,archive)
 
 # rpm-pkg
@@ -185,9 +192,12 @@ perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \
 	--add-file=$$(realpath $(word 3, $^)) \
 	$$(cat $(word 2, $^))^{tree} $$(cat $<)
 
-targets += perf-$(KERNELVERSION).tar
-perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args)
-perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
+
+perf-tarballs := $(addprefix perf-$(KERNELVERSION), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst)
+
+targets += $(perf-tarballs)
+$(perf-tarballs): archive-args = $(perf-archive-args)
+$(perf-tarballs): tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
 	$(call if_changed,archive)
 
 PHONY += perf-tar-src-pkg
Signature · 1 line
2.37.2
#3 lore raw

Re: [PATCH 2/3] kbuild: do not create intermediate *.tar for source tarballs

Quoted 59 lines · Nathan Chancellor
Quoted 6 lines · Masahiro Yamada

Since commit 05e96e96a315 ("kbuild: use git-archive for source package creation"), source tarballs are created in two steps; create *.tar file then compress it. I split the compression as a separate rule because I just thought 'git archive' supported only gzip for compression. I admit the unneeded *.tar file is annoying.

Quoted 6 lines

For other compression algorithms, I could pipe the two commands:

  $ git archive HEAD | xz > linux.tar.xz

I read git-archive(1) carefully, and I realized GIT had provided a more elegant way:

Hooray for documentation :)

Quoted 8 lines
  $ git -c tar.tar.xz.command=xz archive -o linux.tar.xz HEAD

This commit uses 'tar.tar.*.command' configuration to specify the compression backend so we can create a compressed tarball directly.

GIT commit 767cf4579f0e ("archive: implement configurable tar filters") is more than a decade old, so it should be available on almost all build environments.

git 1.7.7 it seems, certainly ancientware in my opinion. If people have issues with this, they can just upgrade git; even RHEL7 has git 1.8.x.

Quoted 1 line
Quoted 26 lines

---

 scripts/Makefile.package | 24 +++++++++++++++++-------
 1 file changed, 17 insertions(+), 7 deletions(-)
scripts/Makefile.package +6 -1
index a205617730c6..7707975f729b 100644
@@ -57,16 +57,23 @@ check-git:
              false; \
      fi
 
+archive-config-tar.gz  = -c tar.tar.gz.command="$(KGZIP)"
+archive-config-tar.bz2 = -c tar.tar.bz2.command="$(KBZIP2)"
+archive-config-tar.xz  = -c tar.tar.xz.command="$(XZ)"
+archive-config-tar.zst = -c tar.tar.zst.command="$(ZSTD)"
+
 quiet_cmd_archive = ARCHIVE $@
-      cmd_archive = git -C $(srctree) archive \
+      cmd_archive = git -C $(srctree) $(archive-config-tar$(suffix $@)) archive \
                     --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args)
 
 # Linux source tarball
 # ---------------------------------------------------------------------------
 
Quoted 7 lines

scripts/Makefile.package check-git: · line 72

Quoted 4 lines
scripts/Makefile.package +1 -3
-targets += linux.tar
-linux.tar: archive-args = $$(cat $<)
-linux.tar: .tmp_HEAD FORCE
+linux-tarballs := $(addprefix linux, .tar.gz)

Is there any reason not to allow other compression formats for linux like you do for perf?

Currently, gzip is only allowed because there is no way to specify the compression algorithm.
I already have a patch locally, but not submitted yet.

I prioritize groundwork, then add new features later.

Signature · 2 lines
Best Regards
Masahiro Yamada
#4 lore raw

PATCH 3/3 kbuild: do not create intermediate *.tar for tar packages

In reply to Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Generate the comressed tar packages directly.

---

 scripts/Makefile.package | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)
scripts/Makefile.package +9 -18
index 7707975f729b..e0e18d7dfbd5 100644
@@ -2,7 +2,6 @@
 # Makefile for the different targets used to generate full packages of a kernel
 
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib
 
 KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE))
 KBUILD_PKG_ROOTCMD ?="fakeroot -u"
@@ -27,21 +26,6 @@ fi ; \
 tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
 	--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3)
 
-# tarball compression
-# ---------------------------------------------------------------------------
-
-%.tar.gz: %.tar
-	$(call cmd,gzip)
-
-%.tar.bz2: %.tar
-	$(call cmd,bzip2)
-
-%.tar.xz: %.tar
-	$(call cmd,xzmisc)
-
-%.tar.zst: %.tar
-	$(call cmd,zstd)
-
 # Git
 # ---------------------------------------------------------------------------
 
@@ -153,10 +137,17 @@ tar-install: FORCE
 	$(Q)$(MAKE) -f $(srctree)/Makefile
 	+$(Q)$(srctree)/scripts/package/buildtar $@
 
+compress-tar.gz  = -I "$(KGZIP)"
+compress-tar.bz2 = -I "$(KBZIP2)"
+compress-tar.xz  = -I "$(XZ)"
+compress-tar.zst = -I "$(ZSTD)"
+
 quiet_cmd_tar = TAR     $@
-      cmd_tar = cd $<; tar cf ../$@ --owner=root --group=root --sort=name *
+      cmd_tar = cd $<; tar cf ../$@ $(compress-tar$(suffix $@)) --owner=root --group=root --sort=name *
+
+dir-tarballs := $(addprefix linux-$(KERNELRELEASE)-$(ARCH), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst)
 
-linux-$(KERNELRELEASE)-$(ARCH).tar: tar-install
+$(dir-tarballs): tar-install
 	$(call cmd,tar)
 
 PHONY += dir-pkg
Signature · 1 line
2.37.2
maintainer Reviewed-by on v1 3/3
#5 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

Quoted 11 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Generate the comressed tar packages directly.

Quoted 62 lines

---

 scripts/Makefile.package | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)
scripts/Makefile.package +9 -18
index 7707975f729b..e0e18d7dfbd5 100644
@@ -2,7 +2,6 @@
 # Makefile for the different targets used to generate full packages of a kernel
 
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib
 
 KERNELPATH := kernel-$(subst -,_,$(KERNELRELEASE))
 KBUILD_PKG_ROOTCMD ?="fakeroot -u"
@@ -27,21 +26,6 @@ fi ; \
 tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \
 	--transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3)
 
-# tarball compression
-# ---------------------------------------------------------------------------
-
-%.tar.gz: %.tar
-	$(call cmd,gzip)
-
-%.tar.bz2: %.tar
-	$(call cmd,bzip2)
-
-%.tar.xz: %.tar
-	$(call cmd,xzmisc)
-
-%.tar.zst: %.tar
-	$(call cmd,zstd)
-
 # Git
 # ---------------------------------------------------------------------------
 
@@ -153,10 +137,17 @@ tar-install: FORCE
 	$(Q)$(MAKE) -f $(srctree)/Makefile
 	+$(Q)$(srctree)/scripts/package/buildtar $@
 
+compress-tar.gz  = -I "$(KGZIP)"
+compress-tar.bz2 = -I "$(KBZIP2)"
+compress-tar.xz  = -I "$(XZ)"
+compress-tar.zst = -I "$(ZSTD)"
+
 quiet_cmd_tar = TAR     $@
-      cmd_tar = cd $<; tar cf ../$@ --owner=root --group=root --sort=name *
+      cmd_tar = cd $<; tar cf ../$@ $(compress-tar$(suffix $@)) --owner=root --group=root --sort=name *
+
+dir-tarballs := $(addprefix linux-$(KERNELRELEASE)-$(ARCH), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst)
 
-linux-$(KERNELRELEASE)-$(ARCH).tar: tar-install
+$(dir-tarballs): tar-install
 	$(call cmd,tar)
 
 PHONY += dir-pkg
Signature · 1 line
2.37.2
#6 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

Quoted 9 lines · Nathan Chancellor
Quoted 8 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Quoted 6 lines
Quoted 3 lines

Generate the comressed tar packages directly.

Hi,

We started seeing the failure below in rc7.
We narrowed it down to your patches:

3c65a2704cdd kbuild: do not create intermediate *.tar for tar packages f8d94++c4e403c kbuild: do not create intermediate *.tar for source tarballs f6d8283549bc kbuild: merge cmd_archive_linux and cmd_archive_perf aa7d233f45b4 kbuild: give up untracked files for source package builds

Can you please take a look and advise?

Regards,
Tariq

[root@c-237-113-200-203 linux]# make -j24 rpm-pkg
sh ./scripts/package/mkspec >./kernel.spec
rpmbuild --target x86_64-linux -bs kernel.spec \
--define='_smp_mflags %{nil}' --define='_sourcedir rpmbuild/SOURCES' --define='_srcrpmdir .' Building target platforms: x86_64-linux
Building for target x86_64-linux
Wrote: ./kernel-6.3.0_rc7+-1.src.rpm
rpmbuild --target x86_64-linux -rb kernel-6.3.0_rc7+-1.src.rpm \
--define='_smp_mflags %{nil}'
Installing kernel-6.3.0_rc7+-1.src.rpm
Building target platforms: x86_64-linux
Building for target x86_64-linux
Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.yDFEga
+ umask 022
+ cd /root/rpmbuild/BUILD
+ cd /root/rpmbuild/BUILD
+ rm -rf linux
+ /usr/bin/gzip -dc /root/rpmbuild/SOURCES/linux.tar.gz
+ /usr/bin/tar -xof -
+ STATUS=0
+ '[' 0 -ne 0 ']'
+ cd linux
/var/tmp/rpm-tmp.yDFEga: line 37: cd: linux: No such file or directory
error: Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)

RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)
make[1]: *** [scripts/Makefile.package:69: rpm-pkg] Error 1
make: *** [Makefile:1656: rpm-pkg] Error 2

#7 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

Quoted 61 lines · Tariq Toukan
Quoted 9 lines · Nathan Chancellor
Quoted 8 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Quoted 6 lines
Quoted 3 lines

Generate the comressed tar packages directly.

Hi,

We started seeing the failure below in rc7.
We narrowed it down to your patches:

3c65a2704cdd kbuild: do not create intermediate *.tar for tar packages f8d94++c4e403c kbuild: do not create intermediate *.tar for source tarballs f6d8283549bc kbuild: merge cmd_archive_linux and cmd_archive_perf aa7d233f45b4 kbuild: give up untracked files for source package builds

Can you please take a look and advise?

Regards,
Tariq

[root@c-237-113-200-203 linux]# make -j24 rpm-pkg
sh ./scripts/package/mkspec >./kernel.spec
rpmbuild --target x86_64-linux -bs kernel.spec \
--define='_smp_mflags %{nil}' --define='_sourcedir rpmbuild/SOURCES' --define='_srcrpmdir .'
Building target platforms: x86_64-linux
Building for target x86_64-linux
Wrote: ./kernel-6.3.0_rc7+-1.src.rpm
rpmbuild --target x86_64-linux -rb kernel-6.3.0_rc7+-1.src.rpm \
--define='_smp_mflags %{nil}'
Installing kernel-6.3.0_rc7+-1.src.rpm
Building target platforms: x86_64-linux
Building for target x86_64-linux
Executing(%prep): /bin/sh -e /var/tmp/rpm-tmp.yDFEga
+ umask 022
+ cd /root/rpmbuild/BUILD
+ cd /root/rpmbuild/BUILD
+ rm -rf linux
+ /usr/bin/gzip -dc /root/rpmbuild/SOURCES/linux.tar.gz
+ /usr/bin/tar -xof -
+ STATUS=0
+ '[' 0 -ne 0 ']'
+ cd linux
/var/tmp/rpm-tmp.yDFEga: line 37: cd: linux: No such file or directory
error: Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)

Quoted 4 lines

RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)
make[1]: *** [scripts/Makefile.package:69: rpm-pkg] Error 1
make: *** [Makefile:1656: rpm-pkg] Error 2

Thanks for the report. It should/will be fixed with
https://lore.kernel.org/linux-kbuild/[email protected]/

Kind regards,
Nicolas

#8 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

Quoted 29 lines · Nicolas Schier
Quoted 28 lines · Tariq Toukan
Quoted 9 lines · Nathan Chancellor
Quoted 8 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Quoted 6 lines
Quoted 3 lines

Generate the comressed tar packages directly.

Hi,

We started seeing the failure below in rc7.
We narrowed it down to your patches:

3c65a2704cdd kbuild: do not create intermediate *.tar for tar packages f8d94c4e403c kbuild: do not create intermediate *.tar for source tarballs f6d8283549bc kbuild: merge cmd_archive_linux and cmd_archive_perf aa7d233f45b4 kbuild: give up untracked files for source package builds

<...>

Quoted 7 lines
Quoted 4 lines

RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)
make[1]: *** [scripts/Makefile.package:69: rpm-pkg] Error 1
make: *** [Makefile:1656: rpm-pkg] Error 2

Thanks for the report. It should/will be fixed with
https://lore.kernel.org/linux-kbuild/[email protected]/

Thanks for the prompt response.

I have a general question, why commits listed by Tariq were not delayed to merge window?

Only one of them has Fixes line, but even that patch doesn't talk about error, but code refactoring "To simplify the code, ...".

Thanks

Quoted 3 lines

Kind regards,
Nicolas

#9 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

Quoted 22 lines · Leon Romanovsky
Quoted 20 lines · Nicolas Schier
Quoted 19 lines · Tariq Toukan
Quoted 15 lines · Nathan Chancellor
Quoted 11 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Generate the comressed tar packages directly.

Quoted 8 lines
Quoted 8 lines
Quoted 8 lines

Hi,

We started seeing the failure below in rc7.
We narrowed it down to your patches:

3c65a2704cdd kbuild: do not create intermediate *.tar for tar packages f8d94c4e403c kbuild: do not create intermediate *.tar for source tarballs f6d8283549bc kbuild: merge cmd_archive_linux and cmd_archive_perf

Strictly speaking, these are not bugs,
but gave a bad user experience.
(build slowness and extra disk space consumed).

I got a complaint about the intermediate *.tar file.

https://lore.kernel.org/linux-kbuild/[email protected]/

I noticed fixing it was not difficult, so I merged. (but, apparently I introduced another regression, sorry about that)

Quoted 24 lines
Quoted 1 line
Quoted 1 line

aa7d233f45b4 kbuild: give up untracked files for source package builds

<...>

Quoted 7 lines
Quoted 4 lines

RPM build errors:
Bad exit status from /var/tmp/rpm-tmp.yDFEga (%prep)
make[1]: *** [scripts/Makefile.package:69: rpm-pkg] Error 1
make: *** [Makefile:1656: rpm-pkg] Error 2

Thanks for the report. It should/will be fixed with
https://lore.kernel.org/linux-kbuild/[email protected]/

Thanks for the prompt response.

I have a general question, why commits listed by Tariq were not delayed to merge window?

Only one of them has Fixes line, but even that patch doesn't talk about error, but code refactoring "To simplify the code, ...".

Thanks

Quoted 3 lines

Kind regards,
Nicolas

Signature · 2 lines
Best Regards
Masahiro Yamada
#10 lore raw

Re: [PATCH 3/3] kbuild: do not create intermediate *.tar for tar packages

In reply to Masahiro Yamada
Quoted 22 lines · Masahiro Yamada

Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules.

With the previous commit, we got back to the situation where compressed source tarballs are created by a single rule. There is no reason to keep the separate compression rules.

Generate the comressed tar packages directly.

---

 scripts/Makefile.package | 27 +++++++++------------------
 1 file changed, 9 insertions(+), 18 deletions(-)
scripts/Makefile.package +0 -0
index 7707975f729b..e0e18d7dfbd5 100644

scripts/Makefile.package line 5

Quoted 4 lines
scripts/Makefile.package +0 -1
@@ -2,7 +2,6 @@
 # Makefile for the different targets used to generate full packages of a kernel
 
 include $(srctree)/scripts/Kbuild.include
-include $(srctree)/scripts/Makefile.lib

I noticed a bug.

I will keep this include directive. Otherwise, perf-tar*-src-pkg targets would be broken. 'cmd_copy' is still used.

Signature · 2 lines
Best Regards
Masahiro Yamada
maintainer Reviewed-by on v1 1/3
#11 lore raw
In reply to Masahiro Yamada
Quoted 5 lines · Masahiro Yamada

The two commands, cmd_archive_linux and cmd_archive_perf, are similar. Merge them to make it easier to add more changes to the git-archive command.

Quoted 55 lines

---

 scripts/Makefile.package | 19 +++++++++----------
 1 file changed, 9 insertions(+), 10 deletions(-)
scripts/Makefile.package +9 -10
index 61f72eb8d9be..a205617730c6 100644
@@ -57,16 +57,17 @@ check-git:
 		false; \
 	fi
 
+quiet_cmd_archive = ARCHIVE $@
+      cmd_archive = git -C $(srctree) archive \
+                    --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args)
+
 # Linux source tarball
 # ---------------------------------------------------------------------------
 
-quiet_cmd_archive_linux = ARCHIVE $@
-      cmd_archive_linux = \
-	git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ $$(cat $<)
-
 targets += linux.tar
+linux.tar: archive-args = $$(cat $<)
 linux.tar: .tmp_HEAD FORCE
-	$(call if_changed,archive_linux)
+	$(call if_changed,archive)
 
 # rpm-pkg
 # ---------------------------------------------------------------------------
@@ -180,16 +181,14 @@ quiet_cmd_perf_version_file = GEN     $@
 .tmp_perf/PERF-VERSION-FILE: .tmp_HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN | .tmp_perf
 	$(call cmd,perf_version_file)
 
-quiet_cmd_archive_perf = ARCHIVE $@
-      cmd_archive_perf = \
-	git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ \
-	--add-file=$$(realpath $(word 2, $^)) \
+perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \
 	--add-file=$$(realpath $(word 3, $^)) \
 	$$(cat $(word 2, $^))^{tree} $$(cat $<)
 
 targets += perf-$(KERNELVERSION).tar
+perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args)
 perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE
-	$(call if_changed,archive_perf)
+	$(call if_changed,archive)
 
 PHONY += perf-tar-src-pkg
 perf-tar-src-pkg: perf-$(KERNELVERSION).tar
Signature · 1 line
2.37.2