From aa7d233f45b4c549750044c9921f7afcbe50925b Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Mon, 10 Apr 2023 21:09:07 +0900 Subject: kbuild: give up untracked files for source package builds When the source tree is dirty and contains untracked files, package builds may fail, for example, when a broken symlink exists, a file path contains whitespaces, etc. Since commit 05e96e96a315 ("kbuild: use git-archive for source package creation"), the source tarball only contains committed files because it is created by 'git archive'. scripts/package/gen-diff-patch tries to address the diff from HEAD, but including untracked files by the hand-crafted script introduces more complexity. I wrote a patch [1] to make it work in most cases, but still wonder if this is what we should aim for. To simplify the code, this patch just gives up untracked files. Going forward, it is your responsibility to do 'git add' for what you want in the source package. The script shows a warning just in case you forgot to do so. It should be checked only when building source packages. [1]: https://lore.kernel.org/all/CAK7LNAShbZ56gSh9PrbLnBDYKnjtTkHMoCXeGrhcxMvqXGq9=g@mail.gmail.com/2-0001-kbuild-make-package-builds-more-robust.patch Fixes: 05e96e96a315 ("kbuild: use git-archive for source package creation") Signed-off-by: Masahiro Yamada Reviewed-by: Nicolas Schier --- scripts/Makefile.package | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'scripts/Makefile.package') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 61f72eb8d9be..49aff12cb6ab 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -94,7 +94,7 @@ binrpm-pkg: $(UTS_MACHINE)-linux -bb $(objtree)/binkernel.spec quiet_cmd_debianize = GEN $@ - cmd_debianize = $(srctree)/scripts/package/mkdebian + cmd_debianize = $(srctree)/scripts/package/mkdebian $(mkdebian-opts) debian: FORCE $(call cmd,debianize) @@ -103,6 +103,7 @@ PHONY += debian-orig debian-orig: private source = $(shell dpkg-parsechangelog -S Source) debian-orig: private version = $(shell dpkg-parsechangelog -S Version | sed 's/-[^-]*$$//') debian-orig: private orig-name = $(source)_$(version).orig.tar.gz +debian-orig: mkdebian-opts = --need-source debian-orig: linux.tar.gz debian $(Q)if [ "$(df --output=target .. 2>/dev/null)" = "$(df --output=target $< 2>/dev/null)" ]; then \ ln -f $< ../$(orig-name); \ -- cgit v1.2.3 From f6d8283549bc200e2babdd627239ece3547d634c Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 7 Apr 2023 19:16:27 +0900 Subject: kbuild: merge cmd_archive_linux and cmd_archive_perf The two commands, cmd_archive_linux and cmd_archive_perf, are similar. Merge them to make it easier to add more changes to the git-archive command. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor --- scripts/Makefile.package | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'scripts/Makefile.package') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index 49aff12cb6ab..d80a9b59f784 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -57,16 +57,17 @@ check-git: false; \ fi +quiet_cmd_archive = ARCHIVE $@ + cmd_archive = git -C $(srctree) archive \ + --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args) + # Linux source tarball # --------------------------------------------------------------------------- -quiet_cmd_archive_linux = ARCHIVE $@ - cmd_archive_linux = \ - git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ $$(cat $<) - targets += linux.tar +linux.tar: archive-args = $$(cat $<) linux.tar: .tmp_HEAD FORCE - $(call if_changed,archive_linux) + $(call if_changed,archive) # rpm-pkg # --------------------------------------------------------------------------- @@ -181,16 +182,14 @@ quiet_cmd_perf_version_file = GEN $@ .tmp_perf/PERF-VERSION-FILE: .tmp_HEAD $(srctree)/tools/perf/util/PERF-VERSION-GEN | .tmp_perf $(call cmd,perf_version_file) -quiet_cmd_archive_perf = ARCHIVE $@ - cmd_archive_perf = \ - git -C $(srctree) archive --output=$$(realpath $@) --prefix=$(basename $@)/ \ - --add-file=$$(realpath $(word 2, $^)) \ +perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \ --add-file=$$(realpath $(word 3, $^)) \ $$(cat $(word 2, $^))^{tree} $$(cat $<) targets += perf-$(KERNELVERSION).tar +perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args) perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE - $(call if_changed,archive_perf) + $(call if_changed,archive) PHONY += perf-tar-src-pkg perf-tar-src-pkg: perf-$(KERNELVERSION).tar -- cgit v1.2.3 From f8d94c4e403c89ec6b09ba69f65e4547ba99dd07 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 7 Apr 2023 19:16:28 +0900 Subject: kbuild: do not create intermediate *.tar for source tarballs Since commit 05e96e96a315 ("kbuild: use git-archive for source package creation"), a source tarball is created in two steps; create *.tar file then compress it. I split the compression as a separate rule because I just thought 'git archive' supported only gzip. For other compression algorithms, I could pipe the two commands: $ git archive HEAD | xz > linux.tar.xz I read git-archive(1) carefully, and I realized GIT had provided a more elegant way: $ git -c tar.tar.xz.command=xz archive -o linux.tar.xz HEAD This commit uses 'tar.tar.*.command' configuration to specify the compression backend so we can compress a source tarball on-the-fly. GIT commit 767cf4579f0e ("archive: implement configurable tar filters") is more than a decade old, so it should be available on almost all build environments. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor --- scripts/Makefile.package | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) (limited to 'scripts/Makefile.package') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index d80a9b59f784..ed1784be72ce 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -57,16 +57,23 @@ check-git: false; \ fi +git-config-tar.gz = -c tar.tar.gz.command="$(KGZIP)" +git-config-tar.bz2 = -c tar.tar.bz2.command="$(KBZIP2)" +git-config-tar.xz = -c tar.tar.xz.command="$(XZ)" +git-config-tar.zst = -c tar.tar.zst.command="$(ZSTD)" + quiet_cmd_archive = ARCHIVE $@ - cmd_archive = git -C $(srctree) archive \ + cmd_archive = git -C $(srctree) $(git-config-tar$(suffix $@)) archive \ --output=$$(realpath $@) --prefix=$(basename $@)/ $(archive-args) # Linux source tarball # --------------------------------------------------------------------------- -targets += linux.tar -linux.tar: archive-args = $$(cat $<) -linux.tar: .tmp_HEAD FORCE +linux-tarballs := $(addprefix linux, .tar.gz) + +targets += $(linux-tarballs) +$(linux-tarballs): archive-args = $$(cat $<) +$(linux-tarballs): .tmp_HEAD FORCE $(call if_changed,archive) # rpm-pkg @@ -186,9 +193,12 @@ perf-archive-args = --add-file=$$(realpath $(word 2, $^)) \ --add-file=$$(realpath $(word 3, $^)) \ $$(cat $(word 2, $^))^{tree} $$(cat $<) -targets += perf-$(KERNELVERSION).tar -perf-$(KERNELVERSION).tar: archive-args = $(perf-archive-args) -perf-$(KERNELVERSION).tar: tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE + +perf-tarballs := $(addprefix perf-$(KERNELVERSION), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst) + +targets += $(perf-tarballs) +$(perf-tarballs): archive-args = $(perf-archive-args) +$(perf-tarballs): tools/perf/MANIFEST .tmp_perf/HEAD .tmp_perf/PERF-VERSION-FILE FORCE $(call if_changed,archive) PHONY += perf-tar-src-pkg -- cgit v1.2.3 From 3c65a2704cdd2a0cd0766352e587bae4a6268155 Mon Sep 17 00:00:00 2001 From: Masahiro Yamada Date: Fri, 7 Apr 2023 19:16:29 +0900 Subject: kbuild: do not create intermediate *.tar for tar packages Commit 05e96e96a315 ("kbuild: use git-archive for source package creation") split the compression as a separate step to factor out the common build rules. With the previous commit, we got back to the situation where source tarballs are compressed on-the-fly. There is no reason to keep the separate compression rules. Generate the comressed tar packages directly. Signed-off-by: Masahiro Yamada Reviewed-by: Nathan Chancellor --- scripts/Makefile.package | 26 +++++++++----------------- 1 file changed, 9 insertions(+), 17 deletions(-) (limited to 'scripts/Makefile.package') diff --git a/scripts/Makefile.package b/scripts/Makefile.package index ed1784be72ce..4d90691505b1 100644 --- a/scripts/Makefile.package +++ b/scripts/Makefile.package @@ -27,21 +27,6 @@ fi ; \ tar -I $(KGZIP) -c $(RCS_TAR_IGNORE) -f $(2).tar.gz \ --transform 's:^:$(2)/:S' $(TAR_CONTENT) $(3) -# tarball compression -# --------------------------------------------------------------------------- - -%.tar.gz: %.tar - $(call cmd,gzip) - -%.tar.bz2: %.tar - $(call cmd,bzip2) - -%.tar.xz: %.tar - $(call cmd,xzmisc) - -%.tar.zst: %.tar - $(call cmd,zstd) - # Git # --------------------------------------------------------------------------- @@ -154,10 +139,17 @@ tar-install: FORCE $(Q)$(MAKE) -f $(srctree)/Makefile +$(Q)$(srctree)/scripts/package/buildtar $@ +compress-tar.gz = -I "$(KGZIP)" +compress-tar.bz2 = -I "$(KBZIP2)" +compress-tar.xz = -I "$(XZ)" +compress-tar.zst = -I "$(ZSTD)" + quiet_cmd_tar = TAR $@ - cmd_tar = cd $<; tar cf ../$@ --owner=root --group=root --sort=name * + cmd_tar = cd $<; tar cf ../$@ $(compress-tar$(suffix $@)) --owner=root --group=root --sort=name * + +dir-tarballs := $(addprefix linux-$(KERNELRELEASE)-$(ARCH), .tar .tar.gz .tar.bz2 .tar.xz .tar.zst) -linux-$(KERNELRELEASE)-$(ARCH).tar: tar-install +$(dir-tarballs): tar-install $(call cmd,tar) PHONY += dir-pkg -- cgit v1.2.3