From ba592dc715eed67b0c939ded94b4c31bc5ba1070 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 16:24:42 +0200 Subject: [PATCH 1/5] mkosi: Replace submodules with our own thing Unfortunately, git submodules break in all sorts of ways: - Various github workflows (dependabot, github pages) try to do a shallow clone of git submodules which does not work at all when the git repository is hosted on pagure (https://pagure.io/pagure/issue/5453, https://github.com/dependabot/dependabot-core/issues/9391). - If the git forge hosting the git repository uses SHA256, then it breaks our usage of it as a submodule as SHA256 repositories cannot be used as submodules in SHA1 repositories (src.opensuse.org moved to SHA256 which broke our usage of opensuse's systemd spec as a submodule). - git submodules completely break usage of git worktrees. - ... Let's avoid all these issues by just doing our own home grown implementation of git submodules. We lose the automatic dependabot updates this way but since dependabot fails to run more often that not with submodules we don't really lose anything. --- .github/dependabot.yml | 5 ----- .gitmodules | 20 ------------------ docs/HACKING.md | 9 -------- .../system/mkosi.conf.d/10-arch/mkosi.conf | 5 +++++ .../system/mkosi.conf.d/10-centos/mkosi.conf | 5 +++++ .../mkosi.conf.d/10-debian-ubuntu/mkosi.conf | 5 +++++ .../system/mkosi.conf.d/10-fedora/mkosi.conf | 5 +++++ .../mkosi.conf.d/10-opensuse/mkosi.conf | 5 +++++ mkosi.images/system/mkosi.sync | 21 +++++++++++++++---- pkg/arch | 1 - pkg/centos | 1 - pkg/debian | 1 - pkg/fedora | 1 - pkg/opensuse | 1 - tools/git-setup.sh | 18 ---------------- tools/git-submodule-update-hook.sh | 4 ---- 16 files changed, 42 insertions(+), 65 deletions(-) delete mode 160000 pkg/arch delete mode 160000 pkg/centos delete mode 160000 pkg/debian delete mode 160000 pkg/fedora delete mode 160000 pkg/opensuse delete mode 100755 tools/git-submodule-update-hook.sh diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 54593c50f7..3e067c176f 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -17,8 +17,3 @@ updates: schedule: interval: "monthly" open-pull-requests-limit: 2 - - package-ecosystem: "gitsubmodule" - directory: "/" - schedule: - interval: "daily" - open-pull-requests-limit: 2 diff --git a/.gitmodules b/.gitmodules index 7ae6ace680..e69de29bb2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,20 +0,0 @@ -[submodule "pkg/fedora"] - path = pkg/fedora - url = https://src.fedoraproject.org/rpms/systemd.git - branch = rawhide -[submodule "pkg/opensuse"] - path = pkg/opensuse - url = https://code.opensuse.org/package/systemd.git - branch = master -[submodule "pkg/debian"] - path = pkg/debian - url = https://salsa.debian.org/systemd-team/systemd.git - branch = debian/master -[submodule "pkg/centos"] - path = pkg/centos - url = https://git.centos.org/rpms/systemd.git - branch = c9s-sig-hyperscale -[submodule "pkg/arch"] - path = pkg/arch - url = https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git - branch = main diff --git a/docs/HACKING.md b/docs/HACKING.md index 980a45929e..51499d7f79 100644 --- a/docs/HACKING.md +++ b/docs/HACKING.md @@ -22,15 +22,6 @@ If adding a new source file, consider adding a matching test executable. For features at a higher level, tests in `src/test/` are very strongly recommended. If that is not possible, integration tests in `test/` are encouraged. -```shell -$ git config submodule.recurse true -$ git config fetch.recurseSubmodules on-demand -$ git config push.recurseSubmodules no -$ cp .git/hooks/pre-commit.sample .git/hooks/pre-commit -$ cp tools/git-submodule-update-hook.sh .git/hooks/post-rewrite -$ cp tools/git-submodule-update-hook.sh .git/hooks/post-checkout -``` - Please always test your work before submitting a PR. For many of the components of systemd testing is straightforward as you can simply compile systemd and run the relevant tool from the build directory. diff --git a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf index 942db4cb96..9c7d09d656 100644 --- a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf @@ -4,6 +4,11 @@ Distribution=arch [Content] +Environment= + GIT_URL=https://gitlab.archlinux.org/archlinux/packaging/packages/systemd.git + GIT_BRANCH=main + GIT_COMMIT=dc6c099e0785753c1c88b4adcbcbfc209a8d12e3 + VolatilePackages= systemd systemd-libs diff --git a/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf index 15f888168d..395a22b56a 100644 --- a/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-centos/mkosi.conf @@ -4,6 +4,11 @@ Distribution=centos [Content] +Environment= + GIT_URL=https://git.centos.org/rpms/systemd.git + GIT_BRANCH=c9s-sig-hyperscale + GIT_COMMIT=8cf2aed0181920611421384f7374720db269d6c7 + Packages= kernel-modules # For squashfs rpmautospec-rpm-macros diff --git a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf index 7e870ace87..047dbc1dc3 100644 --- a/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-debian-ubuntu/mkosi.conf @@ -5,6 +5,11 @@ Distribution=|debian Distribution=|ubuntu [Content] +Environment= + GIT_URL=https://salsa.debian.org/systemd-team/systemd.git + GIT_BRANCH=debian/master + GIT_COMMIT=18201fa98d74172fa1a17242326e3275995cde13 + VolatilePackages= libnss-myhostname libnss-mymachines diff --git a/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf index 3acec1092c..7b122e3c69 100644 --- a/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-fedora/mkosi.conf @@ -4,6 +4,11 @@ Distribution=fedora [Content] +Environment= + GIT_URL=https://src.fedoraproject.org/rpms/systemd.git + GIT_BRANCH=rawhide + GIT_COMMIT=74810c5bc4fe7d872e54c253447ffd61bbc8839f + Packages= btrfs-progs compsize diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf index b3172bfc14..90844d4db4 100644 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf @@ -7,6 +7,11 @@ Distribution=opensuse InitrdInclude=initrd/ [Content] +Environment= + GIT_URL=https://src.opensuse.org/rpm/systemd + GIT_BRANCH=factory + GIT_COMMIT=28c9bef337dd47c88adc1d948cedfdc79c788e2955f17a534bd76da6419722dd + VolatilePackages= systemd systemd-boot diff --git a/mkosi.images/system/mkosi.sync b/mkosi.images/system/mkosi.sync index 3e42a09b2c..a4f0ab94ec 100755 --- a/mkosi.images/system/mkosi.sync +++ b/mkosi.images/system/mkosi.sync @@ -2,8 +2,21 @@ # SPDX-License-Identifier: LGPL-2.1-or-later set -e -if [ -z "$(ls --almost-all "pkg/$DISTRIBUTION")" ] || [ -f "pkg/$DISTRIBUTION/.git" ]; then - PKG_SUBDIR="$(realpath "pkg/$DISTRIBUTION" --relative-to "$PWD")" - git submodule sync "$PKG_SUBDIR" - git submodule update --init "$PKG_SUBDIR" +if ((NO_SYNC)); then + exit 0 fi + +PKG_SUBDIR="$(realpath --canonicalize-missing "pkg/$DISTRIBUTION" --relative-to "$PWD")" + +if [[ -d "$PKG_SUBDIR/.git" ]] && [[ "$(git -C "$PKG_SUBDIR" rev-parse HEAD)" == "$GIT_COMMIT" ]]; then + exit 0 +fi + +if [[ ! -e "$PKG_SUBDIR" ]] || [[ -z "$(ls --almost-all "$PKG_SUBDIR")" ]]; then + git clone "$GIT_URL" --branch "$GIT_BRANCH" "$PKG_SUBDIR" +else + git -C "$PKG_SUBDIR" remote set-url origin "$GIT_URL" + git -C "$PKG_SUBDIR" fetch origin "$GIT_BRANCH" +fi + +git -C "$PKG_SUBDIR" -c advice.detachedHead=false checkout "$GIT_COMMIT" diff --git a/pkg/arch b/pkg/arch deleted file mode 160000 index b578e90bb6..0000000000 --- a/pkg/arch +++ /dev/null @@ -1 +0,0 @@ -Subproject commit b578e90bb68deaae28300300c57b6a99c916b201 diff --git a/pkg/centos b/pkg/centos deleted file mode 160000 index 61bb5f4701..0000000000 --- a/pkg/centos +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 61bb5f47014707a927bc033a21e8dbf685c5d2c5 diff --git a/pkg/debian b/pkg/debian deleted file mode 160000 index 824c52fafc..0000000000 --- a/pkg/debian +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 824c52fafc3afda4c8ad3892596e8243c3b45c91 diff --git a/pkg/fedora b/pkg/fedora deleted file mode 160000 index 8fe1f037d2..0000000000 --- a/pkg/fedora +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 8fe1f037d21c9d68d96728843f22e5036d769521 diff --git a/pkg/opensuse b/pkg/opensuse deleted file mode 160000 index c3399411ac..0000000000 --- a/pkg/opensuse +++ /dev/null @@ -1 +0,0 @@ -Subproject commit c3399411ac70a440ca9080d6df391eb897bba4e9 diff --git a/tools/git-setup.sh b/tools/git-setup.sh index 8cc1bfdfc2..b5903b4ca1 100755 --- a/tools/git-setup.sh +++ b/tools/git-setup.sh @@ -4,12 +4,6 @@ set -eu cd "${MESON_SOURCE_ROOT:?}" -if [ -e .git ]; then - git config submodule.recurse true - git config fetch.recurseSubmodules on-demand - git config push.recurseSubmodules no -fi - ret=2 if [ -f .git/hooks/pre-commit.sample ] && [ ! -f .git/hooks/pre-commit ]; then @@ -19,16 +13,4 @@ if [ -f .git/hooks/pre-commit.sample ] && [ ! -f .git/hooks/pre-commit ]; then ret=0 fi -if [ ! -f .git/hooks/post-rewrite ]; then - cp -p tools/git-submodule-update-hook.sh .git/hooks/post-rewrite - echo 'Activated post-rewrite hook' - ret=0 -fi - -if [ ! -f .git/hooks/post-checkout ]; then - cp -p tools/git-submodule-update-hook.sh .git/hooks/post-checkout - echo 'Activated post-checkout hook' - ret=0 -fi - exit $ret diff --git a/tools/git-submodule-update-hook.sh b/tools/git-submodule-update-hook.sh deleted file mode 100755 index 78feb9dbba..0000000000 --- a/tools/git-submodule-update-hook.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -# SPDX-License-Identifier: LGPL-2.1-or-later - -exec git submodule update From 8c335463d9fe6c2e2f3d9584d7c26871800065e2 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 20:54:06 +0200 Subject: [PATCH 2/5] mkosi: Don't install wireguard-tools on Debian Package was removed from testing. This can be reverted next week once it's added back to testing. --- mkosi.images/system/mkosi.conf | 1 - mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf | 1 + mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf | 1 + 5 files changed, 4 insertions(+), 1 deletion(-) diff --git a/mkosi.images/system/mkosi.conf b/mkosi.images/system/mkosi.conf index f543b6410f..6455b0477e 100644 --- a/mkosi.images/system/mkosi.conf +++ b/mkosi.images/system/mkosi.conf @@ -73,7 +73,6 @@ Packages= util-linux valgrind which - wireguard-tools xfsprogs zsh zstd diff --git a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf index 9c7d09d656..51f28903b3 100644 --- a/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-arch/mkosi.conf @@ -58,6 +58,7 @@ Packages= tpm2-tools tpm2-tss vim + wireguard-tools InitrdPackages= btrfs-progs diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf index 799de75e53..65bf20e856 100644 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.conf @@ -65,6 +65,7 @@ Packages= util-linux veritysetup vim-common + wireguard-tools InitrdPackages= tpm2-tools diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf index 90844d4db4..188f548643 100644 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.conf @@ -85,6 +85,7 @@ Packages= user(root) veritysetup vim + wireguard-tools xz InitrdPackages= diff --git a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf index 86f9736ed9..c52621d41a 100644 --- a/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf +++ b/mkosi.images/system/mkosi.conf.d/10-ubuntu/mkosi.conf @@ -8,3 +8,4 @@ Packages= linux-image-generic linux-tools-common linux-tools-virtual + wireguard-tools From 3b046883addad621550aa74e873256e6fa18373c Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:16:13 +0200 Subject: [PATCH 3/5] mkosi: Skip debuginfo workaround on newer rpm The corresponding bug was fixed in rpm 4.20 (of which the alpha is in rawhide as rpm 4.19.91) so skip the workaround when we detect a newer rpm version. --- .../mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 8 +++++--- .../system/mkosi.conf.d/10-opensuse/mkosi.build.chroot | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index a8e4e7e90f..6d9380c87b 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -20,13 +20,15 @@ else TS="${SOURCE_DATE_EPOCH:-$(date +%s)}" fi -# Fix the %install override so debuginfo packages are generated even when --build-in-place is used. -# See https://github.com/rpm-software-management/rpm/issues/3042. -tee --append /usr/lib/rpm/redhat/macros <<'EOF' +if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.19.91"; then + # Fix the %install override so debuginfo packages are generated even when --build-in-place is used. + # See https://github.com/rpm-software-management/rpm/issues/3042. + tee --append /usr/lib/rpm/redhat/macros <<'EOF' %install %{?_enable_debug_packages:%{debug_package}}\ %%install\ %{nil} EOF +fi VERSION="$(cat meson.version)" RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index 8b6fc7ab13..e7f586cf8b 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -26,12 +26,14 @@ fi # extension. find "pkg/$ID" -name "files.*" -exec sed --in-place 's/\.gz$//' {} \; -# Fix the %install override so debuginfo packages are generated. -tee --append /usr/lib/rpm/suse/macros <<'EOF' +if systemd-analyze compare-versions "$(rpm --version | cut -d ' ' -f3)" lt "4.20"; then + # Fix the %install override so debuginfo packages are generated. + tee --append /usr/lib/rpm/suse/macros <<'EOF' %install %{debug_package}\ %%install\ %{nil} EOF +fi VERSION="$(cat meson.version)" RELEASE="$(date "+%Y%m%d%H%M%S" --date "@$TS")" From e74a994ebf7c813979b9e045f37907e6a5064f23 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:17:29 +0200 Subject: [PATCH 4/5] mkosi: Disable add-determinism on Fedora This was added for reproducible builds which we don't really care about when building rpms in mkosi, so disable it from running as it's broken and breaks our rpm builds. --- .../system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 1 + 1 file changed, 1 insertion(+) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index 6d9380c87b..3a09f00c25 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -72,6 +72,7 @@ ANNOBIN="no-active-checks" rpmbuild \ --define "__brp_strip_comment_note %{nil}" \ --define "__brp_strip_static_archive %{nil}" \ --define "__brp_check_rpaths %{nil}" \ + --undefine __brp_add_determinism \ --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ From 1baea8628775da6b7713863a30381ddb0101e36d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Thu, 30 May 2024 22:19:00 +0200 Subject: [PATCH 5/5] mkosi: Override rpm's _fixperms script to be a noop Since rpm 4.20 it tries to fix the permissions of all source files in the sources. This takes forever and fails in our case with a permission error. Since we don't want rpm touching our source files, override the script to be a noop. --- .../system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot | 1 + mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot | 1 + 2 files changed, 2 insertions(+) diff --git a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot index 3a09f00c25..7618f07839 100755 --- a/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-centos-fedora/mkosi.build.chroot @@ -77,6 +77,7 @@ ANNOBIN="no-active-checks" rpmbuild \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ --define "_fortify_level 0" \ + --define "_fixperms true" \ --undefine _lto_cflags \ --undefine _package_note_flags \ --noclean \ diff --git a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot index e7f586cf8b..b2c56fda77 100755 --- a/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot +++ b/mkosi.images/system/mkosi.conf.d/10-opensuse/mkosi.build.chroot @@ -76,6 +76,7 @@ build() { --define "__elf_exclude_path ^/usr/lib/systemd/tests/unit-tests/.*$" \ --define "__script_requires %{nil}" \ --define "_find_debuginfo_dwz_opts %{nil}" \ + --define "_fixperms true" \ --noclean \ "$@" \ "pkg/$ID/systemd.spec"