From e49fdecd161b3d391e55311652fda3220d851fa1 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 14:27:33 +0100 Subject: [PATCH 1/6] test: Add option to save in progress test journals to /tmp The journal isn't the best at being fast, especially when writing to disk and not to memory, which can cause integration tests to grind to a halt on beefy systems due to all the systemd-journal-remote instances not being able to write journal entries to disk fast enough. Let's introduce an option to allow writing in progress test journals to use /tmp which can be used on beefy systems with lots of memory to speed things up. --- test/README.md | 4 ++++ test/integration-test-wrapper.py | 12 +++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/test/README.md b/test/README.md index f880a984f1..6e625bf2b3 100644 --- a/test/README.md +++ b/test/README.md @@ -151,6 +151,10 @@ that make use of `run_testcases`. `TEST_SKIP_TESTCASE=testcase`: takes a space separated list of testcases to skip. +`TEST_JOURNAL_USE_TMP=1`: Write test journal to `/tmp` while the test is in +progress and only move the journal to its final location in the build directory +(`$BUILD_DIR/test/journal`) when the test is finished. + ### SELinux AVCs To have `TEST-06-SELINUX` check for SELinux denials, write the following to diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index d9d92fcba3..1c28cf3776 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -10,6 +10,7 @@ import json import os import re import shlex +import shutil import subprocess import sys import tempfile @@ -441,7 +442,11 @@ def main() -> None: """ ) - journal_file = (args.meson_build_dir / (f'test/journal/{name}.journal')).absolute() + if os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1': + journal_file = Path(f'/tmp/systemd-integration-tests/journal/{name.journal}') + else: + journal_file = (args.meson_build_dir / f'test/journal/{name}.journal').absolute() + journal_file.unlink(missing_ok=True) if not sys.stderr.isatty(): @@ -551,6 +556,11 @@ def main() -> None: ): journal_file.unlink(missing_ok=True) + if os.getenv('TEST_JOURNAL_USE_TMP', '0') == '1': + dst = args.meson_build_dir / f'test/journal/{name}.journal' + dst.parent.mkdir(parents=True, exist_ok=True) + shutil.move(journal_file, dst) + if shell or (result.returncode in (args.exit_code, 77) and not coredumps and not sanitizer): exit(0 if shell or result.returncode == args.exit_code else 77) From 84b30442d257102a9a39122f9a537fa48fb0bfda Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 14:29:58 +0100 Subject: [PATCH 2/6] test: Don't register machines with machined unless we're in interactive mode --- test/integration-test-wrapper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 1c28cf3776..94904cfbc4 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -516,7 +516,7 @@ def main() -> None: ] ), '--credential', f"journal.storage={'persistent' if sys.stderr.isatty() else args.storage}", - *(['--runtime-build-sources=no'] if not sys.stderr.isatty() else []), + *(['--runtime-build-sources=no', '--register=no'] if not sys.stderr.isatty() else []), 'vm' if args.vm or os.getuid() != 0 or os.getenv('TEST_PREFER_QEMU', '0') == '1' else 'boot', ] # fmt: skip From 1f17ec0ed419627a686ee6e719ac7f55cf082ada Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 14:51:24 +0100 Subject: [PATCH 3/6] test: Move StateDirectory= directive into dropin The integration-test-setup calls require StateDirectory= but some tests override the test unit used which then won't have StateDirectory= so let's move StateDirectory= into the dropin as well to avoid this issue. --- test/integration-test-wrapper.py | 1 + test/test.service.in | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/test/integration-test-wrapper.py b/test/integration-test-wrapper.py index 94904cfbc4..610c34c903 100755 --- a/test/integration-test-wrapper.py +++ b/test/integration-test-wrapper.py @@ -470,6 +470,7 @@ def main() -> None: [Service] ExecStartPre=/usr/lib/systemd/tests/testdata/integration-test-setup.sh setup ExecStopPost=/usr/lib/systemd/tests/testdata/integration-test-setup.sh finalize + StateDirectory=%N """ ) diff --git a/test/test.service.in b/test/test.service.in index 75f703698f..790c513da4 100644 --- a/test/test.service.in +++ b/test/test.service.in @@ -10,4 +10,3 @@ ExecStartPre=rm -f /failed /testok ExecStart=@command@ Type=oneshot MemoryAccounting=@memory-accounting@ -StateDirectory=%N From c32a8cdaa0f03ae29e9edade1213cc2001b28000 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 15:26:37 +0100 Subject: [PATCH 4/6] fmf: Bump inotify limits to avoid systemd-nspawn failures --- test/fmf/integration-tests/test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/fmf/integration-tests/test.sh b/test/fmf/integration-tests/test.sh index d1e43b1d7d..34bf1abfa8 100755 --- a/test/fmf/integration-tests/test.sh +++ b/test/fmf/integration-tests/test.sh @@ -13,6 +13,10 @@ lsmem echo "Clock source: $(cat /sys/devices/system/clocksource/clocksource0/current_clocksource)" +# Bump inotify limits so nspawn containers don't run out of inotify file descriptors. +sysctl fs.inotify.max_user_watches=65536 +sysctl fs.inotify.max_user_instances=1024 + # Allow running the integration tests downstream in dist-git with something like # the following snippet which makes the dist-git sources available in $TMT_SOURCE_DIR: # From 53546c71fe0a1b30ee296df84bb8c3577f5675a4 Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 15:26:54 +0100 Subject: [PATCH 5/6] fmf: Use different heuristic on beefy systems If we save journals in /tmp, we can run a larger number of tests in parallel so let's make use of the larger number of CPUs if the tests run on a beefy machine. --- test/fmf/integration-tests/test.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/fmf/integration-tests/test.sh b/test/fmf/integration-tests/test.sh index 34bf1abfa8..06a98bfd7a 100755 --- a/test/fmf/integration-tests/test.sh +++ b/test/fmf/integration-tests/test.sh @@ -116,6 +116,14 @@ if [[ ! -e /dev/kvm ]]; then export TEST_NO_QEMU=1 fi +NPROC="$(nproc)" +if [[ "$NPROC" -ge 10 ]]; then + export TEST_JOURNAL_USE_TMP=1 + NPROC="$((NPROC / 3))" +else + NPROC="$((NPROC - 1))" +fi + # Create missing mountpoint for mkosi sandbox. mkdir -p /etc/pacman.d/gnupg @@ -131,7 +139,7 @@ mkosi -f sandbox \ --suite integration-tests \ --print-errorlogs \ --no-stdsplit \ - --num-processes "$(($(nproc) - 1))" && EC=0 || EC=$? + --num-processes "$NPROC" && EC=0 || EC=$? find build/meson-logs -type f -exec mv {} "$TMT_TEST_DATA" \; find build/test/journal -type f -exec mv {} "$TMT_TEST_DATA" \; From e0c2fd6a3345d26afdf4159406c38cd9101d2e0d Mon Sep 17 00:00:00 2001 From: Daan De Meyer Date: Fri, 10 Jan 2025 15:29:28 +0100 Subject: [PATCH 6/6] fmf: Skip TEST-21-DFUZZER Similar to Github Actions, since we don't build with sanitizers in the packit job, let's skip TEST-21-DFUZZER. --- test/fmf/integration-tests/test.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/fmf/integration-tests/test.sh b/test/fmf/integration-tests/test.sh index 06a98bfd7a..f82961f959 100755 --- a/test/fmf/integration-tests/test.sh +++ b/test/fmf/integration-tests/test.sh @@ -124,6 +124,10 @@ else NPROC="$((NPROC - 1))" fi +# This test is only really useful if we're building with sanitizers and takes a long time, so let's skip it +# for now. +export TEST_SKIP="TEST-21-DFUZZER" + # Create missing mountpoint for mkosi sandbox. mkdir -p /etc/pacman.d/gnupg