From a0bf8ee16db874224ec4335d08e651e6637e9c65 Mon Sep 17 00:00:00 2001 From: Mark Ridgwell Date: Thu, 16 Jul 2026 09:28:34 +0100 Subject: [PATCH 1/2] fix: remove --long-running/--parallel-algorithm flags from buildtest's dotnet test invocation Some benchmark test projects' test host rejects these as invalid arguments (exit code 5, "Zero tests ran") instead of running, confirmed by direct reproduction against FunFair.Ethereum.DataTypes.Benchmark.Tests. Removing --parallel-algorithm aggressive also addresses part of the OOM racing problem tracked in #679. Closes #723 --- CHANGELOG.md | 1 + development/buildtest | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0a36d7ef..e01b4647 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,6 +49,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release - check: guard against empty file list before checking — prevents false-positive success when no scripts are found - git/ignore-changelog: skip push hooks when pushing to target repositories - git/fetch and git/switchtomain now warn and skip a repo on error instead of aborting the whole run, so remaining repos still get processed. +- development/buildtest no longer passes --long-running/--parallel-algorithm flags to dotnet test, since some benchmark test projects' test host rejects them as invalid arguments (exit code 5, zero tests ran) instead of running ### Changed - Replace raw echo with standard output helpers (die/info/success) in github/cancel-workflows - Replace raw echo with standard output helpers (die/info/success) in git/update-repos-personal diff --git a/development/buildtest b/development/buildtest index fbfde657..a8a84152 100755 --- a/development/buildtest +++ b/development/buildtest @@ -166,10 +166,8 @@ dotnet test \ --configuration Release \ --no-restore \ --no-build \ - --long-running 10 \ $TEST_BENCHMARKS \ $TEST_INTEGRATION \ - --parallel-algorithm aggressive \ --ignore-exit-code 8 \ "-p:ApiCompatGenerateSuppressionFile=true" \ "-p:ContinuousIntegrationBuild=true" \ From 6e022f582f1fd623754cdc53cb9868bfb27bac2c Mon Sep 17 00:00:00 2001 From: Mark Ridgwell Date: Thu, 16 Jul 2026 09:37:19 +0100 Subject: [PATCH 2/2] fix: split buildtest into a unit-test pass and a separate benchmark pass Mirrors credfeto-global-pre-commit's src/scripts/buildtest design: the main dotnet test run now always excludes every benchmark test project (keeping --long-running/--parallel-algorithm aggressive, safe now that benchmarks aren't mixed in), and any benchmark projects found are run individually afterwards, one dotnet test invocation per project, without those two flags - some benchmark projects' test host rejects them as invalid arguments. Also fixes the -b/--no-benchmmarks flag's broken unquoted TEST_BENCHMARKS assignment, which is now load-bearing for skipping the new benchmark phase. Refs #679 --- CHANGELOG.md | 2 +- development/buildtest | 48 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e01b4647..34183a64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -49,7 +49,7 @@ Please ADD ALL Changes to the UNRELEASED SECTION and not a specific release - check: guard against empty file list before checking — prevents false-positive success when no scripts are found - git/ignore-changelog: skip push hooks when pushing to target repositories - git/fetch and git/switchtomain now warn and skip a repo on error instead of aborting the whole run, so remaining repos still get processed. -- development/buildtest no longer passes --long-running/--parallel-algorithm flags to dotnet test, since some benchmark test projects' test host rejects them as invalid arguments (exit code 5, zero tests ran) instead of running +- development/buildtest now runs unit tests first, always excluding benchmark test projects, then runs any benchmark projects found individually without the --long-running/--parallel-algorithm flags, since some benchmark projects' test host rejects them as invalid arguments (exit code 5, zero tests ran) instead of running ### Changed - Replace raw echo with standard output helpers (die/info/success) in github/cancel-workflows - Replace raw echo with standard output helpers (die/info/success) in git/update-repos-personal diff --git a/development/buildtest b/development/buildtest index a8a84152..a23b2a61 100755 --- a/development/buildtest +++ b/development/buildtest @@ -67,7 +67,7 @@ while [ $# -gt 0 ]; do shift # past argument ;; -b|--no-benchmmarks) - TEST_BENCHMARKS=--filter-not-namespace "*.BenchMark.Tests" --filter-not-namespace "*.BenchMark.Tests.*" + TEST_BENCHMARKS=1 shift # past argument ;; -i|--no-integratopn) @@ -161,12 +161,27 @@ dotnet build \ -nodeReuse:False || die "Build Failed" info "Testing..." +# Every benchmark test project is always excluded here, regardless of the +# -b/--no-benchmmarks flag; benchmarks (if any) are run separately below, one +# project per dotnet test invocation, since some benchmark projects' test +# host rejects --long-running/--parallel-algorithm as invalid arguments when +# passed to a combined run. +# +# Keep this split in sync with src/scripts/buildtest in +# credfeto/credfeto-global-pre-commit - that repo is the source of truth for +# this design; this file is a local fork/vendored copy, not consumed +# directly from there (see credfeto/scripts#679, credfeto/scripts#723). # shellcheck disable=SC2086 dotnet test \ --configuration Release \ --no-restore \ --no-build \ - $TEST_BENCHMARKS \ + --long-running 10 \ + --parallel-algorithm aggressive \ + --filter-not-namespace "*.Benchmark.Tests" \ + --filter-not-namespace "*.Benchmark.Tests.*" \ + --filter-not-namespace "*.BenchMark.Tests" \ + --filter-not-namespace "*.BenchMark.Tests.*" \ $TEST_INTEGRATION \ --ignore-exit-code 8 \ "-p:ApiCompatGenerateSuppressionFile=true" \ @@ -177,6 +192,35 @@ dotnet test \ "-p:Version=0.0.0.1-test" \ || die "Tests Failed" +if [ -z "$TEST_BENCHMARKS" ]; then + BENCH_PROJECTS=$(find "$SOURCE_DIR" -type f -iname "*.csproj" -not -path "*/obj/*" -not -path "*/bin/*" | while IFS= read -r proj; do + bn=$(basename "$proj") + printf '%s\n' "$bn" | grep -qiE '(^|[.])bench(mark)?[.]tests?[.]csproj$' && printf '%s\n' "$proj" + done) + + if [ -n "$BENCH_PROJECTS" ]; then + while IFS= read -r bench_project; do + [ -n "$bench_project" ] || continue + + info "Running benchmark tests: $bench_project" + dotnet test "$bench_project" \ + --configuration Release \ + --no-restore \ + --no-build \ + --ignore-exit-code 8 \ + "-p:ApiCompatGenerateSuppressionFile=false" \ + "-p:ContinuousIntegrationBuild=true" \ + "-p:IsProduction=true" \ + "-p:Optimize=true" \ + "-p:SuppressNETCoreSdkPreviewMessage=true" \ + "-p:Version=0.0.0.1-test" \ + || die "Benchmark Tests Failed ($bench_project)" + done <