From 7c690a3070b371b73efb163f9bd3b8deeaf6f46e Mon Sep 17 00:00:00 2001 From: Rafael Vuijk Date: Sat, 5 Sep 2026 16:47:55 +0000 Subject: [PATCH] The inter-version benchmark reaches two releases further back Two of the five rows in key-commits.txt read "did not build". The reason was not the case being measured: CommonFunctionsInterVersion uses nothing but the public API and compiles against every one of them. It was the other cases in the same project. MatchingEngine reaches `MatchedRules` and `Patterns`, which were internal until v2.3.0, and TransformationLayer names `Transformation.Rationalization`, which did not exist at v2.1.0 -- so the project failed to compile and took the row with it, and the file that would have run was never the problem. `-p:InterVersionOnly=true` compiles the case that runs and leaves the rest out. Measured rather than argued: v2.1.0 and v2.2.0 now build and run, and v2.1.0 produced a full set of figures -- fifteen cases with means and allocations, written to measured-CommonFunctionsInterVersion.json the way every other row is. The unflagged build is unchanged and still compiles every case. The note this replaces said reaching further back would mean conditioning the benchmark project on which commit it is pointed at, and that this would break the comparison because the reason the columns compare is that the benchmark does not vary. The first half was right and the second does not follow. The flag is per-purpose, not per-commit: the script passes it for every row including master, so each column compiles the same files and runs the same file. Conditioning per commit would have broken the comparison; conditioning per purpose does not touch it. Closes #529 -- the measurement half landed in #1140, and this was the limitation recorded against it. The two steps deliberately left out there are unchanged and remain decisions rather than work: the schedule is weekly plus workflow_dispatch rather than daily, and nothing is pushed to a website because neither AngouriMathLab/performance-reports nor its tools repository exists. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_012sonx8iAspMiwRwokT1Ura --- .../DotnetBenchmark/DotnetBenchmark.csproj | 24 +++++++++++++++++++ Sources/Tests/DotnetBenchmark/Program.cs | 14 +++++++---- Sources/Tests/DotnetBenchmark/key-commits.txt | 21 +++++++++++----- Sources/Utils/benchmark_key_commits.sh | 14 ++++++++++- 4 files changed, 62 insertions(+), 11 deletions(-) diff --git a/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj b/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj index 0037a406c..74d0f9d08 100644 --- a/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj +++ b/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj @@ -12,6 +12,30 @@ ..\..\AngouriMath\key.snk + + + + + + + + + + $(DefineConstants);INTERVERSION_ONLY + + diff --git a/Sources/Tests/DotnetBenchmark/Program.cs b/Sources/Tests/DotnetBenchmark/Program.cs index 6c8dd50b5..76dc1d244 100644 --- a/Sources/Tests/DotnetBenchmark/Program.cs +++ b/Sources/Tests/DotnetBenchmark/Program.cs @@ -79,12 +79,17 @@ public static int Main(string[] args) .Select(arg => arg switch { - "RAMUsageTest" => GetReportByBenchmark(typeof(RAMUsageTest), "Gen 0", "Gen 1", "Gen 2", "Allocated"), - // Allocated: the class has carried [MemoryDiagnoser] all along, so the - // figure was being collected on every run and then dropped here. - "CommonFunctionsInterVersion" => GetReportByBenchmark(typeof(CommonFunctionsInterVersion), "Mean", "Error", "StdDev", "Allocated"), +// Every case but the inter-version one is compiled out when the project is built for the +// key-commit run, because a case that reaches API an older kernel did not have stops the whole +// project compiling and takes the row with it. Only CommonFunctionsInterVersion is executed by +// that run. https://github.com/asc-community/AngouriMath/issues/529 // Allocated as well as Mean: the regressions this one exists to catch // show up in allocation and are invisible in the timings. + "CommonFunctionsInterVersion" => GetReportByBenchmark(typeof(CommonFunctionsInterVersion), "Mean", "Error", "StdDev", "Allocated"), +#if !INTERVERSION_ONLY + // Allocated: the class has carried [MemoryDiagnoser] all along, so the + // figure was being collected on every run and then dropped here. + "RAMUsageTest" => GetReportByBenchmark(typeof(RAMUsageTest), "Gen 0", "Gen 1", "Gen 2", "Allocated"), "TransformationLayer" => GetReportByBenchmark(typeof(TransformationLayer), "Mean", "Error", "StdDev", "Allocated"), // Ratio as well: this one exists to compare two forms of the same rule, // and the absolute nanoseconds matter far less than the factor between @@ -97,6 +102,7 @@ public static int Main(string[] args) "BenchLinqCompilation" => GetReportByBenchmark(typeof(BenchLinqCompilation), "Mean", "Error", "StdDev", "Allocated"), "CacheCompiledFunc" => GetReportByBenchmark(typeof(CacheCompiledFunc), "Mean", "Error", "StdDev", "Allocated"), "NumbersBenchmark" => GetReportByBenchmark(typeof(NumbersBenchmark), "Mean", "Error", "StdDev"), +#endif _ => throw new($"Unexpected benchmark {arg}") }).ToArray(); // active action Console.WriteLine(); diff --git a/Sources/Tests/DotnetBenchmark/key-commits.txt b/Sources/Tests/DotnetBenchmark/key-commits.txt index 02d5f2ab9..aa0dcf99c 100644 --- a/Sources/Tests/DotnetBenchmark/key-commits.txt +++ b/Sources/Tests/DotnetBenchmark/key-commits.txt @@ -19,12 +19,21 @@ # is reported as not building rather than skipped, since "this cannot be measured any more" is # itself worth seeing. # -# Measured, so the bound is known rather than discovered. At the time of writing v2.3.0 and later -# build; v2.2.0 and earlier do not, because the benchmark's MatchingEngine case reaches -# `MatchedRules` and `Patterns`, which were internal until v2.3.0. Reaching further back means -# compiling only the inter-version case, which means conditioning the benchmark project on which -# commit it is pointed at -- and the whole reason these columns compare is that the benchmark does -# not vary. Left as it is; the table says which rows it could not measure. +# How far back this reaches, and why it is further than it was. v2.3.0 and later used to be the +# bound: v2.2.0 and earlier did not build, because the benchmark's MatchingEngine case reaches +# `MatchedRules` and `Patterns`, which were internal until v2.3.0, and TransformationLayer names +# `Transformation.Rationalization`, which did not exist at v2.1.0. +# +# Neither of those cases is executed by this run. Only CommonFunctionsInterVersion is, and it uses +# nothing but the public API -- so the rows were being lost to files that were compiled and never +# called. `-p:InterVersionOnly=true` compiles the case that runs and leaves the rest out, and the +# script passes it for every row. +# +# The earlier note here said reaching further back would mean conditioning the project on which +# commit it is pointed at, and that this would break the comparison because the benchmark must not +# vary. The first half was right and the second does not follow: the flag is per-purpose, not +# per-commit, so every column compiles the same file and runs the same file. Measured: v2.1.0 and +# v2.2.0 build and run under it. v2.1.0 v2.2.0 diff --git a/Sources/Utils/benchmark_key_commits.sh b/Sources/Utils/benchmark_key_commits.sh index 644bb1247..31fb8daa6 100755 --- a/Sources/Utils/benchmark_key_commits.sh +++ b/Sources/Utils/benchmark_key_commits.sh @@ -18,6 +18,13 @@ # calls the public API as it is now, so an old enough commit genuinely cannot be measured this # way, and that is worth seeing rather than silently omitting. # +# What stopped older commits building was not that case, though. It was the *other* cases in the +# same project -- MatchingEngine reaches internals that were internal until v2.3.0, and +# TransformationLayer names Transformation.Rationalization, which did not exist at v2.1.0 -- so +# the project failed to compile and took the row with it while the case that runs would have +# compiled fine. `-p:InterVersionOnly=true` leaves those out. It is passed for every row, so the +# columns still compare: the same file is compiled and the same file runs in each of them. +# # Run from anywhere. set -uo pipefail @@ -60,7 +67,12 @@ for commit in $commits; do cp -r "$repository/$benchmark" "$tree/Sources/Tests/" rm -rf "$tree/$benchmark/bin" "$tree/$benchmark/obj" "$tree/$benchmark/benchmark_results.csv" - if ! (cd "$tree/$benchmark" && dotnet build -c Release > "$results/$commit.build.log" 2>&1); then + # InterVersionOnly compiles the one case this run executes and leaves out the rest. A case + # that reaches API an older kernel did not have stops the whole project compiling and takes + # the row with it: MatchedRules and Patterns were internal until v2.3.0, and + # Transformation.Rationalization did not exist until after v2.1.0. Passed for every row + # including master, so what is compiled does not vary between columns. + if ! (cd "$tree/$benchmark" && dotnet build -c Release -p:InterVersionOnly=true > "$results/$commit.build.log" 2>&1); then echo " did not build — see $results/$commit.build.log" echo "did not build" > "$results/$commit.status" continue