diff --git a/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj b/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj index 0037a406c..74d0f9d08 100644 --- a/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj +++ b/Sources/Tests/DotnetBenchmark/DotnetBenchmark.csproj @@ -12,6 +12,30 @@ ..\..\AngouriMath\key.snk + + + + + + + + + + $(DefineConstants);INTERVERSION_ONLY + + diff --git a/Sources/Tests/DotnetBenchmark/Program.cs b/Sources/Tests/DotnetBenchmark/Program.cs index 6c8dd50b5..76dc1d244 100644 --- a/Sources/Tests/DotnetBenchmark/Program.cs +++ b/Sources/Tests/DotnetBenchmark/Program.cs @@ -79,12 +79,17 @@ public static int Main(string[] args) .Select(arg => arg switch { - "RAMUsageTest" => GetReportByBenchmark(typeof(RAMUsageTest), "Gen 0", "Gen 1", "Gen 2", "Allocated"), - // Allocated: the class has carried [MemoryDiagnoser] all along, so the - // figure was being collected on every run and then dropped here. - "CommonFunctionsInterVersion" => GetReportByBenchmark(typeof(CommonFunctionsInterVersion), "Mean", "Error", "StdDev", "Allocated"), +// Every case but the inter-version one is compiled out when the project is built for the +// key-commit run, because a case that reaches API an older kernel did not have stops the whole +// project compiling and takes the row with it. Only CommonFunctionsInterVersion is executed by +// that run. https://github.com/asc-community/AngouriMath/issues/529 // Allocated as well as Mean: the regressions this one exists to catch // show up in allocation and are invisible in the timings. + "CommonFunctionsInterVersion" => GetReportByBenchmark(typeof(CommonFunctionsInterVersion), "Mean", "Error", "StdDev", "Allocated"), +#if !INTERVERSION_ONLY + // Allocated: the class has carried [MemoryDiagnoser] all along, so the + // figure was being collected on every run and then dropped here. + "RAMUsageTest" => GetReportByBenchmark(typeof(RAMUsageTest), "Gen 0", "Gen 1", "Gen 2", "Allocated"), "TransformationLayer" => GetReportByBenchmark(typeof(TransformationLayer), "Mean", "Error", "StdDev", "Allocated"), // Ratio as well: this one exists to compare two forms of the same rule, // and the absolute nanoseconds matter far less than the factor between @@ -97,6 +102,7 @@ public static int Main(string[] args) "BenchLinqCompilation" => GetReportByBenchmark(typeof(BenchLinqCompilation), "Mean", "Error", "StdDev", "Allocated"), "CacheCompiledFunc" => GetReportByBenchmark(typeof(CacheCompiledFunc), "Mean", "Error", "StdDev", "Allocated"), "NumbersBenchmark" => GetReportByBenchmark(typeof(NumbersBenchmark), "Mean", "Error", "StdDev"), +#endif _ => throw new($"Unexpected benchmark {arg}") }).ToArray(); // active action Console.WriteLine(); diff --git a/Sources/Tests/DotnetBenchmark/key-commits.txt b/Sources/Tests/DotnetBenchmark/key-commits.txt index 02d5f2ab9..aa0dcf99c 100644 --- a/Sources/Tests/DotnetBenchmark/key-commits.txt +++ b/Sources/Tests/DotnetBenchmark/key-commits.txt @@ -19,12 +19,21 @@ # is reported as not building rather than skipped, since "this cannot be measured any more" is # itself worth seeing. # -# Measured, so the bound is known rather than discovered. At the time of writing v2.3.0 and later -# build; v2.2.0 and earlier do not, because the benchmark's MatchingEngine case reaches -# `MatchedRules` and `Patterns`, which were internal until v2.3.0. Reaching further back means -# compiling only the inter-version case, which means conditioning the benchmark project on which -# commit it is pointed at -- and the whole reason these columns compare is that the benchmark does -# not vary. Left as it is; the table says which rows it could not measure. +# How far back this reaches, and why it is further than it was. v2.3.0 and later used to be the +# bound: v2.2.0 and earlier did not build, because the benchmark's MatchingEngine case reaches +# `MatchedRules` and `Patterns`, which were internal until v2.3.0, and TransformationLayer names +# `Transformation.Rationalization`, which did not exist at v2.1.0. +# +# Neither of those cases is executed by this run. Only CommonFunctionsInterVersion is, and it uses +# nothing but the public API -- so the rows were being lost to files that were compiled and never +# called. `-p:InterVersionOnly=true` compiles the case that runs and leaves the rest out, and the +# script passes it for every row. +# +# The earlier note here said reaching further back would mean conditioning the project on which +# commit it is pointed at, and that this would break the comparison because the benchmark must not +# vary. The first half was right and the second does not follow: the flag is per-purpose, not +# per-commit, so every column compiles the same file and runs the same file. Measured: v2.1.0 and +# v2.2.0 build and run under it. v2.1.0 v2.2.0 diff --git a/Sources/Utils/benchmark_key_commits.sh b/Sources/Utils/benchmark_key_commits.sh index 644bb1247..31fb8daa6 100755 --- a/Sources/Utils/benchmark_key_commits.sh +++ b/Sources/Utils/benchmark_key_commits.sh @@ -18,6 +18,13 @@ # calls the public API as it is now, so an old enough commit genuinely cannot be measured this # way, and that is worth seeing rather than silently omitting. # +# What stopped older commits building was not that case, though. It was the *other* cases in the +# same project -- MatchingEngine reaches internals that were internal until v2.3.0, and +# TransformationLayer names Transformation.Rationalization, which did not exist at v2.1.0 -- so +# the project failed to compile and took the row with it while the case that runs would have +# compiled fine. `-p:InterVersionOnly=true` leaves those out. It is passed for every row, so the +# columns still compare: the same file is compiled and the same file runs in each of them. +# # Run from anywhere. set -uo pipefail @@ -60,7 +67,12 @@ for commit in $commits; do cp -r "$repository/$benchmark" "$tree/Sources/Tests/" rm -rf "$tree/$benchmark/bin" "$tree/$benchmark/obj" "$tree/$benchmark/benchmark_results.csv" - if ! (cd "$tree/$benchmark" && dotnet build -c Release > "$results/$commit.build.log" 2>&1); then + # InterVersionOnly compiles the one case this run executes and leaves out the rest. A case + # that reaches API an older kernel did not have stops the whole project compiling and takes + # the row with it: MatchedRules and Patterns were internal until v2.3.0, and + # Transformation.Rationalization did not exist until after v2.1.0. Passed for every row + # including master, so what is compiled does not vary between columns. + if ! (cd "$tree/$benchmark" && dotnet build -c Release -p:InterVersionOnly=true > "$results/$commit.build.log" 2>&1); then echo " did not build — see $results/$commit.build.log" echo "did not build" > "$results/$commit.status" continue