diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index 53ac9b1ceb1..e10416a0078 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -183,6 +183,14 @@ Make sure you have the v8 engine installed and in the PATH. Follow the installat #### Run the benchmarks with the interpreter +`NoWASM` excludes benchmarks that still require WASM-specific support. Async +benchmarks are not excluded solely because they return `Task` or `ValueTask`: +BenchmarkDotNet awaits them by default. The self-contained async benchmarks +in `Perf_AsyncMethods`, `ValueTaskPerfTest`, `MemoryStreamTests`, and +`MemoryStreamChunkedTests` are eligible for WASM runs. Other benchmarks retain +their exclusions until their platform dependencies and runtime behavior are +validated individually. + 1. Configure NuGet Feeds to include local packages. Open the `NuGet.config` and modify `packageSources` section ```xml @@ -207,6 +215,11 @@ Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`: #### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK +For CoreCLR WASM, `MicroBenchmarks.Wasm.targets` sets `RuntimeFlavor=CoreCLR` +after BenchmarkDotNet applies `UseMonoRuntime=false`. This lets the WebAssembly +SDK select the correct Webcil format, with or without ReadyToRun, while leaving +Mono configuration unchanged. + Use an exact SDK version together with a NuGet source containing the matching browser runtime/ref, WebAssembly SDK, host-specific Crossgen2, and ILLink packages, plus the complete `wasm-tools` pack closure. The source can be a diff --git a/scripts/tests/test_wasm_coreclr_r2r.py b/scripts/tests/test_wasm_coreclr_r2r.py index e59f34a4327..2ae08bcbf51 100644 --- a/scripts/tests/test_wasm_coreclr_r2r.py +++ b/scripts/tests/test_wasm_coreclr_r2r.py @@ -1,4 +1,7 @@ +import json import os +import shutil +import subprocess import sys import xml.etree.ElementTree as ET from argparse import Namespace @@ -136,6 +139,63 @@ def test_ready_to_run_validates_resolved_runtime_pack_items(): assert all("_PerformanceWasmRuntimePackName" not in condition for condition in conditions) +@pytest.mark.parametrize("ready_to_run", ["false", "true"]) +@pytest.mark.parametrize( + ("use_mono_runtime", "initial_flavor", "expected_flavor"), + [ + ("false", "", "CoreCLR"), + ("false", "Mono", "CoreCLR"), + ("true", "", ""), + ("true", "Mono", "Mono"), + ("", "", ""), + ("", "Mono", "Mono"), + ], +) +def test_wasm_runtime_flavor_after_generated_project_imports( + tmp_path, ready_to_run, use_mono_runtime, initial_flavor, expected_flavor): + dotnet_cli = shutil.which("dotnet") + if dotnet_cli is None: + pytest.skip("A .NET SDK is required for MSBuild property evaluation.") + + micro_dir = scripts_dir.parent / "src" / "benchmarks" / "micro" + project = ET.Element("Project") + properties = ET.SubElement(project, "PropertyGroup") + for name, value in { + "UseMonoRuntime": "true", + "RuntimeFlavor": initial_flavor, + "PERFLAB_WASM_READY_TO_RUN": ready_to_run, + "PublishReadyToRun": "false", + }.items(): + ET.SubElement(properties, name).text = value + + # BenchmarkDotNet overrides UseMonoRuntime after .props but before .targets. + ET.SubElement(project, "Import", Project=str(micro_dir / "MicroBenchmarks.Wasm.props")) + properties = ET.SubElement(project, "PropertyGroup") + ET.SubElement(properties, "UseMonoRuntime").text = use_mono_runtime + ET.SubElement(project, "Import", Project=str(micro_dir / "MicroBenchmarks.Wasm.targets")) + project_path = tmp_path / "GeneratedWasm.proj" + ET.ElementTree(project).write(project_path, encoding="utf-8") + + result = subprocess.run( + [ + dotnet_cli, "msbuild", str(project_path), "-nologo", + "-getProperty:UseMonoRuntime,RuntimeFlavor,PublishReadyToRun", + ], + cwd=tmp_path, + capture_output=True, + text=True, + timeout=60, + ) + + assert result.returncode == 0, result.stdout + result.stderr + evaluated = json.loads(result.stdout)["Properties"] + assert evaluated["UseMonoRuntime"] == use_mono_runtime + assert evaluated["RuntimeFlavor"] == expected_flavor + assert evaluated["PublishReadyToRun"] == ( + "true" if use_mono_runtime == "false" and ready_to_run == "true" else "false" + ) + + def test_coreclr_payload_detects_local_toolchain_package_version(tmp_path): artifact = tmp_path / "artifact" / "staging" ref_pack = artifact / "dotnet-none" / "packs" / "Microsoft.NETCore.App.Ref" / "11.0.0-rc.1.26431.109" diff --git a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets index b6e4216cf79..e1a4340fe4c 100644 --- a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets +++ b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets @@ -10,6 +10,11 @@ <_WasmStrictVersionMatch>false + + + CoreCLR + + true false diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 28e9eca5312..bca89678baa 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -59,8 +59,7 @@ static async Task Main(string[] args) exclusionFilterValue: exclusionFilterValue, categoryExclusionFilterValue: categoryExclusionFilterValue, getDiffableDisasm: getDiffableDisasm, - msBuildArguments: msBuildArguments) - .AddValidator(new NoWasmValidator(Categories.NoWASM))) + msBuildArguments: msBuildArguments)) .ConfigureAwait(false); return summaries.ToExitCode(); diff --git a/src/benchmarks/micro/libraries/System.IO/MemoryStreamChunkedTests.cs b/src/benchmarks/micro/libraries/System.IO/MemoryStreamChunkedTests.cs index 46ec5b66949..7999fa042b9 100644 --- a/src/benchmarks/micro/libraries/System.IO/MemoryStreamChunkedTests.cs +++ b/src/benchmarks/micro/libraries/System.IO/MemoryStreamChunkedTests.cs @@ -69,7 +69,6 @@ public int ReadSpan() } [Benchmark] - [BenchmarkCategory(Categories.NoWASM)] [MemoryRandomization] public async Task ReadAsyncMemory() { @@ -113,7 +112,6 @@ public void WriteSpan() } [Benchmark] - [BenchmarkCategory(Categories.NoWASM)] [MemoryRandomization] public async Task WriteAsyncMemory() { @@ -138,7 +136,6 @@ public void CopyToWithBufferSize() } [Benchmark] - [BenchmarkCategory(Categories.NoWASM)] [MemoryRandomization] public async Task CopyToAsyncWithBufferSize() { diff --git a/src/benchmarks/micro/libraries/System.IO/MemoryStreamTests.cs b/src/benchmarks/micro/libraries/System.IO/MemoryStreamTests.cs index 24db4f653ee..a1999513dd5 100644 --- a/src/benchmarks/micro/libraries/System.IO/MemoryStreamTests.cs +++ b/src/benchmarks/micro/libraries/System.IO/MemoryStreamTests.cs @@ -69,7 +69,6 @@ public void CopyTo() } [Benchmark] - [BenchmarkCategory(Categories.NoWASM)] [MemoryRandomization] public async Task CopyToAsync() { diff --git a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs index c02f2653bd7..b4b04834303 100644 --- a/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs +++ b/src/benchmarks/micro/libraries/System.Threading.Tasks.Extensions/Perf.ValueTask.cs @@ -10,7 +10,7 @@ namespace System.Threading.Tasks { - [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)] + [BenchmarkCategory(Categories.Libraries)] [MinWarmupCount(2, forceAutoWarmup: true)] // these benchmarks require more warmups than in our default config [MaxWarmupCount(10, forceAutoWarmup: true)] public class ValueTaskPerfTest diff --git a/src/benchmarks/micro/libraries/System.Threading.Tasks/Perf.AsyncMethods.cs b/src/benchmarks/micro/libraries/System.Threading.Tasks/Perf.AsyncMethods.cs index 38dbaaccf30..efc6fac81ea 100644 --- a/src/benchmarks/micro/libraries/System.Threading.Tasks/Perf.AsyncMethods.cs +++ b/src/benchmarks/micro/libraries/System.Threading.Tasks/Perf.AsyncMethods.cs @@ -9,7 +9,7 @@ namespace System.Threading.Tasks.Tests { - [BenchmarkCategory(Categories.Libraries, Categories.NoWASM)] + [BenchmarkCategory(Categories.Libraries)] public class Perf_AsyncMethods { [Benchmark(OperationsPerInvoke = 100_000)] diff --git a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs b/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs deleted file mode 100644 index 68fe73a4645..00000000000 --- a/src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs +++ /dev/null @@ -1,47 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Reflection; -using System.Threading.Tasks; -using BenchmarkDotNet.Validators; - -namespace BenchmarkDotNet.Extensions -{ - /// - /// This class makes sure that every benchmark that returns an awaitable object belongs to the "NoWASM" category. - /// CI uses it to filter out multithreaded benchmarks, which are not supported by WASM. - /// - public class NoWasmValidator : IValidator - { - private readonly string _noWasmCategory; - - public bool TreatsWarningsAsErrors => true; - - public NoWasmValidator(string noWasmCategory) => _noWasmCategory = noWasmCategory; - - public IAsyncEnumerable ValidateAsync(ValidationParameters validationParameters) - => validationParameters.Benchmarks - .Where(benchmark => IsAsyncMethod(benchmark.Descriptor.WorkloadMethod) && !benchmark.Descriptor.Categories.Any(category => category.Equals(_noWasmCategory, StringComparison.Ordinal))) - .Select(benchmark => benchmark.Descriptor.GetFilterName()) - .Distinct() - .Select(benchmarkId => - new ValidationError( - isCritical: TreatsWarningsAsErrors, - $"{benchmarkId} returns an awaitable object and has no: {_noWasmCategory} category applied. Use [BenchmarkCategory(Categories.NoWASM)]") - ) - .ToAsyncEnumerable(); - - private bool IsAsyncMethod(MethodInfo workloadMethod) - { - Type returnType = workloadMethod.ReturnType; - - return returnType == typeof(Task) - || returnType == typeof(ValueTask) - || (returnType.IsGenericType && (returnType.GetGenericTypeDefinition() == typeof(Task<>) || returnType.GetGenericTypeDefinition() == typeof(ValueTask<>))); - } - } -} \ No newline at end of file diff --git a/src/tests/harness/BenchmarkDotNet.Extensions.Tests/WasmAsyncBenchmarkSelectionTests.cs b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/WasmAsyncBenchmarkSelectionTests.cs new file mode 100644 index 00000000000..cc7ca3b0c5e --- /dev/null +++ b/src/tests/harness/BenchmarkDotNet.Extensions.Tests/WasmAsyncBenchmarkSelectionTests.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO.Pipelines.Tests; +using System.IO.Tests; +using System.Linq; +using System.Threading.Tasks; +using System.Threading.Tasks.Tests; +using BenchmarkDotNet.Running; +using MicroBenchmarks; +using Xunit; + +namespace Tests +{ + public class WasmAsyncBenchmarkSelectionTests + { + [Theory] + [InlineData(typeof(Perf_AsyncMethods), nameof(Perf_AsyncMethods.Yield))] + [InlineData(typeof(ValueTaskPerfTest), nameof(ValueTaskPerfTest.CreateAndAwait_FromYieldingAsyncMethod))] + [InlineData(typeof(MemoryStreamTests), nameof(MemoryStreamTests.CopyToAsync))] + [InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.ReadAsyncMemory))] + [InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.WriteAsyncMemory))] + [InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.CopyToAsyncWithBufferSize))] + public void AsyncBenchmarkIsNotExcludedFromWasm(Type benchmarkType, string methodName) + { + var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType).BenchmarksCases + .Where(benchmark => benchmark.Descriptor.WorkloadMethod.Name == methodName) + .ToArray(); + + Assert.NotEmpty(benchmarks); + Assert.All(benchmarks, benchmark => Assert.DoesNotContain(Categories.NoWASM, benchmark.Descriptor.Categories)); + } + + [Fact] + public void OtherAsyncBenchmarksRemainExcludedFromWasm() + { + var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(Perf_Pipe)).BenchmarksCases; + + Assert.NotEmpty(benchmarks); + Assert.All(benchmarks, benchmark => Assert.Contains(Categories.NoWASM, benchmark.Descriptor.Categories)); + } + } +}