Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions docs/benchmarking-workflow-dotnet-runtime.md
Original file line number Diff line number Diff line change
Expand Up @@ -183,6 +183,14 @@ Make sure you have the v8 engine installed and in the PATH. Follow the installat

#### Run the benchmarks with the interpreter

`NoWASM` excludes benchmarks that still require WASM-specific support. Async
benchmarks are not excluded solely because they return `Task` or `ValueTask`:
BenchmarkDotNet awaits them by default. The self-contained async benchmarks
in `Perf_AsyncMethods`, `ValueTaskPerfTest`, `MemoryStreamTests`, and
`MemoryStreamChunkedTests` are eligible for WASM runs. Other benchmarks retain
their exclusions until their platform dependencies and runtime behavior are
validated individually.

1. Configure NuGet Feeds to include local packages. Open the `NuGet.config` and modify `packageSources` section

```xml
Expand All @@ -207,6 +215,11 @@ Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`:

#### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK

For CoreCLR WASM, `MicroBenchmarks.Wasm.targets` sets `RuntimeFlavor=CoreCLR`
after BenchmarkDotNet applies `UseMonoRuntime=false`. This lets the WebAssembly
SDK select the correct Webcil format, with or without ReadyToRun, while leaving
Mono configuration unchanged.

Use an exact SDK version together with a NuGet source containing the matching
browser runtime/ref, WebAssembly SDK, host-specific Crossgen2, and ILLink
packages, plus the complete `wasm-tools` pack closure. The source can be a
Expand Down
60 changes: 60 additions & 0 deletions scripts/tests/test_wasm_coreclr_r2r.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,7 @@
import json
import os
import shutil
import subprocess
import sys
import xml.etree.ElementTree as ET
from argparse import Namespace
Expand Down Expand Up @@ -136,6 +139,63 @@ def test_ready_to_run_validates_resolved_runtime_pack_items():
assert all("_PerformanceWasmRuntimePackName" not in condition for condition in conditions)


@pytest.mark.parametrize("ready_to_run", ["false", "true"])
@pytest.mark.parametrize(
("use_mono_runtime", "initial_flavor", "expected_flavor"),
[
("false", "", "CoreCLR"),
("false", "Mono", "CoreCLR"),
("true", "", ""),
("true", "Mono", "Mono"),
("", "", ""),
("", "Mono", "Mono"),
],
)
def test_wasm_runtime_flavor_after_generated_project_imports(
tmp_path, ready_to_run, use_mono_runtime, initial_flavor, expected_flavor):
dotnet_cli = shutil.which("dotnet")
if dotnet_cli is None:
pytest.skip("A .NET SDK is required for MSBuild property evaluation.")

micro_dir = scripts_dir.parent / "src" / "benchmarks" / "micro"
project = ET.Element("Project")
properties = ET.SubElement(project, "PropertyGroup")
for name, value in {
"UseMonoRuntime": "true",
"RuntimeFlavor": initial_flavor,
"PERFLAB_WASM_READY_TO_RUN": ready_to_run,
"PublishReadyToRun": "false",
}.items():
ET.SubElement(properties, name).text = value

# BenchmarkDotNet overrides UseMonoRuntime after .props but before .targets.
ET.SubElement(project, "Import", Project=str(micro_dir / "MicroBenchmarks.Wasm.props"))
properties = ET.SubElement(project, "PropertyGroup")
ET.SubElement(properties, "UseMonoRuntime").text = use_mono_runtime
ET.SubElement(project, "Import", Project=str(micro_dir / "MicroBenchmarks.Wasm.targets"))
project_path = tmp_path / "GeneratedWasm.proj"
ET.ElementTree(project).write(project_path, encoding="utf-8")

result = subprocess.run(
[
dotnet_cli, "msbuild", str(project_path), "-nologo",
"-getProperty:UseMonoRuntime,RuntimeFlavor,PublishReadyToRun",
],
cwd=tmp_path,
capture_output=True,
text=True,
timeout=60,
)

assert result.returncode == 0, result.stdout + result.stderr
evaluated = json.loads(result.stdout)["Properties"]
assert evaluated["UseMonoRuntime"] == use_mono_runtime
assert evaluated["RuntimeFlavor"] == expected_flavor
assert evaluated["PublishReadyToRun"] == (
"true" if use_mono_runtime == "false" and ready_to_run == "true" else "false"
)


def test_coreclr_payload_detects_local_toolchain_package_version(tmp_path):
artifact = tmp_path / "artifact" / "staging"
ref_pack = artifact / "dotnet-none" / "packs" / "Microsoft.NETCore.App.Ref" / "11.0.0-rc.1.26431.109"
Expand Down
5 changes: 5 additions & 0 deletions src/benchmarks/micro/MicroBenchmarks.Wasm.targets
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,11 @@
<_WasmStrictVersionMatch>false</_WasmStrictVersionMatch>
</PropertyGroup>

<PropertyGroup Condition="'$(UseMonoRuntime)' == 'false'">
<!-- BenchmarkDotNet sets UseMonoRuntime after importing .Wasm.props. -->
<RuntimeFlavor>CoreCLR</RuntimeFlavor>
</PropertyGroup>

<PropertyGroup Condition="'$(PERFLAB_WASM_READY_TO_RUN)' == 'true' and '$(UseMonoRuntime)' == 'false'">
<PublishReadyToRun>true</PublishReadyToRun>
<PublishReadyToRunComposite>false</PublishReadyToRunComposite>
Expand Down
3 changes: 1 addition & 2 deletions src/benchmarks/micro/Program.cs
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,7 @@ static async Task<int> Main(string[] args)
exclusionFilterValue: exclusionFilterValue,
categoryExclusionFilterValue: categoryExclusionFilterValue,
getDiffableDisasm: getDiffableDisasm,
msBuildArguments: msBuildArguments)
.AddValidator(new NoWasmValidator(Categories.NoWASM)))
msBuildArguments: msBuildArguments))
.ConfigureAwait(false);

return summaries.ToExitCode();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public int ReadSpan()
}

[Benchmark]
[BenchmarkCategory(Categories.NoWASM)]
[MemoryRandomization]
public async Task<int> ReadAsyncMemory()
{
Expand Down Expand Up @@ -113,7 +112,6 @@ public void WriteSpan()
}

[Benchmark]
[BenchmarkCategory(Categories.NoWASM)]
[MemoryRandomization]
public async Task WriteAsyncMemory()
{
Expand All @@ -138,7 +136,6 @@ public void CopyToWithBufferSize()
}

[Benchmark]
[BenchmarkCategory(Categories.NoWASM)]
[MemoryRandomization]
public async Task CopyToAsyncWithBufferSize()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ public void CopyTo()
}

[Benchmark]
[BenchmarkCategory(Categories.NoWASM)]
[MemoryRandomization]
public async Task CopyToAsync()
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@

namespace System.Threading.Tasks
{
[BenchmarkCategory(Categories.Libraries, Categories.NoWASM)]
[BenchmarkCategory(Categories.Libraries)]
[MinWarmupCount(2, forceAutoWarmup: true)] // these benchmarks require more warmups than in our default config
[MaxWarmupCount(10, forceAutoWarmup: true)]
public class ValueTaskPerfTest
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace System.Threading.Tasks.Tests
{
[BenchmarkCategory(Categories.Libraries, Categories.NoWASM)]
[BenchmarkCategory(Categories.Libraries)]
public class Perf_AsyncMethods
{
[Benchmark(OperationsPerInvoke = 100_000)]
Expand Down
47 changes: 0 additions & 47 deletions src/harness/BenchmarkDotNet.Extensions/NoWasmValidator.cs

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.

using System;
using System.IO.Pipelines.Tests;
using System.IO.Tests;
using System.Linq;
using System.Threading.Tasks;
using System.Threading.Tasks.Tests;
using BenchmarkDotNet.Running;
using MicroBenchmarks;
using Xunit;

namespace Tests
{
public class WasmAsyncBenchmarkSelectionTests
{
[Theory]
[InlineData(typeof(Perf_AsyncMethods), nameof(Perf_AsyncMethods.Yield))]
[InlineData(typeof(ValueTaskPerfTest), nameof(ValueTaskPerfTest.CreateAndAwait_FromYieldingAsyncMethod))]
[InlineData(typeof(MemoryStreamTests), nameof(MemoryStreamTests.CopyToAsync))]
[InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.ReadAsyncMemory))]
[InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.WriteAsyncMemory))]
[InlineData(typeof(MemoryStreamChunkedTests), nameof(MemoryStreamChunkedTests.CopyToAsyncWithBufferSize))]
public void AsyncBenchmarkIsNotExcludedFromWasm(Type benchmarkType, string methodName)
{
var benchmarks = BenchmarkConverter.TypeToBenchmarks(benchmarkType).BenchmarksCases
.Where(benchmark => benchmark.Descriptor.WorkloadMethod.Name == methodName)
.ToArray();

Assert.NotEmpty(benchmarks);
Assert.All(benchmarks, benchmark => Assert.DoesNotContain(Categories.NoWASM, benchmark.Descriptor.Categories));
}

[Fact]
public void OtherAsyncBenchmarksRemainExcludedFromWasm()
{
var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(Perf_Pipe)).BenchmarksCases;

Assert.NotEmpty(benchmarks);
Assert.All(benchmarks, benchmark => Assert.Contains(Categories.NoWASM, benchmark.Descriptor.Categories));
}
}
}
Loading