diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json new file mode 100644 index 00000000000..09c845ee7e9 --- /dev/null +++ b/.config/dotnet-tools.json @@ -0,0 +1,12 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "microsoft.dotnet.helix.jobmonitor": { + "version": "11.0.0-beta.26473.104", + "commands": [ + "dotnet-helix-job-monitor" + ] + } + } +} diff --git a/NuGet.config b/NuGet.config index c3dce14f9aa..accbce0f625 100644 --- a/NuGet.config +++ b/NuGet.config @@ -7,7 +7,8 @@ - + + diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4cd188a6a74..9a4cdcf2bac 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -12,6 +12,18 @@ parameters: type: boolean default: false +variables: + # This entrypoint has one implicit stage. Keep asynchronous sends paired with its monitor. + - name: EnableHelixJobMonitor + value: true + # Matrix legs share a phase and queue. Keep their monitor identities distinct across channels. + - name: TestRunNamePrefix + value: '$(Agent.JobName)_' + - template: /eng/common/templates/variables/pool-providers.yml + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + # The standalone monitor does not inherit the submitter jobs' variable groups. + - group: DotNet-HelixApi-Access + resources: containers: - container: ubuntu_x64_build_container @@ -67,3 +79,9 @@ jobs: jobParameters: ${{ if parameters.onlySanityCheck }}: onlySanityCheck: true + + - template: /eng/common/core-templates/job/helix-job-monitor.yml + parameters: + timeoutInMinutes: 360 + helixAccessToken: $(HelixApiAccessToken) + allowNoHelixJobs: false diff --git a/docs/benchmarkdotnet.md b/docs/benchmarkdotnet.md index be429f57ad7..657c56bb8b9 100644 --- a/docs/benchmarkdotnet.md +++ b/docs/benchmarkdotnet.md @@ -27,7 +27,6 @@ BenchmarkDotNet is the benchmarking tool that allows to run benchmarks for .NET, - [Running In Process](#running-in-process) - [CoreRun](#corerun) - [dotnet cli](#dotnet-cli) - - [Private CLR Build](#private-clr-build) - [Private CoreRT Build](#private-corert-build) ## Main Concepts @@ -117,6 +116,12 @@ And select one of the benchmarks from the list by either entering its number or ### Command Line +The microbenchmark `--msbuild-arguments` option passes additional arguments to MSBuild for BenchmarkDotNet's generated project. Use slash-prefixed arguments such as `/p:` or `/t:`, not `-p:` or `-t:`: `CommandLineOptions.ParseAndRemoveStringsParameter` stops collecting values at any token beginning with `-`. Embedded `--` within a `/p:` value is fine, for example: + +```text +--msbuild-arguments /p:PublishReadyToRunCrossgen2ExtraArgs=--target-allows-runtime-code-generation:false +``` + #### Filtering the Benchmarks You can filter the benchmarks using `--filter $globPattern` console line argument. The filter is **case insensitive**. diff --git a/docs/benchmarking-workflow-dotnet-runtime.md b/docs/benchmarking-workflow-dotnet-runtime.md index 44d1ecd0d75..53ac9b1ceb1 100644 --- a/docs/benchmarking-workflow-dotnet-runtime.md +++ b/docs/benchmarking-workflow-dotnet-runtime.md @@ -11,6 +11,7 @@ - [Install v8 engine](#install-v8-engine) - [Run the benchmarks with the interpreter](#run-the-benchmarks-with-the-interpreter) - [Run the benchmarks with AOT](#run-the-benchmarks-with-aot) + - [Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK](#run-coreclr-wasm-readytorun-with-a-nightly-or-vmr-sdk) - [Note about "file ... being used by another process" error](#note-about-file--being-used-by-another-process-error) - [dotnet runtime testing for MonoAOT](#dotnet-runtime-testing-for-monoaot) - [Prerequisites (Files either built locally (with build.(sh/cmd) or downloaded from payload above (if same system setup) (in this order))](#prerequisites-files-either-built-locally-with-buildshcmd-or-downloaded-from-payload-above-if-same-system-setup-in-this-order) @@ -204,6 +205,35 @@ Essentially, add `--aotcompilermode wasm` to the `--bdn-arguments=".."`: --bdn-arguments="--category-exclusion-filter NoInterpreter NoWASM NoMono --aotcompilermode wasm --logBuildOutput --buildTimeout 3600 --filter " ``` +#### Run CoreCLR WASM ReadyToRun with a nightly or VMR SDK + +Use an exact SDK version together with a NuGet source containing the matching +browser runtime/ref, WebAssembly SDK, host-specific Crossgen2, and ILLink +packages, plus the complete `wasm-tools` pack closure. The source can be a +downloaded VMR package directory or a coherent feed. Workload manifest updates +are disabled and only exact versions recorded by the selected SDK are restored +from that source, so packages cannot float to another build cohort. + +```cmd +/path/to/dotnet/performance$ python3 ./scripts/benchmarks_ci.py \ + --csproj src/benchmarks/micro/MicroBenchmarks.csproj \ + -f net11.0 \ + --dotnet-versions \ + --wasm \ + --wasm-runtime-flavor CoreCLR \ + --wasm-ready-to-run \ + --wasm-workload-source /path/to/vmr/packages \ + --run-isolated \ + --bdn-artifacts artifacts/BenchmarkDotNet.Artifacts \ + --bdn-arguments="--category-exclusion-filter NoWASM --logBuildOutput --buildTimeout 3600 --filter " +``` + +Omit `--dotnet-versions` to use the existing `main` daily-channel SDK +selection. In that case the package source must contain the product version +recorded in that SDK's `Microsoft.NETCoreSdk.BundledVersions.props`. +`PERFLAB_WASM_PACKAGE_VERSION` should remain unset; that override is reserved +for non-official runtime payloads. + #### Note about "file ... being used by another process" error If you are seeing warnings like: diff --git a/docs/sdk-scenarios.md b/docs/sdk-scenarios.md index 12e2c952289..c72027e1323 100644 --- a/docs/sdk-scenarios.md +++ b/docs/sdk-scenarios.md @@ -4,6 +4,71 @@ An introduction of how to run scenario tests can be found in [Scenarios Tests Gu - [SDK Build Throughput Scenario](#sdk-build-throughput-scenario) +## SDK performance pipeline + +The repository's [azure-pipelines.yml](../azure-pipelines.yml) entrypoint uses +[sdk-perf-jobs.yml](../eng/pipelines/sdk-perf-jobs.yml) for SDK-based scenario and +benchmark jobs. It backs the internal `dotnet-performance` pipeline (definition 306) +and the public `performance-ci` pipeline (definition 38). + +The entrypoint always enables asynchronous submission and includes the standalone +Helix Job Monitor, its pool provider, and its parameters, including on public and PR +runs. The monitor runs alongside submitters in the same implicit stage, waits for +their Helix work items, and reports failures to Azure DevOps. The existing manual +job-selection parameters and schedules are unchanged: public runs always select +correctness jobs. Internal manual runs must select workloads; leaving all three +job-selection flags false still includes the monitor and fails because no Helix jobs +were submitted (`allowNoHelixJobs: false`). Submitter and work-item failures also +continue to fail the run. + +The monitor job has a six-hour timeout (355 minutes for the tool, leaving five minutes +for it to exit). Existing 320-minute submitter limits and Helix work-item timeouts are +unchanged. These limits start when the respective job runs; they are not a whole-run +wall-clock deadline that includes time waiting for agents. + +Internal runs, including internal PR runs, import `DotNet-HelixApi-Access`. The +monitor forwards `$(HelixApiAccessToken)` directly, matching the shared pipeline +convention. Internal runs use the imported credential. Public runs, including public +PR runs, do not import that group and use public Helix access. Direct forwarding +does not change how Azure DevOps expands variables or how the SDK handles tokens. +Azure DevOps timeline access and result reporting still use the job's +`System.AccessToken`. The tool is restored from +`.config/dotnet-tools.json` in its own checkout; `eng/Version.Details.xml` tracks that +pin alongside the Helix SDK. + +Monitor test-run names include the matrix-expanded job display name. This keeps +different channels in the same phase and Helix queue independent while preserving a +stable identity when the same leg is retried. + +Submitter build logs still use the existing `Logs_*` pipeline artifacts. Benchmark +results and diagnostics are still uploaded from the Helix work items to PerfLab and +Helix, respectively. The monitor reports test results and links to Helix consoles; it +does not download arbitrary Helix uploads into pipeline artifacts. Asynchronous sends +skip the submitter's `artifacts/helix-results` downloads, which were not published by +this pipeline. Local `--send-to-helix` runs still wait and download performance reports. + +## Blazor workload preparation + +SDK Blazor jobs install `wasm-tools` once on the build agent through +`PreparePayloadWorkItems`, using the SDK in the correlation payload, its bundled +manifests (`--skip-manifest-update`), and the payload's `NuGet.config`. An installation +failure stops payload preparation before the job is sent to Helix. + +Helix work items use `pre.py publish --has-workload --readonly-dotnet` and +`post.py --readonly-dotnet`. These flags prevent the scenario helpers from changing +the shared workload installation; they do not set filesystem permissions. Application +restore, publish, size measurements, and cleanup of work-item-local outputs still run +on Helix. Standalone scenario scripts retain their existing workload-management +defaults. + +Workload availability does not select the benchmark's compilation mode. The Minimum +Template explicitly enables native relinking, while the non-AOT New Template, Pizza, +and Localized scenarios disable it with `WasmNativeWorkload=false`. AOT scenarios +enable `RunAOTCompilation=true`. The Localized AOT command passes this property directly +instead of referencing item metadata as an MSBuild property, which previously dropped +the AOT argument. Its reported sizes may therefore change when it first runs with AOT +actually enabled. + ## SDK Build Throughput Scenario **SDK Build Throughput** is a scenario test that measures the throughput of SDK build process. To be more specific, our test *implicitly calls* diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 37c4f335c2a..65e5abfade0 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -1,25 +1,25 @@ - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 https://github.com/dotnet/dotnet @@ -27,17 +27,21 @@ - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + https://github.com/dotnet/dotnet - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + cca240e47d0e810f091225e13c7c93aa7e1c5189 - + + https://github.com/dotnet/dotnet + cca240e47d0e810f091225e13c7c93aa7e1c5189 + + https://github.com/dotnet/xharness - 1665037768fd7eec44e046d94fd07b8c6407c966 + 64629bf55dc3bf7bddf2ec24670950c02ba451ea - - 2805bd2ff835d56183bc24790978db678711c770 + + 0fb2d2e873627b4fd88b20ffa17c5d3b20b4441c https://github.com/dotnet/maui - - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + + cca240e47d0e810f091225e13c7c93aa7e1c5189 https://github.com/dotnet/dotnet - - bcd0a2de8927de44b65f3ad41bef8820ee2b3d0b + + cca240e47d0e810f091225e13c7c93aa7e1c5189 https://github.com/dotnet/dotnet - + https://github.com/dotnet/android - ee72c6ff89ce0b2609b21fc5b0897741cae8dc5c + db080b91dbe7947a69005a203ea98d9bb207ad04 4c9d1b112c16716c2479e054e9ad4db8b5b8c70c diff --git a/eng/Versions.props b/eng/Versions.props index 056dec0d038..3a80c2016b5 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -7,13 +7,13 @@ - 11.0.0-rc.2.26458.103 - 11.0.0-rc.2.26458.103 - 11.0.0-rc.2.26458.103 - 11.0.0-rc.2.26458.103 + 11.0.0-rtm.26473.104 + 11.0.0-rtm.26473.104 + 11.0.0-rtm.26473.104 + 11.0.0-rtm.26473.104 0.16.0-nightly.20260801.595 11.0.0-preview.5.26261.101 - 11.0.0-prerelease.26458.2 + 11.0.0-prerelease.26473.1 diff --git a/eng/common/core-templates/job/helix-job-monitor.yml b/eng/common/core-templates/job/helix-job-monitor.yml index 53bbf74927e..7417afdf3ea 100644 --- a/eng/common/core-templates/job/helix-job-monitor.yml +++ b/eng/common/core-templates/job/helix-job-monitor.yml @@ -52,11 +52,23 @@ parameters: type: string default: https://helix.dot.net/ -# Helix API access token forwarded to the tool via the HELIX_ACCESSTOKEN environment variable. +# Helix API access token forwarded via HELIX_ACCESSTOKEN. Not forwarded when +# useEntraAuthentication is true. - name: helixAccessToken type: string default: '' +# Use a refreshable Entra credential instead of a PAT or anonymous access. +- name: useEntraAuthentication + type: boolean + default: false + +# Azure service connection ID authorized for Helix. Required when +# useEntraAuthentication is true. +- name: azureSubscription + type: string + default: '' + # Polling interval in seconds (--polling-interval-seconds). - name: pollingIntervalSeconds type: number @@ -141,6 +153,26 @@ jobs: - checkout: self fetchDepth: 1 + - ${{ if and(eq(parameters.useEntraAuthentication, true), eq(parameters.azureSubscription, '')) }}: + - pwsh: throw "azureSubscription must be set when useEntraAuthentication is true." + displayName: Validate Helix Entra authentication + + - ${{ if eq(parameters.useEntraAuthentication, true) }}: + - task: AzureCLI@2 + displayName: Initialize Helix Entra authentication + inputs: + azureSubscription: ${{ parameters.azureSubscription }} + addSpnToEnvironment: true + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + if ([string]::IsNullOrWhiteSpace($env:servicePrincipalId) -or [string]::IsNullOrWhiteSpace($env:tenantId)) { + throw "The Helix Azure service connection did not provide a service principal or tenant ID." + } + + Write-Host "##vso[task.setvariable variable=HelixEntraClientId]$env:servicePrincipalId" + Write-Host "##vso[task.setvariable variable=HelixEntraTenantId]$env:tenantId" + - ${{ if ne(parameters.toolNupkgArtifactName, '') }}: - task: DownloadPipelineArtifact@2 displayName: Download Helix Job Monitor artifact @@ -214,6 +246,7 @@ jobs: toolArgs=( --helix-base-uri '${{ parameters.helixBaseUri }}' + --use-entra-authentication '${{ parameters.useEntraAuthentication }}' --polling-interval-seconds '${{ parameters.pollingIntervalSeconds }}' --fail-on-failed-tests '${{ parameters.failWorkItemsWithFailedTests }}' --allow-no-helix-jobs '${{ parameters.allowNoHelixJobs }}' @@ -275,4 +308,9 @@ jobs: displayName: Monitor Helix Jobs env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) - HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} + ${{ if eq(parameters.useEntraAuthentication, false) }}: + HELIX_ACCESSTOKEN: ${{ parameters.helixAccessToken }} + ${{ if eq(parameters.useEntraAuthentication, true) }}: + AZURESUBSCRIPTION_CLIENT_ID: $(HelixEntraClientId) + AZURESUBSCRIPTION_TENANT_ID: $(HelixEntraTenantId) + AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: ${{ parameters.azureSubscription }} diff --git a/eng/common/core-templates/job/job.yml b/eng/common/core-templates/job/job.yml index cb60f529784..7da6bdf520e 100644 --- a/eng/common/core-templates/job/job.yml +++ b/eng/common/core-templates/job/job.yml @@ -119,7 +119,8 @@ jobs: - name: ${{ pair.key }} value: ${{ pair.value }} - # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds + # DotNet-HelixApi-Access provides 'HelixApiAccessToken' for internal builds. + # Entra-enabled Helix templates do not forward this value to their processes. - ${{ if and(eq(parameters.enableTelemetry, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - group: DotNet-HelixApi-Access diff --git a/eng/common/core-templates/job/onelocbuild.yml b/eng/common/core-templates/job/onelocbuild.yml index ce077368e40..25af2787900 100644 --- a/eng/common/core-templates/job/onelocbuild.yml +++ b/eng/common/core-templates/job/onelocbuild.yml @@ -5,20 +5,10 @@ parameters: # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool pool: '' - CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex - GithubPat: $(BotAccount-dotnet-bot-repo-PAT) - - # Service connection for WIF-based Entra authentication to ceapex feeds (replaces CeapexPat). - # When set, dnceng/internal builds acquire a federated Entra token instead of using a PAT. - # All other projects (e.g. DevDiv, public), where this dnceng-scoped service connection does not - # exist, and any pipeline that sets this to '' fall back to PAT-based auth via the CeapexPat parameter. + # Project-scoped WIF service connection for Ceapex feed authentication. CeapexServiceConnection: 'dnceng-onelocbuild-ceapex' # GitHub App authentication for the OneLoc check-in PR. - # dnceng/internal and DevDiv/DevDiv are enabled by default with their project-scoped service - # connections. Other projects must explicitly opt in after provisioning equivalent infrastructure. - UseGitHubAppAuthentication: true - UseGitHubAppAuthenticationInOtherProjects: false GitHubAppServiceConnection: 'dnceng-oneloc-githubapp' GitHubAppClientId: 'Iv23lijBU8x3gc9lDOc9' GitHubAppKeyVaultName: 'EngKeyVault' @@ -50,7 +40,6 @@ jobs: displayName: OneLocBuild${{ parameters.JobNameSuffix }} variables: - - group: OneLocBuildVariables # Contains the CeapexPat and GithubPat - name: _GenerateLocProjectArguments value: -SourcesDirectory ${{ parameters.SourcesDirectory }} -LanguageSet "${{ parameters.LanguageSet }}" @@ -81,6 +70,12 @@ jobs: steps: - ${{ if eq(parameters.is1ESPipeline, '') }}: - 'Illegal entry point, is1ESPipeline is not defined. Repository yaml should not directly reference templates in core-templates folder.': error + - ${{ if notin(variables['System.TeamProject'], 'internal', 'DevDiv') }}: + - 'OneLocBuild is supported only in dnceng/internal and DevDiv/DevDiv.': error + - ${{ if eq(parameters.CeapexServiceConnection, '') }}: + - 'CeapexServiceConnection must identify a WIF service connection.': error + - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.GitHubAppServiceConnection, '')) }}: + - 'GitHubAppServiceConnection must identify a WIF service connection for GitHub repositories.': error - ${{ if ne(parameters.SkipLocProjectJsonGeneration, 'true') }}: - task: Powershell@2 @@ -90,18 +85,15 @@ jobs: displayName: Generate LocProject.json condition: ${{ parameters.condition }} - # Acquire an Entra token for ceapex feed access via WIF (dnceng/internal only). - # All other projects use PAT-based auth, since the ceapex service connection is scoped to dnceng/internal. - - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: - - template: /eng/common/templates/steps/get-federated-access-token.yml - parameters: - federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} - outputVariableName: 'CeapexEntraToken' - condition: ${{ parameters.condition }} + # Acquire a short-lived Entra token for Ceapex feed access. + - template: /eng/common/templates/steps/get-federated-access-token.yml + parameters: + federatedServiceConnection: ${{ parameters.CeapexServiceConnection }} + outputVariableName: 'CeapexEntraToken' + condition: ${{ parameters.condition }} - # Mint a short-lived GitHub App installation token for the loc check-in PR. Use the connection - # provisioned in each supported project; other projects must explicitly opt in and override it. - - ${{ if and(eq(parameters.RepoType, 'gitHub'), eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: + # Mint a short-lived GitHub App installation token for the loc check-in PR. + - ${{ if eq(parameters.RepoType, 'gitHub') }}: - template: /eng/common/core-templates/steps/get-github-app-token.yml parameters: is1ESPipeline: ${{ parameters.is1ESPipeline }} @@ -131,16 +123,10 @@ jobs: isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }} isShouldReusePrSelected: ${{ parameters.ReusePr }} packageSourceAuth: patAuth - ${{ if and(ne(parameters.CeapexServiceConnection, ''), eq(variables['System.TeamProject'], 'internal')) }}: - patVariable: $(CeapexEntraToken) - ${{ if or(eq(parameters.CeapexServiceConnection, ''), ne(variables['System.TeamProject'], 'internal')) }}: - patVariable: ${{ parameters.CeapexPat }} + patVariable: $(CeapexEntraToken) ${{ if eq(parameters.RepoType, 'gitHub') }}: repoType: ${{ parameters.RepoType }} - ${{ if and(eq(parameters.UseGitHubAppAuthentication, true), or(eq(variables['System.TeamProject'], 'internal'), eq(variables['System.TeamProject'], 'DevDiv'), eq(parameters.UseGitHubAppAuthenticationInOtherProjects, true))) }}: - gitHubPatVariable: "$(GitHubAppInstallationToken)" - ${{ else }}: - gitHubPatVariable: "${{ parameters.GithubPat }}" + gitHubPatVariable: "$(GitHubAppInstallationToken)" ${{ if ne(parameters.MirrorRepo, '') }}: isMirrorRepoSelected: true gitHubOrganization: ${{ parameters.GitHubOrg }} diff --git a/eng/common/core-templates/steps/send-to-helix.yml b/eng/common/core-templates/steps/send-to-helix.yml index ec7a2000399..62cce4d4ea2 100644 --- a/eng/common/core-templates/steps/send-to-helix.yml +++ b/eng/common/core-templates/steps/send-to-helix.yml @@ -4,7 +4,9 @@ parameters: HelixType: 'tests/default/' # required -- Helix telemetry which identifies what type of data this is; should include "test" for clarity and must end in '/' HelixBuild: $(Build.BuildNumber) # required -- the build number Helix will use to identify this -- automatically set to the AzDO build number HelixTargetQueues: '' # required -- semicolon-delimited list of Helix queues to test on; see https://helix.dot.net/ for a list of queues - HelixAccessToken: '' # required -- access token to make Helix API requests; should be provided by the appropriate variable group + HelixAccessToken: '' # optional -- legacy access token; not forwarded when HelixUseEntraAuthentication is true + HelixUseEntraAuthentication: false # optional -- use refreshable Entra authentication instead of a PAT or anonymous access + HelixAzureSubscription: '' # required when HelixUseEntraAuthentication is true -- Azure service connection ID authorized for Helix HelixProjectPath: 'eng/common/helixpublish.proj' # optional -- path to the project file to build relative to BUILD_SOURCESDIRECTORY HelixProjectArguments: '' # optional -- arguments passed to the build command HelixConfiguration: '' # optional -- additional property attached to a job @@ -32,12 +34,35 @@ parameters: continueOnError: false # optional -- determines whether to continue the build if the step errors; defaults to false steps: + - ${{ if and(eq(parameters.HelixUseEntraAuthentication, true), eq(parameters.HelixAzureSubscription, '')) }}: + - pwsh: throw "HelixAzureSubscription must be set when HelixUseEntraAuthentication is true." + displayName: Validate Helix Entra authentication + condition: ${{ parameters.condition }} + + - ${{ if eq(parameters.HelixUseEntraAuthentication, true) }}: + - task: AzureCLI@2 + displayName: Initialize Helix Entra authentication + inputs: + azureSubscription: ${{ parameters.HelixAzureSubscription }} + addSpnToEnvironment: true + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + if ([string]::IsNullOrWhiteSpace($env:servicePrincipalId) -or [string]::IsNullOrWhiteSpace($env:tenantId)) { + throw "The Helix Azure service connection did not provide a service principal or tenant ID." + } + + Write-Host "##vso[task.setvariable variable=HelixEntraClientId]$env:servicePrincipalId" + Write-Host "##vso[task.setvariable variable=HelixEntraTenantId]$env:tenantId" + condition: ${{ parameters.condition }} + - powershell: > $(Build.SourcesDirectory)\eng\common\msbuild.ps1 $(Build.SourcesDirectory)/${{ parameters.HelixProjectPath }} /restore /p:TreatWarningsAsErrors=false /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} ${{ parameters.HelixProjectArguments }} /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog @@ -49,7 +74,12 @@ steps: HelixBuild: ${{ parameters.HelixBuild }} HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} + ${{ if eq(parameters.HelixUseEntraAuthentication, false) }}: + HelixAccessToken: ${{ parameters.HelixAccessToken }} + ${{ if eq(parameters.HelixUseEntraAuthentication, true) }}: + AZURESUBSCRIPTION_CLIENT_ID: $(HelixEntraClientId) + AZURESUBSCRIPTION_TENANT_ID: $(HelixEntraTenantId) + AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: ${{ parameters.HelixAzureSubscription }} HelixPreCommands: ${{ parameters.HelixPreCommands }} HelixPostCommands: ${{ parameters.HelixPostCommands }} WorkItemDirectory: ${{ parameters.WorkItemDirectory }} @@ -76,6 +106,7 @@ steps: /restore /p:TreatWarningsAsErrors=false /p:EnableHelixJobMonitor=${{ parameters.UseHelixMonitor }} + /p:HelixUseEntraAuthentication=${{ parameters.HelixUseEntraAuthentication }} ${{ parameters.HelixProjectArguments }} /t:Test /bl:$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/SendToHelix.binlog @@ -87,7 +118,12 @@ steps: HelixBuild: ${{ parameters.HelixBuild }} HelixConfiguration: ${{ parameters.HelixConfiguration }} HelixTargetQueues: ${{ parameters.HelixTargetQueues }} - HelixAccessToken: ${{ parameters.HelixAccessToken }} + ${{ if eq(parameters.HelixUseEntraAuthentication, false) }}: + HelixAccessToken: ${{ parameters.HelixAccessToken }} + ${{ if eq(parameters.HelixUseEntraAuthentication, true) }}: + AZURESUBSCRIPTION_CLIENT_ID: $(HelixEntraClientId) + AZURESUBSCRIPTION_TENANT_ID: $(HelixEntraTenantId) + AZURESUBSCRIPTION_SERVICE_CONNECTION_ID: ${{ parameters.HelixAzureSubscription }} HelixPreCommands: ${{ parameters.HelixPreCommands }} HelixPostCommands: ${{ parameters.HelixPostCommands }} WorkItemDirectory: ${{ parameters.WorkItemDirectory }} @@ -108,4 +144,3 @@ steps: SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) continueOnError: ${{ parameters.continueOnError }} - diff --git a/eng/common/dotnet.sh b/eng/common/dotnet.sh old mode 100644 new mode 100755 diff --git a/eng/performance/blazor_scenarios.proj b/eng/performance/blazor_scenarios.proj index 4cba378c7fa..03c2b5a06ee 100644 --- a/eng/performance/blazor_scenarios.proj +++ b/eng/performance/blazor_scenarios.proj @@ -7,9 +7,13 @@ %PERFLAB_TARGET_FRAMEWORKS% %24{PERFLAB_TARGET_FRAMEWORKS} <_MSBuildArgs>/p:_TrimmerDumpDependencies=true;/warnaserror:NU1602,NU1604 - --msbuild "$(_MSBuildArgs)" --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog $(LogDirectory)blazor_publish.binlog + --has-workload --readonly-dotnet --msbuild "$(_MSBuildArgs)" --msbuild-static AdditionalMonoLinkerOptions=%27"%24(AdditionalMonoLinkerOptions) --dump-dependencies"%27 --binlog $(LogDirectory)blazor_publish.binlog $(Python) pre.py publish $(PublishArgs) + <_BlazorDotNet>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'dotnet', 'dotnet')) + <_BlazorDotNet Condition="'$(TargetsWindows)' == 'true'">$(_BlazorDotNet).exe + <_BlazorNuGetConfig>$([MSBuild]::NormalizePath('$(CorrelationPayloadDirectory)', 'performance', 'NuGet.config')) + pub\wwwroot pub/wwwroot @@ -27,6 +31,11 @@ + + "$(_BlazorDotNet)" workload install wasm-tools --skip-manifest-update --configfile "$(_BlazorNuGetConfig)" + $(CorrelationPayloadDirectory)performance + + %(Identity) @@ -35,7 +44,7 @@ - $(Python) post.py + $(Python) post.py --readonly-dotnet 4:00 @@ -78,17 +87,19 @@ $(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(ScenariosDir)blazorlocalized - $(PublishCommand) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(PublishCommand) --msbuild "$(_MSBuildArgs);/p:WasmNativeWorkload=false" -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) - <_PublishArgsWithAOT>--msbuild "$(_MSBuildArgs);/p:RunAOTCompilation=true" $(ScenariosDir)blazorlocalized - $(PublishCommand) $(_PublishArgsWithAOT) -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) + $(PublishCommand) --msbuild "$(_MSBuildArgs);/p:RunAOTCompilation=true" -f $(PerfLabTargetFrameworksEnvVar) && $(Python) test.py sod --scenario-name "%(Identity)" --dirs $(PizzaAppPubLocation) $(ScenarioArgs) diff --git a/eng/pipelines/templates/run-performance-job.yml b/eng/pipelines/templates/run-performance-job.yml index 0045cdb5a03..80a7f771cde 100644 --- a/eng/pipelines/templates/run-performance-job.yml +++ b/eng/pipelines/templates/run-performance-job.yml @@ -57,6 +57,7 @@ parameters: iOSStripSymbols: true # optional -- Whether to strip symbols from the iOS build machinePool: '' # optional -- Machine pool name (e.g. Pixel8, GalaxyA16, iPhone17) additionalSetupParameters: '' # optional -- Additional arguments to pass to the script + wasmWorkloadSource: '' # optional -- Exclusive coherent package source for SDK CoreCLR browser-WASM liveLibrariesBuildConfig: '' # optional -- Build configuration when generating Core_Root for libraries crossBuild: false # optional -- Whether the Core_Root is being cross-compiled @@ -241,6 +242,8 @@ jobs: - '--cross-build' - ${{ if ne(parameters.additionalSetupParameters, '') }}: - '${{ parameters.additionalSetupParameters }}' + - ${{ if and(ne(parameters.wasmWorkloadSource, ''), eq(parameters.runtimeType, 'wasm_coreclr'), eq(parameters.r2rRunType, 'r2r')) }}: + - '--wasm-workload-source "${{ parameters.wasmWorkloadSource }}"' - template: /eng/pipelines/templates/send-to-helix-step.yml parameters: osGroup: ${{ parameters.osGroup }} diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index 479d40959a8..3c4fa42a959 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -16,6 +16,7 @@ parameters: isScenario: false downloadSpecificBuild: null # buildId, pipeline, branchName, project crossBuild: false + wasmWorkloadSource: '' runtimeRepoAlias: runtime performanceRepoAlias: self selfIsRuntime: true diff --git a/global.json b/global.json index 63b826eb09a..91f7652ccac 100644 --- a/global.json +++ b/global.json @@ -1,15 +1,15 @@ { "sdk": { - "version": "11.0.100-preview.6.26359.118", + "version": "11.0.100-rc.1.26425.128", "allowPrerelease": true, "rollForward": "major" }, "tools": { - "dotnet": "11.0.100-preview.6.26359.118" + "dotnet": "11.0.100-rc.1.26425.128" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26458.103", - "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26458.103" + "Microsoft.DotNet.Arcade.Sdk": "11.0.0-beta.26473.104", + "Microsoft.DotNet.Helix.Sdk": "11.0.0-beta.26473.104" }, "native-tools": { "python3": "3.7.1" diff --git a/scripts/benchmarks_ci.py b/scripts/benchmarks_ci.py index 2aaea577db4..17131362e9e 100755 --- a/scripts/benchmarks_ci.py +++ b/scripts/benchmarks_ci.py @@ -101,6 +101,14 @@ def __is_valid_dotnet_path(dp: str) -> str: type=__is_valid_dotnet_path, help='Path to a custom dotnet' ) + parser.add_argument( + '--wasm-workload-source', + dest='wasm_workload_source', + required=False, + help=( + 'Exclusive NuGet source containing the coherent browser-WASM workload ' + 'packages for the selected SDK. Requires CoreCLR WASM ReadyToRun.') + ) parser.add_argument('--upload-to-perflab-container', dest="upload_to_perflab_container", @@ -220,6 +228,14 @@ def main(argv: list[str]): raise Exception("Framework version (-f) must be specified.") target_framework_monikers = dotnet.get_target_framework_monikers(args.frameworks) + if args.wasm_workload_source and not ( + args.wasm + and args.wasm_runtime_flavor == 'CoreCLR' + and args.wasm_ready_to_run): + raise ValueError( + '--wasm-workload-source requires --wasm ' + '--wasm-runtime-flavor CoreCLR --wasm-ready-to-run') + # Acquire necessary tools (dotnet) if not args.dotnet_path: init_tools( @@ -233,6 +249,15 @@ def main(argv: list[str]): else: dotnet.setup_dotnet(args.dotnet_path) + if args.wasm_workload_source: + os.environ.pop('PERFLAB_WASM_PACKAGE_VERSION', None) + dotnet.install_wasm_workload( + architecture=args.architecture, + target_framework_monikers=target_framework_monikers, + package_source=args.wasm_workload_source, + sdk_versions=args.dotnet_versions, + verbose=verbose) + # WORKAROUND # The MicroBenchmarks.csproj targets .NET Core 2.1, 3.0, 3.1 and 5.0 # to avoid a build failure when using older frameworks (error NETSDK1045: diff --git a/scripts/build_runtime_payload.py b/scripts/build_runtime_payload.py index 23ef25323f1..926b7efcfbd 100644 --- a/scripts/build_runtime_payload.py +++ b/scripts/build_runtime_payload.py @@ -350,6 +350,7 @@ def build_wasm_coreclr_payload( browser_wasm_coreclr_archive_or_dir, wasm_built_nugets_dir, prefix="staging/built-nugets/" ) local_package_version = _get_wasm_local_package_version(wasm_built_nugets_dir) + _ensure_wasm_tool_framework_version(wasm_dotnet_dir, local_package_version) # Determine version from the runtime pack directory structure runtime_pack_src = os.path.join( @@ -380,6 +381,36 @@ def build_wasm_coreclr_payload( return local_package_version +def _ensure_wasm_tool_framework_version(wasm_dotnet_dir: str, package_version: str) -> None: + """Make the SDK's shared framework available at the local tool package version.""" + shared_framework_dir = Path(wasm_dotnet_dir) / "shared" / "Microsoft.NETCore.App" + requested_framework_dir = shared_framework_dir / package_version + if requested_framework_dir.is_dir(): + return + + version_parts = package_version.split(".", 2) + if len(version_parts) < 2 or not all(part.isdigit() for part in version_parts[:2]): + raise ValueError(f"Invalid local WebAssembly toolchain package version: {package_version}") + + version_prefix = f"{version_parts[0]}.{version_parts[1]}." + compatible_frameworks = sorted( + path for path in shared_framework_dir.glob(f"{version_prefix}*") if path.is_dir() + ) + if len(compatible_frameworks) != 1: + installed_versions = ", ".join(path.name for path in compatible_frameworks) or "none" + raise ValueError( + f"Expected one installed Microsoft.NETCore.App {version_parts[0]}.{version_parts[1]} " + f"framework for local WebAssembly tools {package_version}, found: {installed_versions}" + ) + + shutil.copytree(compatible_frameworks[0], requested_framework_dir) + getLogger().info( + "Installed Microsoft.NETCore.App framework alias %s from %s for local WebAssembly tools", + package_version, + compatible_frameworks[0].name, + ) + + def _get_wasm_local_package_version(built_nugets_dir: str) -> str: package_prefix = "Microsoft.NET.Sdk.WebAssembly.Pack." wasm_sdk_packages = [ diff --git a/scripts/channel_map.py b/scripts/channel_map.py index a043902174a..0d5de09b272 100644 --- a/scripts/channel_map.py +++ b/scripts/channel_map.py @@ -4,7 +4,7 @@ class ChannelMap(): channel_map = { 'main': { 'tfm': 'net11.0', - 'branch': '12.0', + 'branch': '11.0', 'quality': 'daily' }, '11.0': { diff --git a/scripts/dotnet.py b/scripts/dotnet.py index 90be9aaf918..10fe6582bd7 100755 --- a/scripts/dotnet.py +++ b/scripts/dotnet.py @@ -7,6 +7,10 @@ import re import json import datetime +import platform as platform_module +import shutil +import tempfile +import xml.etree.ElementTree as ET from argparse import ArgumentParser, ArgumentTypeError from glob import iglob from logging import getLogger @@ -68,6 +72,14 @@ def __log_script_header(message: str): CSharpProjFile = NamedTuple('CSharpProjFile', file_name=str, working_directory=str) +WasmPackage = NamedTuple('WasmPackage', package_id=str, version=str) +WasmWorkloadCohort = NamedTuple( + 'WasmWorkloadCohort', + sdk_version=str, + product_version=str, + host_rid=str, + workload_id=str, + packages=tuple[WasmPackage, ...]) @tracer.start_as_current_span("get_target_framework_moniker") def get_target_framework_moniker(framework: str) -> str: @@ -106,6 +118,294 @@ def get_target_framework_monikers(frameworks: list[str]) -> list[str]: # ['net6.0', 'nativeaot6.0'] should become ['net6.0'] return list(set(monikers)) + +def get_host_rid( + architecture: str, + system: Optional[str] = None, + libc: Optional[str] = None) -> str: + system = system or platform_module.system() + normalized_system = system.casefold() + + if normalized_system == 'darwin': + os_rid = 'osx' + elif normalized_system == 'windows': + os_rid = 'win' + elif normalized_system == 'linux': + if libc is None: + libc_name = platform_module.libc_ver()[0] + is_musl = ( + libc_name.casefold() == 'musl' + or path.isfile('/etc/alpine-release') + or any(iglob('/lib/ld-musl-*.so.1'))) + else: + is_musl = libc.casefold() == 'musl' + os_rid = 'linux-musl' if is_musl else 'linux' + else: + raise ValueError(f'Unsupported host operating system for Crossgen2: {system}') + + return f'{os_rid}-{architecture}' + + +def get_wasm_workload_cohort( + dotnet_root: str, + sdk_version: str, + target_framework: str, + host_rid: str) -> WasmWorkloadCohort: + sdk_major_match = re.match(r'^(\d+)\.', sdk_version) + framework_major_match = re.match(r'^net(\d+)\.0$', target_framework) + if not sdk_major_match or not framework_major_match: + raise ValueError( + f'Cannot determine a coherent WASM cohort for SDK {sdk_version} and {target_framework}') + if sdk_major_match.group(1) != framework_major_match.group(1): + raise ValueError( + f'WASM workload acquisition requires the SDK and target framework to have the same ' + f'major version, but got SDK {sdk_version} and {target_framework}') + + bundled_versions_path = path.join( + dotnet_root, 'sdk', sdk_version, 'Microsoft.NETCoreSdk.BundledVersions.props') + if not path.isfile(bundled_versions_path): + raise ValueError( + f'Cannot find SDK bundled product versions at {bundled_versions_path}') + + root = ET.parse(bundled_versions_path).getroot() + + def find_item(name: str, include: str) -> ET.Element: + for item in root.iter(name): + if (item.get('Include') == include + and item.get('TargetFramework') == target_framework): + return item + raise ValueError( + f'SDK {sdk_version} does not define {name} {include} for {target_framework}') + + framework_reference = find_item('KnownFrameworkReference', 'Microsoft.NETCore.App') + runtime_rids = framework_reference.get('RuntimePackRuntimeIdentifiers', '').split(';') + if 'browser-wasm' not in runtime_rids: + raise ValueError( + f'SDK {sdk_version} does not provide a browser-wasm CoreCLR runtime pack') + + crossgen = find_item('KnownCrossgen2Pack', 'Microsoft.NETCore.App.Crossgen2') + crossgen_rids = set( + crossgen.get('Crossgen2PortableRuntimeIdentifiers', '').split(';') + + crossgen.get('Crossgen2RuntimeIdentifiers', '').split(';')) + if host_rid not in crossgen_rids: + raise ValueError( + f'SDK {sdk_version} does not provide a Crossgen2 pack for host RID {host_rid}') + + illink = find_item('KnownILLinkPack', 'Microsoft.NET.ILLink.Tasks') + webassembly = find_item( + 'KnownWebAssemblySdkPack', 'Microsoft.NET.Sdk.WebAssembly.Pack') + package_versions = { + 'Microsoft.NETCore.App.Runtime.browser-wasm': + framework_reference.get('DefaultRuntimeFrameworkVersion'), + framework_reference.get('TargetingPackName', 'Microsoft.NETCore.App.Ref'): + framework_reference.get('TargetingPackVersion'), + 'Microsoft.NET.Sdk.WebAssembly.Pack': + webassembly.get('WebAssemblySdkPackVersion'), + f'Microsoft.NETCore.App.Crossgen2.{host_rid}': + crossgen.get('Crossgen2PackVersion'), + 'Microsoft.NET.ILLink.Tasks': + illink.get('ILLinkPackVersion'), + } + + missing_versions = [ + package_id for package_id, version in package_versions.items() if not version + ] + if missing_versions: + raise ValueError( + f'SDK {sdk_version} has no version for required WASM packages: ' + f'{", ".join(missing_versions)}') + + versions = set(package_versions.values()) + if len(versions) != 1: + formatted_versions = ', '.join( + f'{package_id}={version}' for package_id, version in package_versions.items()) + raise ValueError( + f'SDK {sdk_version} does not define one coherent CoreCLR browser-WASM ' + f'package version: {formatted_versions}') + + product_version = next(iter(versions)) + return WasmWorkloadCohort( + sdk_version=sdk_version, + product_version=product_version, + host_rid=host_rid, + workload_id='wasm-tools', + packages=tuple( + WasmPackage(package_id, version) + for package_id, version in package_versions.items())) + + +def get_wasm_workload_commands( + dotnet_executable: str, + cohort: WasmWorkloadCohort, + config_file: str, + restore_project: str, + package_root: str) -> list[list[str]]: + return [ + [ + dotnet_executable, + 'restore', + restore_project, + '--packages', + package_root, + '--configfile', + config_file, + '--no-http-cache', + ], + [ + dotnet_executable, + 'workload', + 'install', + cohort.workload_id, + '--skip-manifest-update', + '--configfile', + config_file, + '--no-cache', + ], + ] + + +def install_wasm_workload( + architecture: str, + target_framework_monikers: list[str], + package_source: str, + sdk_versions: list[str], + verbose: bool) -> WasmWorkloadCohort: + if len(target_framework_monikers) != 1: + raise ValueError( + '--wasm-workload-source requires exactly one target framework') + if len(sdk_versions) > 1: + raise ValueError( + '--wasm-workload-source supports at most one SDK version') + + dotnet_root = environ.get('DOTNET_ROOT') + if not dotnet_root: + raise ValueError('DOTNET_ROOT is not configured') + dotnet_executable = path.join( + dotnet_root, 'dotnet.exe' if platform == 'win32' else 'dotnet') + + if sdk_versions: + sdk_version = sdk_versions[0] + else: + sdk_version = check_output( + [dotnet_executable, '--version'], + cwd=dotnet_root, + text=True).strip() + + host_rid = get_host_rid(architecture) + cohort = get_wasm_workload_cohort( + dotnet_root, + sdk_version, + target_framework_monikers[0], + host_rid) + + parsed_source = urlparse(package_source) + if parsed_source.scheme not in ('http', 'https', 'file'): + package_source = path.abspath(package_source) + if not path.isdir(package_source): + raise ValueError( + f'WASM workload package source does not exist: {package_source}') + + available_packages = { + path.basename(package).casefold() + for package in iglob( + path.join(package_source, '**', '*.nupkg'), + recursive=True) + } + missing_packages = [ + f'{package.package_id}.{package.version}.nupkg' + for package in cohort.packages + if f'{package.package_id}.{package.version}.nupkg'.casefold() + not in available_packages + ] + if missing_packages: + raise ValueError( + f'WASM workload source {package_source} is missing required coherent ' + f'cohort packages: {", ".join(missing_packages)}') + + with tempfile.TemporaryDirectory(prefix='perflab-wasm-workload-') as temp_dir: + config_path = path.join(temp_dir, 'NuGet.Config') + config_root = ET.Element('configuration') + package_sources = ET.SubElement(config_root, 'packageSources') + ET.SubElement(package_sources, 'clear') + ET.SubElement( + package_sources, + 'add', + {'key': 'wasm-cohort', 'value': package_source}) + ET.ElementTree(config_root).write( + config_path, encoding='utf-8', xml_declaration=True) + + global_json_path = path.join(temp_dir, 'global.json') + with open(global_json_path, 'w', encoding='utf-8') as global_json: + json.dump({ + 'sdk': { + 'version': sdk_version, + 'rollForward': 'disable', + 'allowPrerelease': True, + } + }, global_json) + + restore_project_path = path.join(temp_dir, 'WasmCohort.csproj') + project = ET.Element('Project', {'Sdk': 'Microsoft.NET.Sdk'}) + properties = ET.SubElement(project, 'PropertyGroup') + ET.SubElement(properties, 'TargetFramework').text = target_framework_monikers[0] + ET.SubElement(properties, 'RestoreProjectStyle').text = 'PackageReference' + package_downloads = ET.SubElement(project, 'ItemGroup') + for package in cohort.packages: + ET.SubElement( + package_downloads, + 'PackageDownload', + { + 'Include': package.package_id, + 'Version': f'[{package.version}]', + }) + ET.ElementTree(project).write( + restore_project_path, encoding='utf-8', xml_declaration=True) + + package_root = path.join(temp_dir, 'packages') + commands = get_wasm_workload_commands( + dotnet_executable, + cohort, + config_path, + restore_project_path, + package_root) + try: + for command in commands: + RunCommand(command, verbose=verbose).run(temp_dir) + except CalledProcessError as error: + raise ValueError( + f'Failed to install the exact CoreCLR browser-WASM cohort ' + f'{cohort.product_version} from {package_source}') from error + + for package in cohort.packages: + restored_package = path.join( + package_root, + package.package_id.casefold(), + package.version.casefold()) + if not path.isdir(restored_package): + raise ValueError( + f'NuGet did not restore {package.package_id} {package.version} ' + f'from {package_source}') + installed_package = path.join( + dotnet_root, 'packs', package.package_id, package.version) + shutil.copytree(restored_package, installed_package, dirs_exist_ok=True) + + missing_installed_packs = [ + f'{package.package_id}/{package.version}' + for package in cohort.packages + if not path.isdir(path.join( + dotnet_root, 'packs', package.package_id, package.version)) + ] + if missing_installed_packs: + raise ValueError( + f'CoreCLR browser-WASM workload installation is incomplete: ' + f'{", ".join(missing_installed_packs)}') + + getLogger().info( + 'Installed coherent CoreCLR browser-WASM cohort %s for %s', + cohort.product_version, + cohort.host_rid) + return cohort + _VERSION_RE = re.compile(r'^\d+\.\d+\.\d+') def version_type(value: str) -> str: if not _VERSION_RE.search(value): diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index db14b4511a4..f3b8393240f 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -12,6 +12,7 @@ import tempfile import time from traceback import format_exc +import urllib.parse import urllib.request import xml.etree.ElementTree as ET from typing import Any, Optional @@ -30,6 +31,20 @@ def apt_command(arguments: str, *, executable: str = "apt-get") -> str: return f"sudo {executable} {APT_LOCK_TIMEOUT_OPTION} {arguments}" +def normalize_wasm_workload_source(source: Optional[str]) -> Optional[str]: + normalized_source = source.strip() if source else "" + return normalized_source or None + + +def set_shell_environment_variable( + os_group: str, + name: str, + value: str) -> str: + if os_group == "windows": + return f'set "{name}={value}"' + return f"export {name}={value}" + + def output_counters_for_crank(reports: list[Any]): print("#StartJobStatistics") @@ -123,6 +138,7 @@ class RunPerformanceJobArgs: pdn_path: Optional[str] = None os_version: Optional[str] = None dotnet_version_link: Optional[str] = None + wasm_workload_source: Optional[str] = None target_csproj: Optional[str] = None build_config: str = DEFAULT_BUILD_CONFIG live_libraries_build_config: Optional[str] = None @@ -209,7 +225,12 @@ def get_pre_commands( codegen_type: str, build_config: str, v8_version: str, - wasm_local_package_version: Optional[str] = None): + wasm_local_package_version: Optional[str] = None, + wasm_workload_source: Optional[str] = None): + if os_group == "windows" and runtime_type in ("wasm", "wasm_coreclr"): + raise ValueError( + "WASM performance job prerequisite setup is not supported on Windows") + helix_pre_commands: list[str] = [] # Remember the previous PYTHONPATH that was set so it can be restored in the post commands @@ -286,17 +307,24 @@ def get_pre_commands( # Set up everything needed for WASM runs (both Mono and CoreCLR) if runtime_type in ("wasm", "wasm_coreclr"): + use_workload_source = ( + runtime_type == "wasm_coreclr" and bool(wasm_workload_source)) if runtime_type == "wasm_coreclr": - if not wasm_local_package_version: - raise ValueError("CoreCLR WASM requires a local WebAssembly toolchain package version") - install_prerequisites += [ - f"export PERFLAB_WASM_PACKAGE_VERSION={wasm_local_package_version}" - ] + if wasm_local_package_version: + install_prerequisites += [ + set_shell_environment_variable( + os_group, + "PERFLAB_WASM_PACKAGE_VERSION", + wasm_local_package_version) + ] + elif not use_workload_source: + raise ValueError( + "CoreCLR WASM requires either a private runtime payload or " + "a coherent WASM workload source") if os_distro == "azurelinux": # Azure Linux uses tdnf package manager install_prerequisites += [ - "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets", "sudo tdnf -y update", "sudo tdnf -y remove nodejs", "sudo tdnf -y install ca-certificates curl gnupg nodejs npm", @@ -308,7 +336,6 @@ def get_pre_commands( ] else: install_prerequisites += [ - "export RestoreAdditionalProjectSources=$HELIX_CORRELATION_PAYLOAD/built-nugets", apt_command("-y remove nodejs"), apt_command("update"), apt_command("install -y ca-certificates curl gnupg"), @@ -327,6 +354,18 @@ def get_pre_commands( "${V8_ENGINE_PATH} -e 'console.log(`V8 version: ${this.version()}`)'" ] + if not use_workload_source: + correlation_payload = ( + "%HELIX_CORRELATION_PAYLOAD%\\built-nugets" + if os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/built-nugets") + install_prerequisites.insert( + 0, + set_shell_environment_variable( + os_group, + "RestoreAdditionalProjectSources", + correlation_payload)) + # Add the install_prerequisites to the pre_commands if os_group == "windows": # Chain Windows commands with error checking using && to ensure each command succeeds @@ -567,6 +606,10 @@ def get_bdn_arguments( bdn_arguments += ["\\\"--wasmArgs=--experimental-wasm-exnref\\\""] if runtime_type == "coreclr_r2r_interpreter": + bdn_arguments += [ + "--msbuild-arguments", + "/p:PublishReadyToRunCrossgen2ExtraArgs=--target-allows-runtime-code-generation:false", + ] if os_group == "windows": bdn_arguments += [ "--runtimes", "r2r11_0", @@ -710,7 +753,8 @@ def get_work_item_command( bdn_artifacts_dir: str, wasm_coreclr: bool = False, wasm_ready_to_run: bool = False, - only_sanity_check: bool = False): + only_sanity_check: bool = False, + wasm_workload_source: Optional[str] = None): if os_group == "windows": work_item_command = [ "python", @@ -737,11 +781,25 @@ def get_work_item_command( work_item_command += ["--dotnet-versions", "$DOTNET_VERSION"] if wasm: - work_item_command += ["--run-isolated", "--wasm", "--dotnet-path", "$HELIX_CORRELATION_PAYLOAD/dotnet/"] + dotnet_path = ( + "%HELIX_CORRELATION_PAYLOAD%\\dotnet\\" + if os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/dotnet/") + work_item_command += [ + "--run-isolated", + "--wasm", + "--dotnet-path", + dotnet_path, + ] if wasm_coreclr: work_item_command += ["--wasm-runtime-flavor", "CoreCLR"] if wasm_ready_to_run: work_item_command += ["--wasm-ready-to-run"] + if wasm_ready_to_run and wasm_workload_source: + work_item_command += [ + "--wasm-workload-source", + wasm_workload_source, + ] work_item_command += ["--bdn-artifacts", bdn_artifacts_dir] @@ -963,7 +1021,30 @@ def run_performance_job(args: RunPerformanceJobArgs): v8_version = "" wasm_local_package_version = None - if wasm_coreclr: + wasm_workload_source = normalize_wasm_workload_source( + args.wasm_workload_source) + wasm_sdk_cohort = ( + wasm_coreclr + and args.r2r_run_type == "r2r" + and wasm_workload_source is not None) + helix_wasm_workload_source = ( + wasm_workload_source if wasm_sdk_cohort else None) + if wasm_sdk_cohort and wasm_workload_source: + parsed_workload_source = urllib.parse.urlparse(wasm_workload_source) + local_workload_source = ( + urllib.request.url2pathname(parsed_workload_source.path) + if parsed_workload_source.scheme == "file" + else os.path.abspath(wasm_workload_source)) + if os.path.isdir(local_workload_source): + payload_workload_source = os.path.join( + payload_dir, "wasm-workload-source") + shutil.copytree(local_workload_source, payload_workload_source) + helix_wasm_workload_source = ( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source" + if args.os_group == "windows" + else "$HELIX_CORRELATION_PAYLOAD/wasm-workload-source") + + if wasm_coreclr and not wasm_sdk_cohort: if args.libraries_download_dir is None: raise Exception("Libraries not downloaded for wasm_coreclr runs") @@ -974,7 +1055,7 @@ def run_performance_job(args: RunPerformanceJobArgs): payload_dir, ) - elif wasm: + elif wasm and not wasm_coreclr: if args.libraries_download_dir is None: raise Exception("Libraries not downloaded for wasm runs") @@ -1008,7 +1089,8 @@ def run_performance_job(args: RunPerformanceJobArgs): if args.javascript_engine_path is None: args.javascript_engine_path = f"/home/helixbot/.jsvu/bin/{args.javascript_engine}" - ci_setup_arguments.dotnet_path = f"{payload_dir}/dotnet" + if not wasm_sdk_cohort: + ci_setup_arguments.dotnet_path = f"{payload_dir}/dotnet" if args.dotnet_version_link is not None: if args.dotnet_version_link.startswith("https"): # Version link is a proper url @@ -1164,7 +1246,8 @@ def run_performance_job(args: RunPerformanceJobArgs): args.codegen_type, args.build_config, v8_version, - wasm_local_package_version) + wasm_local_package_version, + helix_wasm_workload_source) helix_post_commands = get_post_commands(args.os_group, args.internal, args.runtime_type) # Point ML.NET at the SSWE model that was pre-downloaded into the correlation payload above, so it @@ -1200,7 +1283,7 @@ def run_performance_job(args: RunPerformanceJobArgs): ci_setup_arguments.target_windows = args.os_group == "windows" ci_setup_arguments.output_file = os.path.join(root_payload_dir, "machine-setup") - if args.is_scenario: + if args.is_scenario or wasm_sdk_cohort: ci_setup_arguments.install_dir = os.path.join(payload_dir, "dotnet") elif wasm_coreclr: # For wasm_coreclr, we already have the SDK in the payload - skip downloading @@ -1413,7 +1496,8 @@ def get_work_item_command_for_artifact_dir(artifact_dir: str): artifact_dir, wasm_coreclr, wasm_coreclr and args.r2r_run_type == "r2r", - args.only_sanity_check) + args.only_sanity_check, + helix_wasm_workload_source) work_item_command = get_work_item_command_for_artifact_dir(bdn_artifacts_directory) baseline_work_item_command = get_work_item_command_for_artifact_dir(bdn_baseline_artifacts_dir) @@ -1570,6 +1654,7 @@ def main(argv: list[str]): "--perf-hash": "perf_hash", "--os-version": "os_version", "--dotnet-version-link": "dotnet_version_link", + "--wasm-workload-source": "wasm_workload_source", "--target-csproj": "target_csproj", "--pdn-path": "pdn_path", "--runtime-repo-dir": "runtime_repo_dir", diff --git a/scripts/tests/test_blazor_workload.py b/scripts/tests/test_blazor_workload.py new file mode 100644 index 00000000000..5fab9879d76 --- /dev/null +++ b/scripts/tests/test_blazor_workload.py @@ -0,0 +1,181 @@ +import importlib +import json +import os +from pathlib import Path +import shlex +import shutil +import subprocess +import sys +from unittest.mock import Mock +import xml.etree.ElementTree as ET + +import pytest + + +repo_root = Path(__file__).resolve().parents[2] + + +@pytest.fixture +def scenario_commands(monkeypatch): + monkeypatch.syspath_prepend(str(repo_root / "scripts")) + monkeypatch.syspath_prepend(str(repo_root / "src" / "scenarios")) + return ( + importlib.import_module("shared.precommands").PreCommands, + importlib.import_module("shared.postcommands").PostCommands, + ) + + +@pytest.fixture(params=["true", "false"], ids=["windows", "unix"]) +def blazor_items(request, tmp_path): + dotnet = shutil.which("dotnet") + if dotnet is None: + pytest.skip("MSBuild evaluation requires a .NET SDK") + + project = ET.parse(repo_root / "eng" / "performance" / "blazor_scenarios.proj") + # Evaluate the real work items without downloading the Helix SDK. + project.getroot().attrib.pop("Sdk") + project.find("Import").set( + "Project", str(repo_root / "eng" / "performance" / "PreparePayloadWorkItems.targets") + ) + project_path = tmp_path / "blazor.proj" + project.write(project_path, encoding="utf-8") + payload = tmp_path / "correlation payload" + result = subprocess.run( + [ + dotnet, "msbuild", str(project_path), "-nologo", + "-getItem:PreparePayloadWorkItem,HelixWorkItem", + f"-p:CorrelationPayloadDirectory={payload}{os.sep}", + f"-p:TargetsWindows={request.param}", + "-p:Python=python", + ], + cwd=tmp_path, + check=True, + capture_output=True, + text=True, + ) + return json.loads(result.stdout)["Items"], payload, request.param == "true" + + +def test_workload_is_prepared_once_in_payload(blazor_items): + items, payload, windows = blazor_items + preparation, = items["PreparePayloadWorkItem"] + executable = payload / "dotnet" / ("dotnet.exe" if windows else "dotnet") + config = payload / "performance" / "NuGet.config" + + assert preparation["Command"] == ( + f'"{executable}" workload install wasm-tools --skip-manifest-update ' + f'--configfile "{config}"' + ) + assert Path(preparation["WorkingDirectory"]) == payload / "performance" + + +@pytest.mark.parametrize("exit_code", [0, 23]) +def test_preparation_propagates_workload_exit_code(blazor_items, tmp_path, exit_code): + _, payload, windows = blazor_items + working_directory = payload / "performance" + working_directory.mkdir(parents=True) + # Substitute only the executable; run the real preparation target and arguments. + (working_directory / "workload").write_text( + "import sys\n" + "assert sys.argv[1:4] == ['install', 'wasm-tools', '--skip-manifest-update']\n" + "print('Workload preparation executed')\n" + f"sys.exit({exit_code})\n", + encoding="utf-8", + ) + result = subprocess.run( + [ + shutil.which("dotnet"), "msbuild", str(tmp_path / "blazor.proj"), "-nologo", + "-t:PreparePayloadWorkItems", + f"-p:CorrelationPayloadDirectory={payload}{os.sep}", + f"-p:TargetsWindows={str(windows).lower()}", + f"-p:_BlazorDotNet={sys.executable}", + ], + cwd=tmp_path, + capture_output=True, + text=True, + ) + + assert "Workload preparation executed" in result.stdout + if exit_code: + assert result.returncode != 0 + assert "MSB3073" in result.stdout + else: + assert result.returncode == 0, result.stdout + result.stderr + + +def test_all_work_items_leave_shared_workload_untouched( + blazor_items, scenario_commands, monkeypatch +): + items, _, _ = blazor_items + PreCommands, PostCommands = scenario_commands + run = Mock(side_effect=AssertionError("A work item must not modify the workload")) + monkeypatch.setattr(subprocess, "run", run) + + assert len(items["HelixWorkItem"]) == 8 + for item in items["HelixWorkItem"]: + command = item["Command"].split(" && ", 1)[0] + monkeypatch.setattr(sys, "argv", shlex.split(command)[1:]) + pre = PreCommands() + assert pre.has_workload + assert pre.readonly_dotnet + pre.install_workload("wasm-tools") + pre.uninstall_workload("wasm-tools") + + monkeypatch.setattr(sys, "argv", shlex.split(item["PostCommands"])[1:]) + post = PostCommands() + assert post.readonly_dotnet + post.uninstall_workload("wasm-tools") + + run.assert_not_called() + + +@pytest.mark.parametrize( + ("name", "property_argument"), + [ + ("SOD - Localized App - Publish", "/p:WasmNativeWorkload=false"), + ("SOD - Localized App - Publish - AOT", "/p:RunAOTCompilation=true"), + ], +) +def test_localized_publish_mode_is_forwarded( + blazor_items, scenario_commands, monkeypatch, name, property_argument +): + items, _, _ = blazor_items + PreCommands, _ = scenario_commands + item = next(item for item in items["HelixWorkItem"] if item["Identity"] == name) + command = item["Command"].split(" && ", 1)[0] + monkeypatch.setattr(sys, "argv", shlex.split(command)[1:]) + pre = PreCommands() + pre.project = Mock() + pre.execute() + + assert pre.project.publish.call_args.args[6] == [ + "/p:_TrimmerDumpDependencies=true", + "/warnaserror:NU1602,NU1604", + property_argument, + ] + + +def test_standalone_workload_install_is_unchanged(scenario_commands, monkeypatch): + PreCommands, _ = scenario_commands + monkeypatch.setattr(sys, "argv", ["pre.py", "publish"]) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + PreCommands().install_workload("wasm-tools") + + run.assert_called_once_with( + ["dotnet", "workload", "install", "wasm-tools", "--skip-manifest-update"], + check=True, + ) + + +def test_readonly_sdk_requires_preinstalled_workload(scenario_commands, monkeypatch): + PreCommands, _ = scenario_commands + monkeypatch.setattr(sys, "argv", ["pre.py", "publish", "--readonly-dotnet"]) + run = Mock() + monkeypatch.setattr(subprocess, "run", run) + + with pytest.raises(Exception, match="has_workload=false"): + PreCommands().install_workload("wasm-tools") + + run.assert_not_called() diff --git a/scripts/tests/test_wasm_coreclr_r2r.py b/scripts/tests/test_wasm_coreclr_r2r.py index 6f88e06adac..e59f34a4327 100644 --- a/scripts/tests/test_wasm_coreclr_r2r.py +++ b/scripts/tests/test_wasm_coreclr_r2r.py @@ -3,6 +3,7 @@ import xml.etree.ElementTree as ET from argparse import Namespace from pathlib import Path +from typing import Optional import pytest @@ -10,8 +11,15 @@ sys.path.insert(0, str(scripts_dir)) import micro_benchmarks +import dotnet from build_runtime_payload import build_wasm_coreclr_payload -from run_performance_job import get_pre_commands, get_run_configurations, get_work_item_command +from run_performance_job import ( + get_pre_commands, + get_run_configurations, + get_work_item_command, + normalize_wasm_workload_source, + set_shell_environment_variable, +) def test_ready_to_run_requires_coreclr_wasm(): @@ -52,6 +60,48 @@ def test_ready_to_run_argument_is_forwarded_to_helix_work_item(): assert "--wasm-ready-to-run" in command +def test_workload_source_is_forwarded_to_helix_work_item(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=True, + wasm_ready_to_run=True, + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + source_index = command.index("--wasm-workload-source") + assert command[source_index + 1] == "https://example.test/cohort/v3/index.json" + + +def test_windows_wasm_work_item_uses_windows_payload_paths(): + command = get_work_item_command( + os_group="windows", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="%HELIX_WORKITEM_UPLOAD_ROOT%\\artifacts", + wasm_coreclr=True, + wasm_ready_to_run=True, + wasm_workload_source=( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source"), + ) + + dotnet_path_index = command.index("--dotnet-path") + workload_source_index = command.index("--wasm-workload-source") + assert command[dotnet_path_index + 1] == ( + "%HELIX_CORRELATION_PAYLOAD%\\dotnet\\") + assert command[workload_source_index + 1] == ( + "%HELIX_CORRELATION_PAYLOAD%\\wasm-workload-source") + assert all("$HELIX_CORRELATION_PAYLOAD" not in argument for argument in command) + + def test_ready_to_run_has_distinct_result_configuration(): configurations = get_run_configurations( run_kind="micro", @@ -89,18 +139,68 @@ def test_ready_to_run_validates_resolved_runtime_pack_items(): def test_coreclr_payload_detects_local_toolchain_package_version(tmp_path): artifact = tmp_path / "artifact" / "staging" ref_pack = artifact / "dotnet-none" / "packs" / "Microsoft.NETCore.App.Ref" / "11.0.0-rc.1.26431.109" + shared_framework = artifact / "dotnet-none" / "shared" / "Microsoft.NETCore.App" / "11.0.0-rc.1.26431.109" runtime_pack = artifact / "microsoft.netcore.app.runtime.browser-wasm" / "Release" built_nugets = artifact / "built-nugets" ref_pack.mkdir(parents=True) + shared_framework.mkdir(parents=True) runtime_pack.mkdir(parents=True) built_nugets.mkdir(parents=True) + (shared_framework / "System.Private.CoreLib.dll").touch() (built_nugets / "Microsoft.NET.Sdk.WebAssembly.Pack.11.0.0-ci.nupkg").touch() (built_nugets / "Microsoft.NETCore.App.Crossgen2.linux-x64.11.0.0-ci.nupkg").touch() (built_nugets / "Microsoft.NET.ILLink.Tasks.11.0.0-ci.nupkg").touch() - version = build_wasm_coreclr_payload(str(artifact.parent), str(tmp_path / "payload")) + payload = tmp_path / "payload" + version = build_wasm_coreclr_payload(str(artifact.parent), str(payload)) assert version == "11.0.0-ci" + assert ( + payload + / "dotnet" + / "shared" + / "Microsoft.NETCore.App" + / "11.0.0-ci" + / "System.Private.CoreLib.dll" + ).is_file() + + +def test_coreclr_payload_aliases_sdk_framework_during_major_version_rollover(tmp_path): + artifact = tmp_path / "artifact" / "staging" + dotnet = artifact / "dotnet-none" + installed_version = "12.0.0-alpha.1.26458.117" + shared_framework = dotnet / "shared" / "Microsoft.NETCore.App" / installed_version + built_nugets = artifact / "built-nugets" + shared_framework.mkdir(parents=True) + built_nugets.mkdir(parents=True) + (shared_framework / "System.Private.CoreLib.dll").write_text("runtime") + (built_nugets / "Microsoft.NET.Sdk.WebAssembly.Pack.12.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NETCore.App.Crossgen2.linux-x64.12.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NET.ILLink.Tasks.12.0.0-ci.nupkg").touch() + + payload = tmp_path / "payload" + version = build_wasm_coreclr_payload(str(artifact.parent), str(payload)) + + aliased_framework = ( + payload / "dotnet" / "shared" / "Microsoft.NETCore.App" / "12.0.0-ci" + ) + assert version == "12.0.0-ci" + assert (aliased_framework / "System.Private.CoreLib.dll").read_text() == "runtime" + + +def test_coreclr_payload_rejects_missing_compatible_sdk_framework(tmp_path): + artifact = tmp_path / "artifact" / "staging" + (artifact / "dotnet-none" / "shared" / "Microsoft.NETCore.App" / "11.0.0").mkdir( + parents=True + ) + built_nugets = artifact / "built-nugets" + built_nugets.mkdir(parents=True) + (built_nugets / "Microsoft.NET.Sdk.WebAssembly.Pack.12.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NETCore.App.Crossgen2.linux-x64.12.0.0-ci.nupkg").touch() + (built_nugets / "Microsoft.NET.ILLink.Tasks.12.0.0-ci.nupkg").touch() + + with pytest.raises(ValueError, match="Expected one installed Microsoft.NETCore.App 12.0"): + build_wasm_coreclr_payload(str(artifact.parent), str(tmp_path / "payload")) def test_coreclr_payload_rejects_mismatched_toolchain_package_versions(tmp_path): @@ -142,3 +242,264 @@ def test_coreclr_pre_commands_export_local_toolchain_package_version(): ) assert any("PERFLAB_WASM_PACKAGE_VERSION=11.0.0-ci" in command for command in commands) + assert any("RestoreAdditionalProjectSources" in command for command in commands) + + +@pytest.mark.parametrize("runtime_type", ["wasm", "wasm_coreclr"]) +def test_windows_wasm_pre_commands_fail_clearly(runtime_type): + with pytest.raises(ValueError, match="not supported on Windows"): + get_pre_commands( + os_group="windows", + os_distro=None, + internal=False, + runtime_type=runtime_type, + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_local_package_version=( + "11.0.0-ci" if runtime_type == "wasm_coreclr" else None), + ) + + +@pytest.mark.parametrize( + ("os_group", "expected"), + [ + ("windows", 'set "NAME=value"'), + ("linux", "export NAME=value"), + ], +) +def test_shell_environment_variable_command(os_group, expected): + assert set_shell_environment_variable(os_group, "NAME", "value") == expected + + +def test_coreclr_sdk_pre_commands_do_not_enable_private_package_overrides(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm_coreclr", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert not any("PERFLAB_WASM_PACKAGE_VERSION" in command for command in commands) + assert not any("RestoreAdditionalProjectSources" in command for command in commands) + + +def test_mono_wasm_pre_commands_preserve_private_package_source(): + commands = get_pre_commands( + os_group="linux", + os_distro="ubuntu", + internal=False, + runtime_type="wasm", + codegen_type="wasm", + build_config="Release", + v8_version="15.1.206", + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert any("RestoreAdditionalProjectSources" in command for command in commands) + + +def test_non_r2r_coreclr_command_ignores_shared_workload_source(): + command = get_work_item_command( + os_group="linux", + target_csproj="src/benchmarks/micro/MicroBenchmarks.csproj", + architecture="x64", + perf_lab_framework="net11.0", + internal=True, + wasm=True, + bdn_artifacts_dir="/tmp/artifacts", + wasm_coreclr=True, + wasm_ready_to_run=False, + wasm_workload_source="https://example.test/cohort/v3/index.json", + ) + + assert "--wasm-workload-source" not in command + + +def test_pipeline_scopes_workload_source_to_coreclr_r2r(): + template = ( + scripts_dir.parent + / "eng" + / "pipelines" + / "templates" + / "run-performance-job.yml" + ).read_text(encoding="utf-8") + + condition = ( + "and(ne(parameters.wasmWorkloadSource, ''), " + "eq(parameters.runtimeType, 'wasm_coreclr'), " + "eq(parameters.r2rRunType, 'r2r'))" + ) + assert condition in template + + +@pytest.mark.parametrize("source", [None, "", " "]) +def test_empty_workload_source_is_not_enabled(source): + assert normalize_wasm_workload_source(source) is None + + +def test_workload_source_is_trimmed(): + assert normalize_wasm_workload_source( + " https://example.test/cohort/v3/index.json " + ) == "https://example.test/cohort/v3/index.json" + + +@pytest.mark.parametrize( + ("system", "architecture", "libc", "expected"), + [ + ("Darwin", "arm64", "", "osx-arm64"), + ("Darwin", "x64", "", "osx-x64"), + ("Linux", "x64", "glibc", "linux-x64"), + ("Linux", "x64", "musl", "linux-musl-x64"), + ("Windows", "x64", "", "win-x64"), + ], +) +def test_crossgen2_host_rid(system, architecture, libc, expected): + assert dotnet.get_host_rid(architecture, system=system, libc=libc) == expected + + +def _write_bundled_versions( + dotnet_root: Path, + sdk_version: str = "11.0.100-preview.1.12345.1", + product_version: str = "11.0.0-preview.1.12345.1", + illink_version: Optional[str] = None) -> None: + sdk_dir = dotnet_root / "sdk" / sdk_version + sdk_dir.mkdir(parents=True) + (sdk_dir / "Microsoft.NETCoreSdk.BundledVersions.props").write_text( + f""" + + + + + + + +""", + encoding="utf-8", + ) + + +def test_cohort_uses_exact_sdk_product_version_and_host_crossgen2(tmp_path): + sdk_version = "11.0.100-preview.1.12345.1" + product_version = "11.0.0-preview.1.12345.1" + _write_bundled_versions(tmp_path, sdk_version, product_version) + + cohort = dotnet.get_wasm_workload_cohort( + str(tmp_path), sdk_version, "net11.0", "osx-arm64") + + assert cohort.product_version == product_version + assert cohort.workload_id == "wasm-tools" + assert dotnet.WasmPackage( + "Microsoft.NETCore.App.Crossgen2.osx-arm64", + product_version, + ) in cohort.packages + assert {package.version for package in cohort.packages} == {product_version} + + +def test_cohort_rejects_mismatched_sdk_package_versions(tmp_path): + _write_bundled_versions( + tmp_path, + illink_version="11.0.0-preview.1.12345.2", + ) + + with pytest.raises(ValueError, match="does not define one coherent"): + dotnet.get_wasm_workload_cohort( + str(tmp_path), + "11.0.100-preview.1.12345.1", + "net11.0", + "linux-x64", + ) + + +def test_local_cohort_source_requires_all_exact_packages(tmp_path, monkeypatch): + sdk_version = "11.0.100-preview.1.12345.1" + product_version = "11.0.0-preview.1.12345.1" + dotnet_root = tmp_path / "dotnet" + package_source = tmp_path / "packages" + package_source.mkdir() + _write_bundled_versions(dotnet_root, sdk_version, product_version) + monkeypatch.setenv("DOTNET_ROOT", str(dotnet_root)) + monkeypatch.setattr(dotnet, "get_host_rid", lambda architecture: "linux-x64") + + for package_id in ( + "Microsoft.NETCore.App.Runtime.browser-wasm", + "Microsoft.NETCore.App.Ref", + "Microsoft.NET.Sdk.WebAssembly.Pack", + "Microsoft.NETCore.App.Crossgen2.linux-x64", + ): + (package_source / f"{package_id}.{product_version}.nupkg").touch() + + with pytest.raises(ValueError, match="Microsoft.NET.ILLink.Tasks"): + dotnet.install_wasm_workload( + architecture="x64", + target_framework_monikers=["net11.0"], + package_source=str(package_source), + sdk_versions=[sdk_version], + verbose=False, + ) + + +def test_cohort_source_rejects_multiple_sdk_versions(): + with pytest.raises(ValueError, match="at most one SDK version"): + dotnet.install_wasm_workload( + architecture="x64", + target_framework_monikers=["net11.0"], + package_source="https://example.test/cohort/v3/index.json", + sdk_versions=["11.0.100-preview.1", "11.0.100-preview.2"], + verbose=False, + ) + + +def test_generated_workload_commands_pin_workload_and_coreclr_r2r_cohort(): + cohort = dotnet.WasmWorkloadCohort( + sdk_version="11.0.100-preview.1.12345.1", + product_version="11.0.0-preview.1.12345.1", + host_rid="linux-x64", + workload_id="wasm-tools", + packages=(), + ) + + commands = dotnet.get_wasm_workload_commands( + "/dotnet/dotnet", + cohort, + "/tmp/NuGet.Config", + "/tmp/WasmCohort.csproj", + "/tmp/packages", + ) + + assert commands[0] == [ + "/dotnet/dotnet", + "restore", + "/tmp/WasmCohort.csproj", + "--packages", + "/tmp/packages", + "--configfile", + "/tmp/NuGet.Config", + "--no-http-cache", + ] + assert commands[1] == [ + "/dotnet/dotnet", + "workload", + "install", + "wasm-tools", + "--skip-manifest-update", + "--configfile", + "/tmp/NuGet.Config", + "--no-cache", + ] diff --git a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets index 93ca49d7232..b6e4216cf79 100644 --- a/src/benchmarks/micro/MicroBenchmarks.Wasm.targets +++ b/src/benchmarks/micro/MicroBenchmarks.Wasm.targets @@ -48,10 +48,6 @@ - - - <_AOT_InternalForceInterpretAssemblies Include="Microsoft.CodeAnalysis.CSharp.dll" /> <_AOT_InternalForceInterpretAssemblies Include="Microsoft.CodeAnalysis.dll" /> diff --git a/src/benchmarks/micro/Program.cs b/src/benchmarks/micro/Program.cs index 8d2a590cae9..28e9eca5312 100644 --- a/src/benchmarks/micro/Program.cs +++ b/src/benchmarks/micro/Program.cs @@ -22,6 +22,7 @@ static async Task Main(string[] args) int? partitionIndex; List exclusionFilterValue; List categoryExclusionFilterValue; + List msBuildArguments; bool getDiffableDisasm; // Parse and remove any additional parameters that we need that aren't part of BDN @@ -31,6 +32,7 @@ static async Task Main(string[] args) argsList = CommandLineOptions.ParseAndRemoveIntParameter(argsList, "--partition-index", out partitionIndex); argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--exclusion-filter", out exclusionFilterValue); argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--category-exclusion-filter", out categoryExclusionFilterValue); + argsList = CommandLineOptions.ParseAndRemoveStringsParameter(argsList, "--msbuild-arguments", out msBuildArguments); CommandLineOptions.ParseAndRemoveBooleanParameter(argsList, "--disasm-diff", out getDiffableDisasm); CommandLineOptions.ValidatePartitionParameters(partitionCount, partitionIndex); @@ -56,7 +58,8 @@ static async Task Main(string[] args) partitionIndex: partitionIndex, exclusionFilterValue: exclusionFilterValue, categoryExclusionFilterValue: categoryExclusionFilterValue, - getDiffableDisasm: getDiffableDisasm) + getDiffableDisasm: getDiffableDisasm, + msBuildArguments: msBuildArguments) .AddValidator(new NoWasmValidator(Categories.NoWASM))) .ConfigureAwait(false); diff --git a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs index c8a469ba68e..2630289ae2c 100644 --- a/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs +++ b/src/harness/BenchmarkDotNet.Extensions/RecommendedConfig.cs @@ -28,7 +28,8 @@ public static IConfig Create( List? exclusionFilterValue = null, List? categoryExclusionFilterValue = null, Job? job = null, - bool getDiffableDisasm = false) + bool getDiffableDisasm = false, + IReadOnlyList? msBuildArguments = null) { if (job is null) { @@ -43,6 +44,11 @@ public static IConfig Create( #pragma warning restore CS0618 } + if (msBuildArguments is not null && msBuildArguments.Count > 0) + { + job = job.WithMsBuildArguments(msBuildArguments.ToArray()); + } + var config = ManualConfig.CreateEmpty() .WithBuildTimeout(TimeSpan.FromMinutes(15)) // for slow machines .AddLogger(ConsoleLogger.Default) // log output to console