Skip to content

Commit 733957e

Browse files
EvangelinkCopilot
Evangelink
and
Copilot
committed
Add MSTestProject and MTPProject items to Helix SDK
Introduces two new MSBuild item types in Microsoft.DotNet.Helix.Sdk so that test projects targeting Microsoft.Testing.Platform (MTP) get the same one-line first-class Helix experience that XUnitV3Project provides today: * MTPProject - generic primitive, any MTP-based framework (MSTest 4.x, NUnit with MTP, TUnit, custom MTP). * MSTestProject - thin discoverability shim that folds into MTPProject before any other target runs. Implementation mirrors XUnitV3Project / xunitv3-runner: a single C# task (CreateMTPWorkItems) plus props/targets under tools/mtp-runner/. MTP test apps are self-hosting executables, so each work item runs 'dotnet exec --roll-forward Major ... <assembly>.dll --results-directory . --report-trx --report-trx-filename testResults.trx'. The TRX file is picked up by the standard arcade Python reporter (TRXFormat parser) without any transcoding. The work item command requires the test project to reference Microsoft.Testing.Extensions.TrxReport. Projects using MSTest.Sdk get this transitively; other MTP frameworks need to add the package. Tracking issue: microsoft/testfx#8926 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 40ac312 commit 733957e

5 files changed

Lines changed: 397 additions & 0 deletions

File tree

‎src/Microsoft.DotNet.Helix/Sdk/CreateMTPWorkItems.cs‎

Lines changed: 162 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,162 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Threading.Tasks;
9+
using Microsoft.Build.Framework;
10+
11+
namespace Microsoft.DotNet.Helix.Sdk
12+
{
13+
/// <summary>
14+
/// MSBuild custom task to create HelixWorkItems for test projects that target
15+
/// Microsoft.Testing.Platform (MTP). Such projects ship as self-hosting executables
16+
/// (a Main entrypoint is emitted by Microsoft.Testing.Platform.MSBuild) and can be
17+
/// run directly with 'dotnet exec'. This applies to MSTest 4.x, xUnit v3 with MTP,
18+
/// NUnit with MTP, TUnit, and any custom MTP-based test framework.
19+
/// </summary>
20+
public class CreateMTPWorkItems : BaseTask
21+
{
22+
/// <summary>
23+
/// An array of MTP project work items containing the following metadata:
24+
/// - [Required] PublishDirectory: the publish output directory of the test project
25+
/// - [Required] TargetPath: the output dll path
26+
/// - [Optional] Arguments: a string of arguments to be passed to the test executable
27+
/// *after* the auto-injected reporter flags
28+
/// The two required parameters are populated automatically by MTPRunner.targets when
29+
/// MTPProject.Identity is set to the path of the test csproj file.
30+
/// </summary>
31+
[Required]
32+
public ITaskItem[] MTPProjects { get; set; }
33+
34+
/// <summary>
35+
/// The path to the dotnet executable on the Helix agent. Defaults to "dotnet".
36+
/// </summary>
37+
public string PathToDotnet { get; set; } = "dotnet";
38+
39+
/// <summary>
40+
/// Boolean true if this is a posix shell, false if not.
41+
/// This does not need to be set by a user; it is automatically determined in
42+
/// Microsoft.DotNet.Helix.Sdk.MonoQueue.targets. Currently unused (the dotnet exec
43+
/// command is identical on every shell) but accepted for symmetry with the other
44+
/// Create*WorkItems tasks and to allow future shell-specific tweaks without an
45+
/// API break.
46+
/// </summary>
47+
[Required]
48+
public bool IsPosixShell { get; set; }
49+
50+
/// <summary>
51+
/// Optional timeout for all created work items.
52+
/// Accepts any value parseable by <see cref="TimeSpan.TryParse(string, out TimeSpan)"/>.
53+
/// Defaults to 5 minutes.
54+
/// </summary>
55+
public string MTPWorkItemTimeout { get; set; }
56+
57+
/// <summary>
58+
/// Optional name of the TRX file produced by Microsoft.Testing.Extensions.TrxReport.
59+
/// Defaults to "testResults.trx". Whatever value is used here is what the arcade
60+
/// Python TRXFormat parser will pick up from the work item's working directory.
61+
/// </summary>
62+
public string TrxReportFilename { get; set; } = "testResults.trx";
63+
64+
/// <summary>
65+
/// An array of ITaskItems of type HelixWorkItem.
66+
/// </summary>
67+
[Output]
68+
public ITaskItem[] MTPWorkItems { get; set; }
69+
70+
public override bool Execute()
71+
{
72+
ExecuteAsync().GetAwaiter().GetResult();
73+
return !Log.HasLoggedErrors;
74+
}
75+
76+
private async Task ExecuteAsync()
77+
{
78+
// De-duplicate inputs by (Identity, AdditionalProperties). This collapses
79+
// accidental duplicates such as a project listed under both MSTestProject and
80+
// MTPProject, while preserving legitimate same-Identity entries that differ in
81+
// AdditionalProperties (the canonical multi-TFM pattern).
82+
var deduped = MTPProjects
83+
.GroupBy(p => (p.ItemSpec, p.GetMetadata("AdditionalProperties") ?? string.Empty))
84+
.Select(g => g.First())
85+
.ToArray();
86+
87+
MTPWorkItems = (await Task.WhenAll(deduped.Select(PrepareWorkItem))).Where(wi => wi != null).ToArray();
88+
}
89+
90+
/// <summary>
91+
/// Prepares a HelixWorkItem for a single MTP test project.
92+
/// </summary>
93+
private async Task<ITaskItem> PrepareWorkItem(ITaskItem mtpProject)
94+
{
95+
// Forces this task to run asynchronously
96+
await Task.Yield();
97+
98+
if (!mtpProject.GetRequiredMetadata(Log, "PublishDirectory", out string publishDirectory))
99+
{
100+
return null;
101+
}
102+
if (!mtpProject.GetRequiredMetadata(Log, "TargetPath", out string targetPath))
103+
{
104+
return null;
105+
}
106+
107+
mtpProject.TryGetMetadata("Arguments", out string arguments);
108+
109+
string assemblyName = Path.GetFileName(targetPath);
110+
string assemblyBaseName = assemblyName;
111+
if (assemblyBaseName.EndsWith(".dll"))
112+
{
113+
assemblyBaseName = assemblyBaseName.Substring(0, assemblyBaseName.Length - 4);
114+
}
115+
116+
// MTP test apps are self-hosting executables. Run the assembly directly with
117+
// 'dotnet exec'. The reporter args below require the test project to reference
118+
// Microsoft.Testing.Extensions.TrxReport (which MSTest.Sdk references by default).
119+
// --results-directory . -> TRX is written next to the assembly, which is the
120+
// work item's cwd and is scanned by the arcade Python
121+
// reporter / EnableHelixJobMonitor.
122+
// --report-trx -> enable the TrxReport extension.
123+
// --report-trx-filename -> deterministic filename so the parser finds it.
124+
//
125+
// Note about the AzureDevOps reporter (Microsoft.Testing.Extensions.AzureDevOpsReport):
126+
// it activates only when '--report-azdo' is explicitly passed AND TF_BUILD=true.
127+
// We never pass '--report-azdo' here, so there is no risk of a duplicate Azure
128+
// DevOps test run on top of the one the Helix Sdk already opened. If a user has
129+
// wired '--report-azdo' into their test project they should remove it for Helix
130+
// runs (it conflicts with the Helix-managed run).
131+
string reporterArgs =
132+
$"--results-directory . --report-trx --report-trx-filename {TrxReportFilename}";
133+
134+
string command = $"{PathToDotnet} exec --roll-forward Major " +
135+
$"--runtimeconfig {assemblyBaseName}.runtimeconfig.json " +
136+
$"--depsfile {assemblyBaseName}.deps.json " +
137+
$"{assemblyName} {reporterArgs}" +
138+
(string.IsNullOrEmpty(arguments) ? "" : " " + arguments);
139+
140+
Log.LogMessage($"Creating MTP work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}");
141+
142+
TimeSpan timeout = TimeSpan.FromMinutes(5);
143+
if (!string.IsNullOrEmpty(MTPWorkItemTimeout))
144+
{
145+
if (!TimeSpan.TryParse(MTPWorkItemTimeout, out timeout))
146+
{
147+
Log.LogWarning($"Invalid value \"{MTPWorkItemTimeout}\" provided for MTPWorkItemTimeout; falling back to default value of \"00:05:00\" (5 minutes)");
148+
}
149+
}
150+
151+
var result = new Microsoft.Build.Utilities.TaskItem(assemblyName, new Dictionary<string, string>()
152+
{
153+
{ "Identity", assemblyName },
154+
{ "PayloadDirectory", publishDirectory },
155+
{ "Command", command },
156+
{ "Timeout", timeout.ToString() },
157+
});
158+
mtpProject.CopyMetadataTo(result);
159+
return result;
160+
}
161+
}
162+
}

‎src/Microsoft.DotNet.Helix/Sdk/Readme.md‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,139 @@ Given a local folder `$(TestFolder)` containing `runtests.cmd`, this will run `r
377377
</Project>
378378
```
379379

380+
### Microsoft Testing Platform (MTP) — MSTest, NUnit, TUnit, and other MTP-based frameworks
381+
382+
Test projects targeting [Microsoft.Testing.Platform](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-intro)
383+
are self-hosting executables (a `Main` entrypoint is emitted by
384+
`Microsoft.Testing.Platform.MSBuild`). The Helix SDK can publish them and turn each into a
385+
Helix work item automatically.
386+
387+
Two item types are recognised:
388+
389+
| Item type | Use when |
390+
| ---------------- | -------------------------------------------------------------------------------------------------------------- |
391+
| `MSTestProject` | A modern MSTest test project that runs on MTP — normally a project using `MSTest.Sdk`, or `Microsoft.NET.Sdk` + `EnableMSTestRunner=true`. Discoverability shim; folded into `MTPProject` before any target runs. |
392+
| `MTPProject` | Any other MTP-based project (NUnit with MTP runner, TUnit, custom MTP). Also works for any MSTest project — `MSTestProject` is just a friendlier name. |
393+
394+
> **`MSTestProject` does _not_ cover legacy VSTest-based MSTest** (the
395+
> `Microsoft.NET.Test.Sdk` + `MSTest.TestAdapter` pattern, no MTP). For those, either
396+
> migrate to MTP (e.g. switch to `MSTest.Sdk` or set `EnableMSTestRunner=true`), or keep
397+
> using a hand-written `<HelixWorkItem>` that invokes `dotnet test --logger trx`.
398+
399+
xUnit users should continue using `XUnitV3Project` (xUnit v3) or `XUnitProject` (xUnit v2);
400+
`XUnitV3Project` already takes the MTP path internally and produces xUnit XML rather than TRX.
401+
402+
#### Prerequisites
403+
404+
The test project must reference `Microsoft.Testing.Extensions.TrxReport`:
405+
406+
- Projects using `MSTest.Sdk` get this transitively — no action required.
407+
- Plain `Microsoft.NET.Sdk` projects (MSTest with `EnableMSTestRunner=true`, NUnit with MTP,
408+
TUnit, …) need to add the package explicitly:
409+
410+
```xml
411+
<PackageReference Include="Microsoft.Testing.Extensions.TrxReport" />
412+
```
413+
414+
If the package is missing the work item command will fail with an unknown-option error from
415+
`Microsoft.Testing.Platform` at runtime.
416+
417+
#### Minimal example
418+
419+
```xml
420+
<Project Sdk="Microsoft.DotNet.Helix.Sdk">
421+
422+
<PropertyGroup>
423+
<HelixSource>pr/example</HelixSource>
424+
<HelixType>test/MyMSTestSuite/</HelixType>
425+
<HelixTargetQueues>Windows.11.Amd64.Open;Ubuntu.2204.Amd64.Open</HelixTargetQueues>
426+
<EnableAzurePipelinesReporter>true</EnableAzurePipelinesReporter>
427+
</PropertyGroup>
428+
429+
<ItemGroup>
430+
<MSTestProject Include="..\test\MyApp.Tests\MyApp.Tests.csproj" />
431+
</ItemGroup>
432+
433+
</Project>
434+
```
435+
436+
That's it. Each project is published, packaged as a per-work-item payload, and the generated
437+
command is:
438+
439+
```text
440+
dotnet exec --roll-forward Major \
441+
--runtimeconfig MyApp.Tests.runtimeconfig.json \
442+
--depsfile MyApp.Tests.deps.json \
443+
MyApp.Tests.dll \
444+
--results-directory . --report-trx --report-trx-filename testResults.trx
445+
```
446+
447+
The TRX file lands in the work item working directory and is picked up by the standard
448+
arcade Helix Python reporter (`TRXFormat` parser), or — when `EnableHelixJobMonitor=true` —
449+
swept into `$HELIX_WORKITEM_UPLOAD_ROOT` for the standalone monitor.
450+
451+
#### Multi-targeted test projects
452+
453+
A single multi-TFM project (e.g. `<TargetFrameworks>net8.0;net9.0</TargetFrameworks>`) can be
454+
sent to Helix twice, once per TFM, by listing the same project with different
455+
`AdditionalProperties`:
456+
457+
```xml
458+
<ItemGroup>
459+
<MSTestProject Include="..\test\MyApp.Tests\MyApp.Tests.csproj"
460+
AdditionalProperties="TargetFramework=net8.0" />
461+
<MSTestProject Include="..\test\MyApp.Tests\MyApp.Tests.csproj"
462+
AdditionalProperties="TargetFramework=net9.0" />
463+
</ItemGroup>
464+
```
465+
466+
Each entry produces its own work item with its own publish output and TRX file. Duplicate
467+
entries that share both `Identity` and `AdditionalProperties` are collapsed automatically.
468+
469+
#### Supported metadata on `MSTestProject` / `MTPProject`
470+
471+
| Metadata | Required | Description |
472+
| ---------------------- | -------- | -------------------------------------------------------------------------------------------------------------- |
473+
| `Identity` | yes | Path to the test `.csproj`. |
474+
| `Arguments` | no | Extra arguments appended to the generated command (e.g. `--filter-not-trait category=failing`). |
475+
| `AdditionalProperties` | no | Additional MSBuild properties passed to Restore/Publish (e.g. `Configuration=Release;TargetFramework=net9.0`). |
476+
477+
#### Properties
478+
479+
| Property | Default | Description |
480+
| ------------------------ | ------------------ | ---------------------------------------------------------------------------------------------------- |
481+
| `MTPWorkItemTimeout` | `00:05:00` | Per-work-item timeout. Any `TimeSpan`-parseable string. |
482+
| `MTPTrxReportFilename` | `testResults.trx` | Name of the TRX file produced by `Microsoft.Testing.Extensions.TrxReport`. Rarely needs changing. |
483+
| `HelixTargetQueues` | _required_ | One or more Helix queues. See <https://helix.dot.net/>. |
484+
485+
#### Azure DevOps reporter — do not pass `--report-azdo` for Helix runs
486+
487+
`Microsoft.Testing.Platform` ships an Azure DevOps reporter
488+
(`Microsoft.Testing.Extensions.AzureDevOpsReport`). It is opt-in — it activates only when
489+
**both** `--report-azdo` is passed _and_ `TF_BUILD=true` — so it stays inert in a Helix work
490+
item by default and the Helix SDK's `EnableAzurePipelinesReporter` path is the single source
491+
of truth for AzDO test runs.
492+
493+
If your test project hard-codes `--report-azdo` (for example via
494+
[`<TestingExtensionsProfile>AllMicrosoft</TestingExtensionsProfile>`](https://learn.microsoft.com/dotnet/core/testing/unit-testing-mstest-sdk)
495+
and a wrapper script that always passes it), remove it from the Helix invocation —
496+
otherwise you will get a duplicate Azure DevOps test run on top of the one the Helix SDK
497+
opens via `StartAzurePipelinesTestRun`.
498+
499+
#### Relation to `XUnitV3Project`
500+
501+
`XUnitV3Project` predates `MTPProject` and is xUnit-specific:
502+
503+
- It produces **xUnit XML** (`--report-xunit-xml`), not TRX. Both formats are handled by the
504+
arcade reporter, but the file shape and on-disk layout differ.
505+
- It does not require `Microsoft.Testing.Extensions.TrxReport`.
506+
- It additionally supports the legacy non-MTP xUnit v3 runner via
507+
`UseMicrosoftTestingPlatformRunner=false`.
508+
509+
If you have an existing `XUnitV3Project` set-up, keep using it — there is no benefit to
510+
migrating to `MTPProject`. `MTPProject` exists so non-xUnit MTP frameworks (MSTest, NUnit,
511+
TUnit, …) get the same one-line first-class experience.
512+
380513
### iOS/Android/WASM workload support (XHarness)
381514
The Helix SDK also supports execution of Android/iOS/WASM workloads where you only need to point it to an Android .apk or an iOS/tvOS/WatchOS .app bundle and it will execute these using a tool called XHarness on a specified emulator/device/JS engine. The workloads have to run on Helix queues that are ready for these types of jobs, meaning they have emulators installed, devices connected or JS engine installed. You can read more about this [here](https://github.com/dotnet/arcade/blob/master/src/Microsoft.DotNet.Helix/Sdk/tools/xharness-runner/Readme.md).
382515

‎src/Microsoft.DotNet.Helix/Sdk/tools/Microsoft.DotNet.Helix.Sdk.props‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
<UsingTask TaskName="GetHelixWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
4848
<UsingTask TaskName="CreateXUnitWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
4949
<UsingTask TaskName="CreateXUnitV3WorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
50+
<UsingTask TaskName="CreateMTPWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
5051
<UsingTask TaskName="CreateXHarnessAndroidWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
5152
<UsingTask TaskName="CreateXHarnessAppleWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />
5253
<UsingTask TaskName="CreateTestsForWorkItems" AssemblyFile="$(MicrosoftDotNetHelixSdkTasksAssembly)" Runtime="NET" Architecture="*" TaskFactory="$(MicrosoftDotNetHelixSdkTasksFactory)" />

‎src/Microsoft.DotNet.Helix/Sdk/tools/mtp-runner/MTPRunner.props‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
<!-- Licensed to the .NET Foundation under one or more agreements. The .NET Foundation licenses this file to you under the MIT license. -->
2+
<Project>
3+
4+
<PropertyGroup>
5+
<_HelixMonoQueueTargets>$(_HelixMonoQueueTargets);$(MSBuildThisFileDirectory)MTPRunner.targets</_HelixMonoQueueTargets>
6+
</PropertyGroup>
7+
8+
</Project>

0 commit comments

Comments
 (0)
⚡