Skip to content

Commit a6bacc0

Browse files
EvangelinkEvangelink
authored
Rename XUnitV3Project to MTPProject in Helix Sdk (breaking) (#16986)
2 parents 822bb86 + 01e75a5 commit a6bacc0

10 files changed

Lines changed: 473 additions & 215 deletions

File tree

‎Documentation/AzureDevOps/SendingJobsToHelix.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -119,9 +119,11 @@ The list of available Helix queues can be found on the [Helix homepage](https://
119119
# condition: succeeded() - defaults to succeeded()
120120
```
121121

122-
### XUnit v3
122+
### Microsoft.Testing.Platform (MTP)
123123

124-
XUnit v3 test projects are self-hosting executables and do not need an external console runner. Instead of `XUnitProjects`, use `XUnitV3Project` items directly in your Helix MSBuild project file (see [the SDK's readme](/src/Microsoft.DotNet.Helix/Sdk/Readme.md) for details). The `XUnitPublishTargetFramework`, `XUnitRuntimeTargetFramework`, and `XUnitRunnerVersion` parameters are not needed for v3 projects.
124+
Test projects that target [Microsoft.Testing.Platform](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-intro) (MTP) are self-hosting executables and do not need an external console runner. This covers MSTest 4.x with the MTP runner, xUnit v3 with MTP (the default for v3), NUnit with the MTP runner, TUnit, and any custom MTP-based framework.
125+
126+
Instead of `XUnitProjects`, use `MTPProject` items directly in your Helix MSBuild project file (see [the SDK's readme](/src/Microsoft.DotNet.Helix/Sdk/Readme.md) for details). Each project must reference `Microsoft.Testing.Extensions.TrxReport`; this is implicit for `MSTest.Sdk` projects and for xUnit v3 projects built with `Microsoft.DotNet.Arcade.Sdk`'s XUnitV3 targets. The `XUnitPublishTargetFramework`, `XUnitRuntimeTargetFramework`, and `XUnitRunnerVersion` parameters are not needed for MTP projects.
125127

126128
## The More Complex Case
127129

‎src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateMTPWorkItemsTests.cs‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System.Collections.Generic;
5+
using System.Linq;
6+
using AwesomeAssertions;
7+
using Microsoft.Arcade.Test.Common;
8+
using Microsoft.Build.Framework;
9+
using Microsoft.Build.Utilities;
10+
using Xunit;
11+
12+
#nullable enable
13+
namespace Microsoft.DotNet.Helix.Sdk.Tests
14+
{
15+
public class CreateMTPWorkItemsTests
16+
{
17+
private static CreateMTPWorkItems CreateTask() =>
18+
new CreateMTPWorkItems
19+
{
20+
BuildEngine = new MockBuildEngine(),
21+
IsPosixShell = false,
22+
};
23+
24+
private static ITaskItem CreateProject(
25+
string itemSpec,
26+
string? publishDirectory = "/publish",
27+
string? targetPath = "/publish/MyApp.Tests.dll",
28+
string? arguments = null,
29+
string? additionalProperties = null)
30+
{
31+
var metadata = new Dictionary<string, string>();
32+
if (publishDirectory != null) metadata["PublishDirectory"] = publishDirectory;
33+
if (targetPath != null) metadata["TargetPath"] = targetPath;
34+
if (arguments != null) metadata["Arguments"] = arguments;
35+
if (additionalProperties != null) metadata["AdditionalProperties"] = additionalProperties;
36+
return new TaskItem(itemSpec, metadata);
37+
}
38+
39+
[Fact]
40+
public void GeneratedCommandHasExpectedShape()
41+
{
42+
var task = CreateTask();
43+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
44+
45+
task.Execute().Should().BeTrue();
46+
47+
task.MTPWorkItems.Should().HaveCount(1);
48+
var workItem = task.MTPWorkItems.Single();
49+
workItem.GetMetadata("Identity").Should().Be("MyApp.Tests.dll");
50+
workItem.GetMetadata("PayloadDirectory").Should().Be("/publish");
51+
workItem.GetMetadata("Timeout").Should().Be("00:05:00");
52+
53+
var command = workItem.GetMetadata("Command");
54+
command.Should().StartWith("dotnet exec --roll-forward Major ");
55+
command.Should().Contain("--runtimeconfig MyApp.Tests.runtimeconfig.json");
56+
command.Should().Contain("--depsfile MyApp.Tests.deps.json");
57+
command.Should().Contain("MyApp.Tests.dll");
58+
command.Should().EndWith(" --results-directory .");
59+
command.Should().NotContain("--report-trx");
60+
command.Should().NotContain("--auto-reporters");
61+
}
62+
63+
[Fact]
64+
public void ArgumentsMetadataIsAppendedAfterReporterArgs()
65+
{
66+
var task = CreateTask();
67+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", arguments: "--filter Category=Smoke") };
68+
69+
task.Execute().Should().BeTrue();
70+
71+
var command = task.MTPWorkItems.Single().GetMetadata("Command");
72+
command.Should().EndWith("--results-directory . --filter Category=Smoke");
73+
}
74+
75+
[Fact]
76+
public void MTPAdditionalArgumentsIsInsertedBeforePerProjectArguments()
77+
{
78+
var task = CreateTask();
79+
task.MTPAdditionalArguments = "--report-trx --report-trx-filename testResults.trx --auto-reporters off";
80+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", arguments: "--filter Category=Smoke") };
81+
82+
task.Execute().Should().BeTrue();
83+
84+
var command = task.MTPWorkItems.Single().GetMetadata("Command");
85+
command.Should().EndWith("--results-directory . --report-trx --report-trx-filename testResults.trx --auto-reporters off --filter Category=Smoke");
86+
}
87+
88+
[Fact]
89+
public void MTPAdditionalArgumentsAloneIsAppendedAfterReporterArgs()
90+
{
91+
var task = CreateTask();
92+
task.MTPAdditionalArguments = "--auto-reporters off";
93+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
94+
95+
task.Execute().Should().BeTrue();
96+
97+
var command = task.MTPWorkItems.Single().GetMetadata("Command");
98+
command.Should().EndWith("--results-directory . --auto-reporters off");
99+
}
100+
101+
[Fact]
102+
public void MissingPublishDirectoryReturnsNoWorkItem()
103+
{
104+
var task = CreateTask();
105+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", publishDirectory: null) };
106+
107+
task.Execute().Should().BeFalse();
108+
task.MTPWorkItems.Should().BeEmpty();
109+
}
110+
111+
[Fact]
112+
public void MissingTargetPathReturnsNoWorkItem()
113+
{
114+
var task = CreateTask();
115+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", targetPath: null) };
116+
117+
task.Execute().Should().BeFalse();
118+
task.MTPWorkItems.Should().BeEmpty();
119+
}
120+
121+
[Fact]
122+
public void ValidTimeoutIsApplied()
123+
{
124+
var task = CreateTask();
125+
task.MTPWorkItemTimeout = "00:12:34";
126+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
127+
128+
task.Execute().Should().BeTrue();
129+
task.MTPWorkItems.Single().GetMetadata("Timeout").Should().Be("00:12:34");
130+
}
131+
132+
[Fact]
133+
public void InvalidTimeoutFallsBackToDefaultWithWarning()
134+
{
135+
var task = CreateTask();
136+
var buildEngine = (MockBuildEngine)task.BuildEngine;
137+
task.MTPWorkItemTimeout = "not-a-timespan";
138+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
139+
140+
task.Execute().Should().BeTrue();
141+
task.MTPWorkItems.Single().GetMetadata("Timeout").Should().Be("00:05:00");
142+
buildEngine.BuildWarningEvents.Should().Contain(e => e.Message != null && e.Message.Contains("not-a-timespan"));
143+
}
144+
145+
[Fact]
146+
public void DuplicateInputsArePassedThroughAsSeparateWorkItems()
147+
{
148+
// The task does not deduplicate - it mirrors the old CreateXUnitV3WorkItems behavior
149+
// and trusts the caller to provide a clean item list (MSBuild's RemoveDuplicates can
150+
// be used upstream if needed).
151+
var task = CreateTask();
152+
task.MTPProjects = new[]
153+
{
154+
CreateProject("MyApp.Tests.csproj"),
155+
CreateProject("MyApp.Tests.csproj"),
156+
};
157+
158+
task.Execute().Should().BeTrue();
159+
task.MTPWorkItems.Should().HaveCount(2);
160+
}
161+
162+
[Fact]
163+
public void SameIdentityWithDifferentAdditionalPropertiesProducesSeparateWorkItems()
164+
{
165+
var task = CreateTask();
166+
task.MTPProjects = new[]
167+
{
168+
CreateProject(
169+
"MyApp.Tests.csproj",
170+
publishDirectory: "/publish/net8.0",
171+
targetPath: "/publish/net8.0/MyApp.Tests.dll",
172+
additionalProperties: "TargetFramework=net8.0"),
173+
CreateProject(
174+
"MyApp.Tests.csproj",
175+
publishDirectory: "/publish/net9.0",
176+
targetPath: "/publish/net9.0/MyApp.Tests.dll",
177+
additionalProperties: "TargetFramework=net9.0"),
178+
};
179+
180+
task.Execute().Should().BeTrue();
181+
task.MTPWorkItems.Should().HaveCount(2);
182+
task.MTPWorkItems.Select(w => w.GetMetadata("PayloadDirectory"))
183+
.Should().BeEquivalentTo(new[] { "/publish/net8.0", "/publish/net9.0" });
184+
}
185+
186+
[Fact]
187+
public void PathToDotnetIsHonored()
188+
{
189+
var task = CreateTask();
190+
task.PathToDotnet = "/custom/dotnet";
191+
task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
192+
193+
task.Execute().Should().BeTrue();
194+
task.MTPWorkItems.Single().GetMetadata("Command").Should().StartWith("/custom/dotnet exec ");
195+
}
196+
}
197+
}

‎src/Microsoft.DotNet.Helix/Sdk/CreateMTPWorkItems.cs‎

Lines changed: 165 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,165 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Linq;
8+
using System.Threading.Tasks;
9+
using Microsoft.Build.Framework;
10+
11+
namespace Microsoft.DotNet.Helix.Sdk
12+
{
13+
/// <summary>
14+
/// MSBuild custom task to create HelixWorkItems for test projects that target
15+
/// Microsoft.Testing.Platform (MTP). Such projects ship as self-hosting executables
16+
/// (a Main entry point is emitted by Microsoft.Testing.Platform.MSBuild) and can be
17+
/// run directly with 'dotnet exec'. This applies to MSTest 4.x, xUnit v3 with MTP,
18+
/// NUnit with MTP, TUnit, and any custom MTP-based test framework.
19+
/// </summary>
20+
public class CreateMTPWorkItems : BaseTask
21+
{
22+
/// <summary>
23+
/// An array of MTP project work items containing the following metadata:
24+
/// - [Required] PublishDirectory: the publish output directory of the test project
25+
/// - [Required] TargetPath: the output dll path
26+
/// - [Optional] Arguments: extra arguments appended to the generated dotnet exec
27+
/// command, after the built-in <c>--results-directory .</c> flag and after any
28+
/// value supplied via <see cref="MTPAdditionalArguments"/>.
29+
/// The two required parameters are populated automatically by MTPRunner.targets when
30+
/// MTPProject.Identity is set to the path of the test csproj file.
31+
/// </summary>
32+
[Required]
33+
public ITaskItem[] MTPProjects { get; set; }
34+
35+
/// <summary>
36+
/// The path to the dotnet executable on the Helix agent. Defaults to "dotnet".
37+
/// </summary>
38+
public string PathToDotnet { get; set; } = "dotnet";
39+
40+
/// <summary>
41+
/// Boolean true if this is a posix shell, false if not.
42+
/// This does not need to be set by a user; it is automatically determined in
43+
/// Microsoft.DotNet.Helix.Sdk.MonoQueue.targets. Currently unused (the dotnet exec
44+
/// command is identical on every shell) but accepted for symmetry with the other
45+
/// Create*WorkItems tasks and to allow future shell-specific tweaks without an
46+
/// API break.
47+
/// </summary>
48+
[Required]
49+
public bool IsPosixShell { get; set; }
50+
51+
/// <summary>
52+
/// Optional timeout for all created work items.
53+
/// Accepts any value parseable by <see cref="TimeSpan.TryParse(string, out TimeSpan)"/>.
54+
/// Defaults to 5 minutes.
55+
/// </summary>
56+
public string MTPWorkItemTimeout { get; set; }
57+
58+
/// <summary>
59+
/// Optional extra command-line arguments to append to every emitted MTP work item
60+
/// command, between the built-in <c>--results-directory</c> flag and any per-project
61+
/// <c>Arguments</c> metadata. Use this to apply framework- or extension-specific
62+
/// switches across all MTP projects in a run. Examples:
63+
/// <list type="bullet">
64+
/// <item><description>
65+
/// <c>--report-trx --report-trx-filename testResults.trx</c> to explicitly
66+
/// control the TRX file name (requires <c>Microsoft.Testing.Extensions.TrxReport</c>).
67+
/// </description></item>
68+
/// <item><description>
69+
/// <c>--auto-reporters off</c> for xUnit v3 with MTP, where it suppresses xUnit's
70+
/// auto-activated reporters; MSTest / NUnit / TUnit reject the option as unknown,
71+
/// so we do not inject it by default.
72+
/// </description></item>
73+
/// </list>
74+
/// </summary>
75+
public string MTPAdditionalArguments { get; set; }
76+
77+
/// <summary>
78+
/// An array of ITaskItems of type HelixWorkItem.
79+
/// </summary>
80+
[Output]
81+
public ITaskItem[] MTPWorkItems { get; set; }
82+
83+
public override bool Execute()
84+
{
85+
ExecuteAsync().GetAwaiter().GetResult();
86+
return !Log.HasLoggedErrors;
87+
}
88+
89+
private async Task ExecuteAsync()
90+
{
91+
MTPWorkItems = (await Task.WhenAll(MTPProjects.Select(PrepareWorkItem))).Where(wi => wi != null).ToArray();
92+
}
93+
94+
/// <summary>
95+
/// Prepares a HelixWorkItem for a single MTP test project.
96+
/// </summary>
97+
private async Task<ITaskItem> PrepareWorkItem(ITaskItem mtpProject)
98+
{
99+
// Forces this task to run asynchronously
100+
await Task.Yield();
101+
102+
if (!mtpProject.GetRequiredMetadata(Log, "PublishDirectory", out string publishDirectory))
103+
{
104+
return null;
105+
}
106+
if (!mtpProject.GetRequiredMetadata(Log, "TargetPath", out string targetPath))
107+
{
108+
return null;
109+
}
110+
111+
mtpProject.TryGetMetadata("Arguments", out string arguments);
112+
113+
string assemblyName = Path.GetFileName(targetPath);
114+
string assemblyBaseName = assemblyName;
115+
if (assemblyBaseName.EndsWith(".dll"))
116+
{
117+
assemblyBaseName = assemblyBaseName.Substring(0, assemblyBaseName.Length - 4);
118+
}
119+
120+
// MTP test apps are self-hosting executables. Run the assembly directly with
121+
// 'dotnet exec'. We pass only --results-directory (a built-in MTP option) so
122+
// any reporter-produced artifacts land in the work item's working directory
123+
// where Helix collects results from.
124+
//
125+
// We deliberately do NOT inject reporter flags here:
126+
// * --report-trx requires the Microsoft.Testing.Extensions.TrxReport
127+
// extension to be referenced; MTP apps without it would error out.
128+
// With MTP's --auto-reporters on by default, projects that DO reference
129+
// the TrxReport extension produce a TRX file automatically (default name
130+
// based on assembly + tfm + timestamp); arcade's TRXFormat reporter
131+
// globs *.trx so the auto-generated name is fine.
132+
// * --auto-reporters off is registered by xUnit v3's MTP integration and
133+
// is rejected as unknown by MSTest / NUnit / TUnit MTP apps.
134+
// Users who need such options can set MTPAdditionalArguments (applies to
135+
// every work item) or the per-project Arguments metadata on MTPProject.
136+
string reporterArgs = "--results-directory .";
137+
138+
string command = $"{PathToDotnet} exec --roll-forward Major " +
139+
$"--runtimeconfig {assemblyBaseName}.runtimeconfig.json " +
140+
$"--depsfile {assemblyBaseName}.deps.json " +
141+
$"{assemblyName} {reporterArgs}" +
142+
(string.IsNullOrEmpty(MTPAdditionalArguments) ? "" : " " + MTPAdditionalArguments) +
143+
(string.IsNullOrEmpty(arguments) ? "" : " " + arguments);
144+
145+
Log.LogMessage($"Creating MTP work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}");
146+
147+
TimeSpan timeout = TimeSpan.FromMinutes(5);
148+
if (!string.IsNullOrEmpty(MTPWorkItemTimeout) && !TimeSpan.TryParse(MTPWorkItemTimeout, out timeout))
149+
{
150+
Log.LogWarning($"Invalid value \"{MTPWorkItemTimeout}\" provided for MTPWorkItemTimeout; falling back to default value of \"00:05:00\" (5 minutes)");
151+
timeout = TimeSpan.FromMinutes(5);
152+
}
153+
154+
var result = new Microsoft.Build.Utilities.TaskItem(assemblyName, new Dictionary<string, string>()
155+
{
156+
{ "Identity", assemblyName },
157+
{ "PayloadDirectory", publishDirectory },
158+
{ "Command", command },
159+
{ "Timeout", timeout.ToString() },
160+
});
161+
mtpProject.CopyMetadataTo(result);
162+
return result;
163+
}
164+
}
165+
}

0 commit comments

Comments
 (0)
⚡