Rename XUnitV3Project to MTPProject in Helix Sdk (breaking)

Documentation/AzureDevOps/SendingJobsToHelix.md

@@ -119,9 +119,11 @@ The list of available Helix queues can be found on the [Helix homepage](https://
       # condition: succeeded() - defaults to succeeded()
 ```
 
-### XUnit v3
+### Microsoft.Testing.Platform (MTP)
 
-XUnit v3 test projects are self-hosting executables and do not need an external console runner. Instead of `XUnitProjects`, use `XUnitV3Project` items directly in your Helix MSBuild project file (see [the SDK's readme](/src/Microsoft.DotNet.Helix/Sdk/Readme.md) for details). The `XUnitPublishTargetFramework`, `XUnitRuntimeTargetFramework`, and `XUnitRunnerVersion` parameters are not needed for v3 projects.
+Test projects that target [Microsoft.Testing.Platform](https://learn.microsoft.com/dotnet/core/testing/microsoft-testing-platform-intro) (MTP) are self-hosting executables and do not need an external console runner. This covers MSTest 4.x with the MTP runner, xUnit v3 with MTP (the default for v3), NUnit with the MTP runner, TUnit, and any custom MTP-based framework.
+
+Instead of `XUnitProjects`, use `MTPProject` items directly in your Helix MSBuild project file (see [the SDK's readme](/src/Microsoft.DotNet.Helix/Sdk/Readme.md) for details). Each project must reference `Microsoft.Testing.Extensions.TrxReport`; this is implicit for `MSTest.Sdk` projects and for xUnit v3 projects built with `Microsoft.DotNet.Arcade.Sdk`'s XUnitV3 targets. The `XUnitPublishTargetFramework`, `XUnitRuntimeTargetFramework`, and `XUnitRunnerVersion` parameters are not needed for MTP projects.
 
 ## The More Complex Case
 

src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/CreateMTPWorkItemsTests.cs

@@ -0,0 +1,197 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Collections.Generic;
+using System.Linq;
+using AwesomeAssertions;
+using Microsoft.Arcade.Test.Common;
+using Microsoft.Build.Framework;
+using Microsoft.Build.Utilities;
+using Xunit;
+
+#nullable enable
+namespace Microsoft.DotNet.Helix.Sdk.Tests
+{
+    public class CreateMTPWorkItemsTests
+    {
+        private static CreateMTPWorkItems CreateTask() =>
+            new CreateMTPWorkItems
+            {
+                BuildEngine = new MockBuildEngine(),
+                IsPosixShell = false,
+            };
+
+        private static ITaskItem CreateProject(
+            string itemSpec,
+            string? publishDirectory = "/publish",
+            string? targetPath = "/publish/MyApp.Tests.dll",
+            string? arguments = null,
+            string? additionalProperties = null)
+        {
+            var metadata = new Dictionary<string, string>();
+            if (publishDirectory != null) metadata["PublishDirectory"] = publishDirectory;
+            if (targetPath != null) metadata["TargetPath"] = targetPath;
+            if (arguments != null) metadata["Arguments"] = arguments;
+            if (additionalProperties != null) metadata["AdditionalProperties"] = additionalProperties;
+            return new TaskItem(itemSpec, metadata);
+        }
+
+        [Fact]
+        public void GeneratedCommandHasExpectedShape()
+        {
+            var task = CreateTask();
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
+
+            task.Execute().Should().BeTrue();
+
+            task.MTPWorkItems.Should().HaveCount(1);
+            var workItem = task.MTPWorkItems.Single();
+            workItem.GetMetadata("Identity").Should().Be("MyApp.Tests.dll");
+            workItem.GetMetadata("PayloadDirectory").Should().Be("/publish");
+            workItem.GetMetadata("Timeout").Should().Be("00:05:00");
+
+            var command = workItem.GetMetadata("Command");
+            command.Should().StartWith("dotnet exec --roll-forward Major ");
+            command.Should().Contain("--runtimeconfig MyApp.Tests.runtimeconfig.json");
+            command.Should().Contain("--depsfile MyApp.Tests.deps.json");
+            command.Should().Contain("MyApp.Tests.dll");
+            command.Should().EndWith(" --results-directory .");
+            command.Should().NotContain("--report-trx");
+            command.Should().NotContain("--auto-reporters");
+        }
+
+        [Fact]
+        public void ArgumentsMetadataIsAppendedAfterReporterArgs()
+        {
+            var task = CreateTask();
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", arguments: "--filter Category=Smoke") };
+
+            task.Execute().Should().BeTrue();
+
+            var command = task.MTPWorkItems.Single().GetMetadata("Command");
+            command.Should().EndWith("--results-directory . --filter Category=Smoke");
+        }
+
+        [Fact]
+        public void MTPAdditionalArgumentsIsInsertedBeforePerProjectArguments()
+        {
+            var task = CreateTask();
+            task.MTPAdditionalArguments = "--report-trx --report-trx-filename testResults.trx --auto-reporters off";
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", arguments: "--filter Category=Smoke") };
+
+            task.Execute().Should().BeTrue();
+
+            var command = task.MTPWorkItems.Single().GetMetadata("Command");
+            command.Should().EndWith("--results-directory . --report-trx --report-trx-filename testResults.trx --auto-reporters off --filter Category=Smoke");
+        }
+
+        [Fact]
+        public void MTPAdditionalArgumentsAloneIsAppendedAfterReporterArgs()
+        {
+            var task = CreateTask();
+            task.MTPAdditionalArguments = "--auto-reporters off";
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
+
+            task.Execute().Should().BeTrue();
+
+            var command = task.MTPWorkItems.Single().GetMetadata("Command");
+            command.Should().EndWith("--results-directory . --auto-reporters off");
+        }
+
+        [Fact]
+        public void MissingPublishDirectoryReturnsNoWorkItem()
+        {
+            var task = CreateTask();
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", publishDirectory: null) };
+
+            task.Execute().Should().BeFalse();
+            task.MTPWorkItems.Should().BeEmpty();
+        }
+
+        [Fact]
+        public void MissingTargetPathReturnsNoWorkItem()
+        {
+            var task = CreateTask();
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj", targetPath: null) };
+
+            task.Execute().Should().BeFalse();
+            task.MTPWorkItems.Should().BeEmpty();
+        }
+
+        [Fact]
+        public void ValidTimeoutIsApplied()
+        {
+            var task = CreateTask();
+            task.MTPWorkItemTimeout = "00:12:34";
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
+
+            task.Execute().Should().BeTrue();
+            task.MTPWorkItems.Single().GetMetadata("Timeout").Should().Be("00:12:34");
+        }
+
+        [Fact]
+        public void InvalidTimeoutFallsBackToDefaultWithWarning()
+        {
+            var task = CreateTask();
+            var buildEngine = (MockBuildEngine)task.BuildEngine;
+            task.MTPWorkItemTimeout = "not-a-timespan";
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
+
+            task.Execute().Should().BeTrue();
+            task.MTPWorkItems.Single().GetMetadata("Timeout").Should().Be("00:05:00");
+            buildEngine.BuildWarningEvents.Should().Contain(e => e.Message != null && e.Message.Contains("not-a-timespan"));
+        }
+
+        [Fact]
+        public void DuplicateInputsArePassedThroughAsSeparateWorkItems()
+        {
+            // The task does not deduplicate - it mirrors the old CreateXUnitV3WorkItems behavior
+            // and trusts the caller to provide a clean item list (MSBuild's RemoveDuplicates can
+            // be used upstream if needed).
+            var task = CreateTask();
+            task.MTPProjects = new[]
+            {
+                CreateProject("MyApp.Tests.csproj"),
+                CreateProject("MyApp.Tests.csproj"),
+            };
+
+            task.Execute().Should().BeTrue();
+            task.MTPWorkItems.Should().HaveCount(2);
+        }
+
+        [Fact]
+        public void SameIdentityWithDifferentAdditionalPropertiesProducesSeparateWorkItems()
+        {
+            var task = CreateTask();
+            task.MTPProjects = new[]
+            {
+                CreateProject(
+                    "MyApp.Tests.csproj",
+                    publishDirectory: "/publish/net8.0",
+                    targetPath: "/publish/net8.0/MyApp.Tests.dll",
+                    additionalProperties: "TargetFramework=net8.0"),
+                CreateProject(
+                    "MyApp.Tests.csproj",
+                    publishDirectory: "/publish/net9.0",
+                    targetPath: "/publish/net9.0/MyApp.Tests.dll",
+                    additionalProperties: "TargetFramework=net9.0"),
+            };
+
+            task.Execute().Should().BeTrue();
+            task.MTPWorkItems.Should().HaveCount(2);
+            task.MTPWorkItems.Select(w => w.GetMetadata("PayloadDirectory"))
+                .Should().BeEquivalentTo(new[] { "/publish/net8.0", "/publish/net9.0" });
+        }
+
+        [Fact]
+        public void PathToDotnetIsHonored()
+        {
+            var task = CreateTask();
+            task.PathToDotnet = "/custom/dotnet";
+            task.MTPProjects = new[] { CreateProject("MyApp.Tests.csproj") };
+
+            task.Execute().Should().BeTrue();
+            task.MTPWorkItems.Single().GetMetadata("Command").Should().StartWith("/custom/dotnet exec ");
+        }
+    }
+}

src/Microsoft.DotNet.Helix/Sdk/CreateMTPWorkItems.cs

@@ -0,0 +1,164 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using Microsoft.Build.Framework;
+
+namespace Microsoft.DotNet.Helix.Sdk
+{
+    /// <summary>
+    /// MSBuild custom task to create HelixWorkItems for test projects that target
+    /// Microsoft.Testing.Platform (MTP). Such projects ship as self-hosting executables
+    /// (a Main entry point is emitted by Microsoft.Testing.Platform.MSBuild) and can be
+    /// run directly with 'dotnet exec'. This applies to MSTest 4.x, xUnit v3 with MTP,
+    /// NUnit with MTP, TUnit, and any custom MTP-based test framework.
+    /// </summary>
+    public class CreateMTPWorkItems : BaseTask
+    {
+        /// <summary>
+        /// An array of MTP project work items containing the following metadata:
+        /// - [Required] PublishDirectory: the publish output directory of the test project
+        /// - [Required] TargetPath: the output dll path
+        /// - [Optional] Arguments: a string of arguments to be passed to the test executable
+        ///   *after* the auto-injected reporter flags
+        /// The two required parameters are populated automatically by MTPRunner.targets when
+        /// MTPProject.Identity is set to the path of the test csproj file.
+        /// </summary>
+        [Required]
+        public ITaskItem[] MTPProjects { get; set; }
+
+        /// <summary>
+        /// The path to the dotnet executable on the Helix agent. Defaults to "dotnet".
+        /// </summary>
+        public string PathToDotnet { get; set; } = "dotnet";
+
+        /// <summary>
+        /// Boolean true if this is a posix shell, false if not.
+        /// This does not need to be set by a user; it is automatically determined in
+        /// Microsoft.DotNet.Helix.Sdk.MonoQueue.targets. Currently unused (the dotnet exec
+        /// command is identical on every shell) but accepted for symmetry with the other
+        /// Create*WorkItems tasks and to allow future shell-specific tweaks without an
+        /// API break.
+        /// </summary>
+        [Required]
+        public bool IsPosixShell { get; set; }
+
+        /// <summary>
+        /// Optional timeout for all created work items.
+        /// Accepts any value parseable by <see cref="TimeSpan.TryParse(string, out TimeSpan)"/>.
+        /// Defaults to 5 minutes.
+        /// </summary>
+        public string MTPWorkItemTimeout { get; set; }
+
+        /// <summary>
+        /// Optional extra command-line arguments to append to every emitted MTP work item
+        /// command, between the built-in <c>--results-directory</c> flag and any per-project
+        /// <c>Arguments</c> metadata. Use this to apply framework- or extension-specific
+        /// switches across all MTP projects in a run. Examples:
+        /// <list type="bullet">
+        ///   <item><description>
+        ///     <c>--report-trx --report-trx-filename testResults.trx</c> to explicitly
+        ///     control the TRX file name (requires <c>Microsoft.Testing.Extensions.TrxReport</c>).
+        ///   </description></item>
+        ///   <item><description>
+        ///     <c>--auto-reporters off</c> for xUnit v3 with MTP, where it suppresses xUnit's
+        ///     auto-activated reporters; MSTest / NUnit / TUnit reject the option as unknown,
+        ///     so we do not inject it by default.
+        ///   </description></item>
+        /// </list>
+        /// </summary>
+        public string MTPAdditionalArguments { get; set; }
+
+        /// <summary>
+        /// An array of ITaskItems of type HelixWorkItem.
+        /// </summary>
+        [Output]
+        public ITaskItem[] MTPWorkItems { get; set; }
+
+        public override bool Execute()
+        {
+            ExecuteAsync().GetAwaiter().GetResult();
+            return !Log.HasLoggedErrors;
+        }
+
+        private async Task ExecuteAsync()
+        {
+            MTPWorkItems = (await Task.WhenAll(MTPProjects.Select(PrepareWorkItem))).Where(wi => wi != null).ToArray();
+        }
+
+        /// <summary>
+        /// Prepares a HelixWorkItem for a single MTP test project.
+        /// </summary>
+        private async Task<ITaskItem> PrepareWorkItem(ITaskItem mtpProject)
+        {
+            // Forces this task to run asynchronously
+            await Task.Yield();
+
+            if (!mtpProject.GetRequiredMetadata(Log, "PublishDirectory", out string publishDirectory))
+            {
+                return null;
+            }
+            if (!mtpProject.GetRequiredMetadata(Log, "TargetPath", out string targetPath))
+            {
+                return null;
+            }
+
+            mtpProject.TryGetMetadata("Arguments", out string arguments);
+
+            string assemblyName = Path.GetFileName(targetPath);
+            string assemblyBaseName = assemblyName;
+            if (assemblyBaseName.EndsWith(".dll"))
+            {
+                assemblyBaseName = assemblyBaseName.Substring(0, assemblyBaseName.Length - 4);
+            }
+
+            // MTP test apps are self-hosting executables. Run the assembly directly with
+            // 'dotnet exec'. We pass only --results-directory (a built-in MTP option) so
+            // any reporter-produced artifacts land in the work item's working directory
+            // where Helix collects results from.
+            //
+            // We deliberately do NOT inject reporter flags here:
+            //   * --report-trx requires the Microsoft.Testing.Extensions.TrxReport
+            //     extension to be referenced; MTP apps without it would error out.
+            //     With MTP's --auto-reporters on by default, projects that DO reference
+            //     the TrxReport extension produce a TRX file automatically (default name
+            //     based on assembly + tfm + timestamp); arcade's TRXFormat reporter
+            //     globs *.trx so the auto-generated name is fine.
+            //   * --auto-reporters off is registered by xUnit v3's MTP integration and
+            //     is rejected as unknown by MSTest / NUnit / TUnit MTP apps.
+            // Users who need such options can set MTPAdditionalArguments (applies to
+            // every work item) or the per-project Arguments metadata on MTPProject.
+            string reporterArgs = "--results-directory .";
+
+            string command = $"{PathToDotnet} exec --roll-forward Major " +
+                $"--runtimeconfig {assemblyBaseName}.runtimeconfig.json " +
+                $"--depsfile {assemblyBaseName}.deps.json " +
+                $"{assemblyName} {reporterArgs}" +
+                (string.IsNullOrEmpty(MTPAdditionalArguments) ? "" : " " + MTPAdditionalArguments) +
+                (string.IsNullOrEmpty(arguments) ? "" : " " + arguments);
+
+            Log.LogMessage($"Creating MTP work item with properties Identity: {assemblyName}, PayloadDirectory: {publishDirectory}, Command: {command}");
+
+            TimeSpan timeout = TimeSpan.FromMinutes(5);
+            if (!string.IsNullOrEmpty(MTPWorkItemTimeout) && !TimeSpan.TryParse(MTPWorkItemTimeout, out timeout))
+            {
+                Log.LogWarning($"Invalid value \"{MTPWorkItemTimeout}\" provided for MTPWorkItemTimeout; falling back to default value of \"00:05:00\" (5 minutes)");
+                timeout = TimeSpan.FromMinutes(5);
+            }
+
+            var result = new Microsoft.Build.Utilities.TaskItem(assemblyName, new Dictionary<string, string>()
+            {
+                { "Identity", assemblyName },
+                { "PayloadDirectory", publishDirectory },
+                { "Command", command },
+                { "Timeout", timeout.ToString() },
+            });
+            mtpProject.CopyMetadataTo(result);
+            return result;
+        }
+    }
+}