diff --git a/.bot/README.md b/.bot/README.md new file mode 100644 index 0000000..de97dce --- /dev/null +++ b/.bot/README.md @@ -0,0 +1,10 @@ +# .bot Workspace + +This folder is reserved for local-only AI working material such as: + +- brainstorm notes +- draft implementation plans +- design alternatives +- temporary agent state + +Keep this folder out of source control. Move only finalized, non-confidential guidance into `AGENTS.md` or `.github/copilot-instructions.md`. diff --git a/.docfx/api/namespaces/Codebelt.Extensions.BenchmarkDotNet.Console.md b/.docfx/api/namespaces/Codebelt.Extensions.BenchmarkDotNet.Console.md index 73b7215..150ba58 100644 --- a/.docfx/api/namespaces/Codebelt.Extensions.BenchmarkDotNet.Console.md +++ b/.docfx/api/namespaces/Codebelt.Extensions.BenchmarkDotNet.Console.md @@ -1,7 +1,9 @@ ---- -uid: Codebelt.Extensions.BenchmarkDotNet.Console -summary: *content ---- -The `Codebelt.Extensions.BenchmarkDotNet.Console` namespace contains types that provide a structured and opinionated console-hosted execution model for `BenchmarkDotNet`. - -[!INCLUDE [availability-modern](../../includes/availability-modern.md)] +--- +uid: Codebelt.Extensions.BenchmarkDotNet.Console +summary: *content +--- +The `Codebelt.Extensions.BenchmarkDotNet.Console` namespace contains types that provide a structured and opinionated console-hosted execution model for `BenchmarkDotNet`. + +Use `BenchmarkProgram.Run` for synchronous benchmark hosts and `BenchmarkProgram.RunAsync` for asynchronous benchmark hosts; both entry points support the default `BenchmarkWorkspace` and custom `IBenchmarkWorkspace` implementations. + +[!INCLUDE [availability-modern](../../includes/availability-modern.md)] diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 27107cc..7ce5415 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -180,6 +180,28 @@ Internal classes and methods must be validated by exercising the public API that - Public entry points provide sufficient coverage of internal code paths. - The internal implementation exists solely as a helper or utility for public-facing functionality. +## 10. ExcludeFromCodeCoverage Prohibition + +**Do not use `ExcludeFromCodeCoverage` attribute on any code.** This includes: + +- Test classes or test methods +- Production code +- Configuration code +- Any other code path + +### Rationale + +- Excluding code from coverage hides gaps and creates false confidence in test completeness. +- If a code path cannot or should not be tested, refactor the code to eliminate that path rather than hiding it from metrics. +- Every executable line should be covered by tests or be genuinely unreachable (dead code to be removed). + +### Alternative Approaches + +- **Untestable code paths**: Refactor to separate concerns and eliminate the untestable path. +- **External dependencies**: Use test doubles (fakes, stubs, spies) instead of excluding from coverage. +- **Configuration-only code**: Move to configuration files or extract into testable methods. +- **Generated or third-party code**: These should not be in the primary codebase; use NuGet packages or dedicated vendor folders if necessary. + --- description: 'Writing Performance Tests in Codebelt.Extensions.BenchmarkDotNet' applyTo: "tuning/**, **/*Benchmark*.cs" diff --git a/.github/workflows/ci-pipeline.yml b/.github/workflows/ci-pipeline.yml index 5a00aaa..dd0cf8b 100644 --- a/.github/workflows/ci-pipeline.yml +++ b/.github/workflows/ci-pipeline.yml @@ -12,6 +12,10 @@ on: options: - Debug - Release + run_mac_tests: + type: boolean + description: Run the macOS test matrix despite the additional cost and runtime. + default: false permissions: contents: read @@ -21,6 +25,7 @@ jobs: name: initialize runs-on: ubuntu-24.04 outputs: + run-mac-tests: ${{ steps.vars.outputs.run-mac-tests }} run-privileged-jobs: ${{ steps.vars.outputs.run-privileged-jobs }} strong-name-key-filename: ${{ steps.vars.outputs.strong-name-key-filename }} build-switches: ${{ steps.vars.outputs.build-switches }} @@ -29,6 +34,12 @@ jobs: name: calculate workflow variables shell: bash run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" && "${{ inputs.run_mac_tests }}" == "true" ]]; then + echo "run-mac-tests=true" >> "$GITHUB_OUTPUT" + else + echo "run-mac-tests=false" >> "$GITHUB_OUTPUT" + fi + if [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]]; then echo "run-privileged-jobs=false" >> "$GITHUB_OUTPUT" echo "strong-name-key-filename=" >> "$GITHUB_OUTPUT" @@ -101,10 +112,72 @@ jobs: restore: true # we need to restore due to xUnitv3 download-pattern: build-${{ matrix.configuration }}-${{ matrix.arch }} + test_mac: + if: ${{ needs.init.outputs.run-mac-tests == 'true' }} + name: call-test-mac + needs: [init, build] + strategy: + fail-fast: false + matrix: + arch: [X64, ARM64] + configuration: [Debug, Release] + uses: codebeltnet/jobs-dotnet-test/.github/workflows/default.yml@v3 + with: + runs-on: ${{ matrix.arch == 'ARM64' && 'macos-26' || 'macos-26-intel' }} + configuration: ${{ matrix.configuration }} + build-switches: -p:SkipSignAssembly=true + restore: true + build: true + download-pattern: build-${{ matrix.configuration }}-${{ matrix.arch }} + + test_qualitygate: + if: ${{ always() }} + name: test-qualitygate + needs: [init, test_linux, test_windows, test_mac] + runs-on: ubuntu-24.04 + steps: + - name: Evaluate test results + shell: bash + env: + RUN_MAC_TESTS: ${{ needs.init.outputs.run-mac-tests }} + TEST_LINUX_RESULT: ${{ needs.test_linux.result }} + TEST_WINDOWS_RESULT: ${{ needs.test_windows.result }} + TEST_MAC_RESULT: ${{ needs.test_mac.result }} + run: | + require_success() { + local job_name="$1" + local job_result="$2" + + if [[ "$job_result" != "success" ]]; then + echo "::error::$job_name finished with '$job_result'." + exit 1 + fi + } + + require_success_or_skip() { + local job_name="$1" + local job_enabled="$2" + local job_result="$3" + + if [[ "$job_enabled" == "true" ]]; then + require_success "$job_name" "$job_result" + return + fi + + if [[ "$job_result" != "success" && "$job_result" != "skipped" ]]; then + echo "::error::$job_name finished with '$job_result' while disabled." + exit 1 + fi + } + + require_success "test_linux" "$TEST_LINUX_RESULT" + require_success "test_windows" "$TEST_WINDOWS_RESULT" + require_success_or_skip "test_mac" "$RUN_MAC_TESTS" "$TEST_MAC_RESULT" + sonarcloud: - if: ${{ needs.init.outputs.run-privileged-jobs == 'true' }} + if: ${{always() && needs.init.outputs.run-privileged-jobs == 'true' && needs.build.result == 'success' && needs.test_qualitygate.result == 'success'}} name: call-sonarcloud - needs: [init, build, test_linux, test_windows] + needs: [init, build, test_qualitygate] uses: codebeltnet/jobs-sonarcloud/.github/workflows/default.yml@v3 with: organization: geekle @@ -113,18 +186,18 @@ jobs: secrets: inherit codecov: - if: ${{ needs.init.outputs.run-privileged-jobs == 'true' }} + if: ${{always() && needs.init.outputs.run-privileged-jobs == 'true' && needs.build.result == 'success' && needs.test_qualitygate.result == 'success'}} name: call-codecov - needs: [init, build, test_linux, test_windows] + needs: [init, build, test_qualitygate] uses: codebeltnet/jobs-codecov/.github/workflows/default.yml@v1 with: repository: codebeltnet/benchmarkdotnet secrets: inherit codeql: - if: ${{ needs.init.outputs.run-privileged-jobs == 'true' }} + if: ${{always() && needs.init.outputs.run-privileged-jobs == 'true' && needs.build.result == 'success' && needs.test_qualitygate.result == 'success'}} name: call-codeql - needs: [init, build, test_linux, test_windows] + needs: [init, build, test_qualitygate] uses: codebeltnet/jobs-codeql/.github/workflows/default.yml@v3 permissions: security-events: write @@ -132,7 +205,7 @@ jobs: deploy: if: github.event_name != 'pull_request' name: call-nuget - needs: [build, pack, test_linux, test_windows, sonarcloud, codecov, codeql] + needs: [build, pack, test_qualitygate, sonarcloud, codecov, codeql] uses: codebeltnet/jobs-nuget-push/.github/workflows/default.yml@v3 with: version: ${{ needs.build.outputs.version }} diff --git a/.gitignore b/.gitignore index b2d91bd..2c4c6f0 100644 --- a/.gitignore +++ b/.gitignore @@ -374,4 +374,8 @@ FodyWeavers.xsd *.code-workspace # Strong-Name Key -*.snk \ No newline at end of file +*.snk + +# Bot workspace (local-only AI agent ideation, PRDs, and agentic loop state) +.bot/* +!.bot/README.md \ No newline at end of file diff --git a/.nuget/Codebelt.Extensions.BenchmarkDotNet.Console/PackageReleaseNotes.txt b/.nuget/Codebelt.Extensions.BenchmarkDotNet.Console/PackageReleaseNotes.txt index 0e4cb5b..4c952aa 100644 --- a/.nuget/Codebelt.Extensions.BenchmarkDotNet.Console/PackageReleaseNotes.txt +++ b/.nuget/Codebelt.Extensions.BenchmarkDotNet.Console/PackageReleaseNotes.txt @@ -1,9 +1,18 @@ -Version: 1.2.7 +Version: 1.3.0 Availability: .NET 10 and .NET 9 - -# ALM -- CHANGED Dependencies have been upgraded to the latest compatible versions for all supported target frameworks (TFMs) - + +# ALM +- CHANGED Microsoft.NET.Test.Sdk has been upgraded from version 18.5.1 to 18.6.0 for improved test framework reliability + +# Improvements +- EXTENDED BenchmarkProgram class in the Codebelt.Extensions.BenchmarkDotNet.Console namespace to include RunAsync equivalent methods of the existing Run methods for improved asynchronous hosting scenarios + +Version: 1.2.7 +Availability: .NET 10 and .NET 9 + +# ALM +- CHANGED Dependencies have been upgraded to the latest compatible versions for all supported target frameworks (TFMs) + Version: 1.2.6 Availability: .NET 10 and .NET 9 diff --git a/.nuget/Codebelt.Extensions.BenchmarkDotNet/PackageReleaseNotes.txt b/.nuget/Codebelt.Extensions.BenchmarkDotNet/PackageReleaseNotes.txt index 0c2dce0..2bb3972 100644 --- a/.nuget/Codebelt.Extensions.BenchmarkDotNet/PackageReleaseNotes.txt +++ b/.nuget/Codebelt.Extensions.BenchmarkDotNet/PackageReleaseNotes.txt @@ -1,9 +1,15 @@ -Version: 1.2.7 +Version: 1.3.0 Availability: .NET 10 and .NET 9 - -# ALM -- CHANGED Dependencies have been upgraded to the latest compatible versions for all supported target frameworks (TFMs) - + +# ALM +- CHANGED Dependencies have been upgraded to the latest compatible versions for all supported target frameworks (TFMs) + +Version: 1.2.7 +Availability: .NET 10 and .NET 9 + +# ALM +- CHANGED Dependencies have been upgraded to the latest compatible versions for all supported target frameworks (TFMs) + Version: 1.2.6 Availability: .NET 10 and .NET 9 diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..a9e7a11 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,66 @@ +# Agent Instructions for Codebelt.Extensions.BenchmarkDotNet + +This document provides guidance for AI agents working in this repository. + +## Project Overview + +Codebelt.Extensions.BenchmarkDotNet is a .NET library that provides a uniform, opinionated, and extensible way of working with BenchmarkDotNet. The solution targets .NET 10.0 and .NET 9.0. + +## Coding Standards + +- **Text encoding:** UTF-8 for text files (enforced via `.editorconfig`) +- **Template rewrites:** Preserve UTF-8 explicitly when scripts or tools rewrite text files; avoid locale-dependent encoding defaults +- **Namespaces:** File-scoped namespaces are required (enforced via `.editorconfig`) +- **Top-level statements:** Not allowed (enforced via `.editorconfig`) +- **Language version:** Always use the latest C# features (`LangVersion=latest`) +- **Nullable:** Enable nullable reference types in all new code +- **XML documentation:** All public APIs must have XML documentation comments +- **Testing:** Use xUnit v3 with Codebelt.Extensions.Xunit.App base classes + +## Project Structure + +- `src/` — Production source code +- `test/` — Unit and integration tests (project names end with `Tests`) +- `tuning/` — Benchmark projects and benchmark source code +- `tooling/` — Solution-level executable tooling such as the benchmark runner host +- `reports/` — Benchmark reports and tuning output produced by tooling +- `.nuget/` — Per-package NuGet metadata (icon, README, release notes) +- `.docfx/` — DocFX documentation configuration +- `.github/` — CI/CD workflows, contributing guidelines, Copilot instructions + +## Test Conventions + +- Test project names must end with `Tests` (e.g. `{PROJECT_NAME}.Tests`) +- Test classes should inherit from the appropriate base class in `Codebelt.Extensions.Xunit` +- Use `Microsoft.Testing.Platform` as the test runner (`UseMicrosoftTestingPlatformRunner=true`) +- All tests are executable (`OutputType=Exe`) + +## Build & CI + +- Centralized package versions via `Directory.Packages.props` +- Resolve new or updated `Directory.Packages.props` versions from NuGet.org and keep them on the latest stable listed releases +- Centralized build configuration via `Directory.Build.props` +- MinVer for semantic versioning from Git tags +- Strong-name signing is enabled in CI environments (`CI=true`) +- Keep `.github/dependabot.yml` enabled at the repo root so central NuGet package management stays current + +## .bot/ Folder + +If a `.bot/` folder exists at the root, it contains **confidential, local-only** working material for AI agents — product requirement documents (PRDs), design proposals, agentic loop state, and brainstorming outputs. This folder is gitignored and never committed. + +When starting creative or design work (new features, architecture decisions, PRD drafts), use the [brainstorming skill](https://skills.sh/obra/superpowers/brainstorming) and save outputs to `.bot/`. Only move finalized, non-confidential instructions into `AGENTS.md` or `.github/copilot-instructions.md`. + +## Git Operations Safeguards + +Agents must never automatically commit code changes or push to remote repositories. Both actions require explicit user approval: + +- **Commits**: Always request confirmation from the user before staging and committing code. Present a clear summary of the changes and wait for approval before executing the commit. +- **Remote Operations**: Do not push, pull, fetch, or interact with `origin` or any remote repository without explicit user instruction. These operations modify repository history and can cause data loss if performed unexpectedly. + +**Rationale:** Automatic commits can clutter history with incomplete work, temporary debugging code, or unintended changes. Unexpected remote operations risk overwriting or losing commits on shared branches. Always require explicit user approval before performing these actions. + +## Official Documentation + +- Public API conventions belong in `.docfx/api/namespaces/` and should be treated as the official documentation source for library behavior and naming vocabulary. +- When adding or renaming public APIs, update the relevant namespace page in `.docfx/api/namespaces/` if the change introduces or clarifies a convention. +- Keep internal reasoning, exploratory notes, and agent discussion out of DocFX pages; summarize only stable public guidance. diff --git a/CHANGELOG.md b/CHANGELOG.md index 559dac0..cc6cfab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,27 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), For more details, please refer to `PackageReleaseNotes.txt` on a per assembly basis in the `.nuget` folder. +## [1.3.0] - 2026-06-05 + +This is a minor release focused on code organization improvements, test coverage expansion, CI/CD pipeline hardening, and comprehensive agent guidance. The release refactors target framework parsing logic for better testability, expands the test suite for core workspace classes, extends CI/CD support to macOS environments, and establishes clear conventions and standards for AI agent collaboration in the repository. + +### Added + +- Comprehensive agent guidance document (AGENTS.md) for AI agents working in the repository, covering project overview, coding standards, project structure, test conventions, and CI/CD practices, +- Local-only `.bot/` folder for agent ideation, product requirement documents, design proposals, and agentic loop state, +- Code coverage standards to agent guidance establishing that `ExcludeFromCodeCoverage` attribute must not be used; unmeasurable code paths should be refactored or removed, +- macOS test matrix to CI/CD pipeline with conditional execution for platform-specific testing, +- Extracted `ParseTargetFrameworkMoniker` method in `BenchmarkWorkspaceOptions` class for improved code organization and testability, +- Expanded test coverage for `BenchmarkWorkspaceOptions`, `BenchmarkProgram`, and `ServiceCollectionExtensions` classes. + +### Changed + +- `BenchmarkProgram` class in the Codebelt.Extensions.BenchmarkDotNet.Console namespace was significantly enhanced with comprehensive async support including `RunAsync()` method overloads (both generic and non-generic variants) for asynchronous benchmark execution, +- Refactored target framework moniker parsing logic in `BenchmarkWorkspaceOptions` for better separation of concerns and improved testability, +- Upgraded `Microsoft.NET.Test.Sdk` from 18.5.1 to 18.6.0, +- Improved CI/CD pipeline configuration with more robust conditional execution logic, +- Updated documentation API references in Console namespace. + ## [1.2.7] - 2026-05-22 This is a service update that focuses on package dependencies. @@ -87,7 +108,10 @@ This is the initial stable release of the `Codebelt.Extensions.BenchmarkDotNet` - ADDED `BenchmarkProgram` class in the Codebelt.Extensions.BenchmarkDotNet.Console namespace that provides the main entry point for hosting and running benchmarks using BenchmarkDotNet, - ADDED `BenchmarkWorker` class in the Codebelt.Extensions.BenchmarkDotNet.Console namespace that is responsible for executing benchmarks within the console host. -[Unreleased]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.5...HEAD +[Unreleased]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.3.0...HEAD +[1.3.0]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.7...v1.3.0 +[1.2.7]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.6...v1.2.7 +[1.2.6]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.5...v1.2.6 [1.2.5]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.4...v1.2.5 [1.2.4]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.3...v1.2.4 [1.2.3]: https://github.com/codebeltnet/benchmarkdotnet/compare/v1.2.2...v1.2.3 diff --git a/Codebelt.Extensions.BenchmarkDotNet.slnx b/Codebelt.Extensions.BenchmarkDotNet.slnx index bc97d92..3486db2 100644 --- a/Codebelt.Extensions.BenchmarkDotNet.slnx +++ b/Codebelt.Extensions.BenchmarkDotNet.slnx @@ -4,7 +4,9 @@ + + diff --git a/Directory.Packages.props b/Directory.Packages.props index 6fbea36..4de402c 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -5,10 +5,10 @@ - - - - + + + + diff --git a/src/Codebelt.Extensions.BenchmarkDotNet.Console/BenchmarkProgram.cs b/src/Codebelt.Extensions.BenchmarkDotNet.Console/BenchmarkProgram.cs index 91ea3c0..f10d006 100644 --- a/src/Codebelt.Extensions.BenchmarkDotNet.Console/BenchmarkProgram.cs +++ b/src/Codebelt.Extensions.BenchmarkDotNet.Console/BenchmarkProgram.cs @@ -92,22 +92,76 @@ public static void Run(string[] args, Action /// This method configures the host builder with the necessary services, builds the host, and runs it to execute benchmarks. /// - public static void Run(string[] args, Action serviceConfigurator = null, Action setup = null) where TWorkspace : class, IBenchmarkWorkspace - { - var builder = CreateHostBuilder(args); - - builder.Services.Configure(o => o.SuppressStatusMessages = !IsDebugBuild); - builder.Services.AddSingleton(new BenchmarkContext(args)); - builder.Services.AddBenchmarkWorkspace(setup); - serviceConfigurator?.Invoke(builder.Services); - - using var host = builder.Build(); - host.Run(); - } - - /// - /// Runs the actual benchmarks as envisioned by BenchmarkDotNet. - /// + public static void Run(string[] args, Action serviceConfigurator = null, Action setup = null) where TWorkspace : class, IBenchmarkWorkspace + { + using var host = BuildHost(args, serviceConfigurator, setup); + host.Run(); + } + + /// + /// Runs benchmarks asynchronously using the default implementation. + /// + /// The command-line arguments passed to the application. + /// The which may be configured. + /// A task that represents the asynchronous benchmark host operation. + /// + /// This method configures the host builder with the necessary services, builds the host, and runs it asynchronously to execute benchmarks. + /// + public static Task RunAsync(string[] args, Action setup = null) + { + return RunAsync(args, null, setup); + } + + /// + /// Runs benchmarks asynchronously using the default implementation. + /// + /// The command-line arguments passed to the application. + /// The delegate that will be invoked to configure additional services in the . + /// The which may be configured. + /// A task that represents the asynchronous benchmark host operation. + /// + /// This method configures the host builder with the necessary services, builds the host, and runs it asynchronously to execute benchmarks. + /// + public static Task RunAsync(string[] args, Action serviceConfigurator = null, Action setup = null) + { + return RunAsync(args, serviceConfigurator, setup); + } + + /// + /// Runs benchmarks asynchronously using a custom implementation of . + /// + /// The type of the workspace that implements . + /// The command-line arguments passed to the application. + /// The which may be configured. + /// A task that represents the asynchronous benchmark host operation. + /// + /// This method configures the host builder with the necessary services, builds the host, and runs it asynchronously to execute benchmarks. + /// + public static Task RunAsync(string[] args, Action setup = null) where TWorkspace : class, IBenchmarkWorkspace + { + return RunAsync(args, null, setup); + } + + /// + /// Runs benchmarks asynchronously using a custom implementation of . + /// + /// The type of the workspace that implements . + /// The command-line arguments passed to the application. + /// The delegate that will be invoked to configure additional services in the . + /// The which may be configured. + /// A task that represents the asynchronous benchmark host operation. + /// + /// This method configures the host builder with the necessary services, builds the host, and runs it asynchronously to execute benchmarks. + /// + public static async Task RunAsync(string[] args, Action serviceConfigurator = null, Action setup = null) where TWorkspace : class, IBenchmarkWorkspace + { + using var host = BuildHost(args, serviceConfigurator, setup); + await host.RunAsync().ConfigureAwait(false); + } + + /// + /// Runs the actual benchmarks as envisioned by BenchmarkDotNet. + /// /// The service provider. /// The cancellation token. /// A completed task when benchmark execution has finished. @@ -133,12 +187,24 @@ public override Task RunAsync(IServiceProvider serviceProvider, CancellationToke workspace.PostProcessArtifacts(); } - return Task.CompletedTask; - } - - private static void ConfigureBenchmarkDotNetFiltersForExistingReports(BenchmarkWorkspaceOptions options, Assembly[] assemblies) - { - var benchmarkTypes = assemblies + return Task.CompletedTask; + } + + private static IHost BuildHost(string[] args, Action serviceConfigurator, Action setup) where TWorkspace : class, IBenchmarkWorkspace + { + var builder = CreateHostBuilder(args); + + builder.Services.Configure(o => o.SuppressStatusMessages = !IsDebugBuild); + builder.Services.AddSingleton(new BenchmarkContext(args)); + builder.Services.AddBenchmarkWorkspace(setup); + serviceConfigurator?.Invoke(builder.Services); + + return builder.Build(); + } + + private static void ConfigureBenchmarkDotNetFiltersForExistingReports(BenchmarkWorkspaceOptions options, Assembly[] assemblies) + { + var benchmarkTypes = assemblies .SelectMany(a => a.GetTypes().Where(t => t.Name.EndsWith("Benchmark", StringComparison.Ordinal))) .ToList(); diff --git a/src/Codebelt.Extensions.BenchmarkDotNet/BenchmarkWorkspaceOptions.cs b/src/Codebelt.Extensions.BenchmarkDotNet/BenchmarkWorkspaceOptions.cs index e4f3201..8dfec54 100644 --- a/src/Codebelt.Extensions.BenchmarkDotNet/BenchmarkWorkspaceOptions.cs +++ b/src/Codebelt.Extensions.BenchmarkDotNet/BenchmarkWorkspaceOptions.cs @@ -9,7 +9,6 @@ using Cuemon.Configuration; using Perfolizer.Horology; using System; -using System.Globalization; using System.IO; using System.Linq; using System.Reflection; @@ -247,49 +246,72 @@ private static Job GetDefaultConfiguredJob() .DontEnforcePowerPlan(); // make sure BDN does not try to enforce High Performance power plan on Windows } - private static string ResolveCurrentTfm() + private static string ParseTargetFrameworkMoniker(string frameworkName) { + if (string.IsNullOrEmpty(frameworkName)) + { + return null; + } + try { - var entry = Assembly.GetEntryAssembly(); - var tfa = entry?.GetCustomAttribute(); - if (!string.IsNullOrEmpty(tfa?.FrameworkName)) + var fn = new FrameworkName(frameworkName); + var v = fn.Version; + + // .NET Framework → net11, net20, net35, net40, net403, net45, net451, ..., net48, net481 + if (fn.Identifier.Equals(".NETFramework", StringComparison.OrdinalIgnoreCase)) { - var fn = new FrameworkName(tfa.FrameworkName); - var v = fn.Version; + // Base: net4{minor} or net{major}{minor} for older ones + var tfm = $"net{v.Major}{v.Minor}"; - // .NET Framework → net11, net20, net35, net40, net403, net45, net451, ..., net48, net481 - if (fn.Identifier.Equals(".NETFramework", StringComparison.OrdinalIgnoreCase)) + // For 4.x: append Build when present (4.0.3 → net403, 4.5.1 → net451, 4.8.1 → net481) + if (v.Major >= 4 && v.Build > 0) { - // Base: net4{minor} or net{major}{minor} for older ones - var tfm = $"net{v.Major}{v.Minor}"; + tfm += v.Build; + } - // For 4.x: append Build when present (4.0.3 → net403, 4.5.1 → net451, 4.8.1 → net481) - if (v.Major >= 4 && v.Build > 0) - { - tfm += v.Build; - } + return tfm; + } - return tfm; - } + // .NET Standard → netstandard1.0–2.1 + if (fn.Identifier.Equals(".NETStandard", StringComparison.OrdinalIgnoreCase)) + { + return $"netstandard{v.Major}.{v.Minor}"; + } - // .NET Standard → netstandard1.0–2.1 - if (fn.Identifier.Equals(".NETStandard", StringComparison.OrdinalIgnoreCase)) + // .NET Core / .NET (CoreApp) + if (fn.Identifier.Equals(".NETCoreApp", StringComparison.OrdinalIgnoreCase)) + { + // .NET Core 1.0–3.1 use netcoreappX.Y + if (v.Major <= 3) { - return $"netstandard{v.Major}.{v.Minor}"; + return $"netcoreapp{v.Major}.{v.Minor}"; } - // .NET Core / .NET (CoreApp) - if (fn.Identifier.Equals(".NETCoreApp", StringComparison.OrdinalIgnoreCase)) + // .NET 5+ uses netX.Y + return $"net{v.Major}.{v.Minor}"; + } + } + catch + { + // ignore invalid framework names + } + + return null; + } + + private static string ResolveCurrentTfm() + { + try + { + var entry = Assembly.GetEntryAssembly(); + var tfa = entry?.GetCustomAttribute(); + if (!string.IsNullOrEmpty(tfa?.FrameworkName)) + { + var result = ParseTargetFrameworkMoniker(tfa.FrameworkName); + if (result != null) { - // .NET Core 1.0–3.1 use netcoreappX.Y - if (v.Major <= 3) - { - return $"netcoreapp{v.Major}.{v.Minor}"; - } - - // .NET 5+ uses netX.Y - return $"net{v.Major}.{v.Minor}"; + return result; } } } diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/BenchmarkProgramTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/BenchmarkProgramTest.cs new file mode 100644 index 0000000..36c23ec --- /dev/null +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/BenchmarkProgramTest.cs @@ -0,0 +1,455 @@ +using BenchmarkDotNet.Configs; +using Codebelt.Extensions.Xunit; +using Microsoft.Extensions.DependencyInjection; +using System; +using System.IO; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using Xunit; + +namespace Codebelt.Extensions.BenchmarkDotNet.Console; + +/// +/// Functional tests for that exercise the full hosted-application lifecycle +/// including , , +/// and the benchmark-filter pipeline (SkipBenchmarksWithReports). +/// +public class BenchmarkProgramTest : Test +{ + public BenchmarkProgramTest(ITestOutputHelper output) : base(output) + { + } + + // --------------------------------------------------------------------------- + // RunAsync – direct instantiation tests (avoid starting the full host) + // --------------------------------------------------------------------------- + + [Fact] + public async Task RunAsync_ShouldComplete_WhenWorkspaceReturnsNoAssemblies_AndArgsAreEmpty() + { + // Arrange + var options = new BenchmarkWorkspaceOptions(); + var workspace = new EmptyWorkspace(); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act – empty assemblies + empty args → empty foreach → no BenchmarkRunner call + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + Assert.True(workspace.PostProcessArtifactsCalled, "PostProcessArtifacts should always be called in the finally block."); + + TestOutput.WriteLine("RunAsync completed cleanly with empty workspace and no args."); + } + + [Fact] + public async Task RunAsync_ShouldUseBenchmarkSwitcher_WhenArgsAreNonEmpty() + { + // Arrange + var options = new BenchmarkWorkspaceOptions(); + var workspace = new EmptyWorkspace(); + + // "--list flat" asks BenchmarkSwitcher to list (not run) benchmarks; safe with an empty assembly list. + var context = new BenchmarkContext(new[] { "--list", "flat" }); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act – BenchmarkSwitcher path (context.Args.Length > 0) + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + Assert.True(workspace.PostProcessArtifactsCalled); + + TestOutput.WriteLine("RunAsync completed cleanly via BenchmarkSwitcher path."); + } + + [Fact] + public async Task RunAsync_ShouldSkipFiltering_WhenSkipBenchmarksWithReportsIsFalse() + { + // Arrange + var options = new BenchmarkWorkspaceOptions { SkipBenchmarksWithReports = false }; + var workspace = new EmptyWorkspace(); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert – no filtering attempted, completes normally + Assert.Null(exception); + } + + [Fact] + public async Task RunAsync_ShouldApplyFilters_WhenSkipBenchmarksWithReportsIsTrue_AndTuningDirDoesNotExist() + { + // Arrange – tuning dir does not exist → ApplyReportFilters early-returns + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var config = ManualConfig.CreateEmpty().WithArtifactsPath(Path.Combine(tempPath, "artifacts")); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning", + SkipBenchmarksWithReports = true + }; + + // Return the functional-test assembly so benchmarkTypes is populated with KnownBenchmark + var workspace = new AssemblyWorkspace(typeof(BenchmarkProgramTest).Assembly); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act – tuning dir absent → ApplyReportFilters returns config unchanged + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } + + [Fact] + public async Task RunAsync_ShouldApplyFilters_WhenSkipBenchmarksWithReportsIsTrue_AndMatchingReportExists() + { + // Arrange – a report file whose name contains "KnownBenchmark" exists in the tuning dir. + // KnownBenchmark (defined below) is in this assembly so it will be found by benchmarkTypes. + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + Directory.CreateDirectory(tuningDir); + + // Report file whose name encodes the KnownBenchmark type + File.WriteAllText( + Path.Combine(tuningDir, "Codebelt.Extensions.BenchmarkDotNet.Console.KnownBenchmark-report-github.md"), + "# benchmark report"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning", + SkipBenchmarksWithReports = true + }; + + var workspace = new AssemblyWorkspace(typeof(BenchmarkProgramTest).Assembly); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act – filter IS applied for KnownBenchmark (matchingType != null path) + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + + TestOutput.WriteLine("Filter applied for KnownBenchmark report."); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } + + [Fact] + public async Task RunAsync_ShouldSkipReport_WhenFilenameHasLeadingDash() + { + // Arrange – a file whose name starts with "-" causes Split('-').FirstOrDefault() == "" + // which exercises the first 'return null' branch in FindMatchingBenchmarkType. + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + Directory.CreateDirectory(tuningDir); + + File.WriteAllText(Path.Combine(tuningDir, "-leading-dash-report.md"), "edge case"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning", + SkipBenchmarksWithReports = true + }; + + var workspace = new AssemblyWorkspace(typeof(BenchmarkProgramTest).Assembly); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert – no filter applied (FindMatchingBenchmarkType returned null), no exception + Assert.Null(exception); + + TestOutput.WriteLine("Leading-dash filename correctly produced null match."); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } + + [Fact] + public async Task RunAsync_ShouldSkipReport_WhenFilenameHasEmptyTypePart() + { + // Arrange – ".-something.md" → potentialTypeFullName = "." → Split('.').LastOrDefault() == "" + // which exercises the second 'return null' branch in FindMatchingBenchmarkType. + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + Directory.CreateDirectory(tuningDir); + + File.WriteAllText(Path.Combine(tuningDir, ".-empty-type-part.md"), "edge case"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning", + SkipBenchmarksWithReports = true + }; + + var workspace = new AssemblyWorkspace(typeof(BenchmarkProgramTest).Assembly); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + + TestOutput.WriteLine("Dot-only type-name segment correctly produced null match."); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } + + [Fact] + public async Task RunAsync_ShouldSkipReport_WhenFilenameDoesNotMatchAnyBenchmarkType() + { + // Arrange – well-formed filename but no matching type in the assembly exercises + // the 'return null' path where FirstOrDefault returns null from benchmarkTypes. + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + Directory.CreateDirectory(tuningDir); + + File.WriteAllText( + Path.Combine(tuningDir, "SomeNamespace.NonExistentBenchmark-report.md"), + "no matching type"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning", + SkipBenchmarksWithReports = true + }; + + var workspace = new AssemblyWorkspace(typeof(BenchmarkProgramTest).Assembly); + var context = new BenchmarkContext(Array.Empty()); + + using var sp = BuildServiceProvider(options, workspace, context); + var program = new BenchmarkProgram(); + + // Act + var exception = await Record.ExceptionAsync(() => program.RunAsync(sp, CancellationToken.None)); + + // Assert + Assert.Null(exception); + + TestOutput.WriteLine("No matching benchmark type found – report correctly skipped."); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } + + // --------------------------------------------------------------------------- + // BenchmarkProgram.Run – full hosted-application lifecycle tests + // --------------------------------------------------------------------------- + + [Fact] + public void Run_WithFakeWorkspace_ShouldCompleteWithoutException() + { + // Exercises the static Run(string[], Action) overload (line 82) + // and the full 4-param generic Run (lines 97-105), which starts the host, calls RunAsync, + // and stops the host after RunAsync returns. + // Using named parameter 'setup' disambiguates from the 3-param overload. + var exception = Record.Exception(() => + BenchmarkProgram.Run(Array.Empty(), setup: (Action)null)); + + Assert.Null(exception); + TestOutput.WriteLine("Run completed without exception."); + } + + [Fact] + public void Run_WithFakeWorkspace_AndServiceConfigurator_ShouldInvokeConfigurator() + { + // Exercises Run(string[], Action, Action) + // and verifies that the serviceConfigurator delegate is invoked (line 102). + var configuratorWasCalled = false; + + var exception = Record.Exception(() => + BenchmarkProgram.Run( + Array.Empty(), + serviceConfigurator: services => + { + configuratorWasCalled = true; + services.AddSingleton(_ => null!); // harmless extra registration + })); + + Assert.Null(exception); + Assert.True(configuratorWasCalled, "serviceConfigurator should have been invoked during host setup."); + TestOutput.WriteLine("serviceConfigurator was invoked as expected."); + } + + [Fact] + public void Run_WithFakeWorkspace_AndNonEmptyArgs_ShouldUseBenchmarkSwitcherPath() + { + // Passing "--list flat" exercises the BenchmarkSwitcher branch in ExecuteBenchmarks. + // Using named parameter 'setup' disambiguates from the 3-param overload. + var exception = Record.Exception(() => + BenchmarkProgram.Run(new[] { "--list", "flat" }, setup: (Action)null)); + + Assert.Null(exception); + TestOutput.WriteLine("Run with '--list flat' completed (BenchmarkSwitcher path)."); + } + + // --------------------------------------------------------------------------- + // BenchmarkProgram.RunAsync – full hosted-application lifecycle tests + // --------------------------------------------------------------------------- + + [Fact] + public async Task RunAsync_WithFakeWorkspace_ShouldCompleteWithoutException() + { + var exception = await Record.ExceptionAsync(() => + BenchmarkProgram.RunAsync(Array.Empty(), setup: (Action)null)); + + Assert.Null(exception); + TestOutput.WriteLine("RunAsync completed without exception."); + } + + [Fact] + public async Task RunAsync_WithFakeWorkspace_AndServiceConfigurator_ShouldInvokeConfigurator() + { + var configuratorWasCalled = false; + + var exception = await Record.ExceptionAsync(() => + BenchmarkProgram.RunAsync( + Array.Empty(), + serviceConfigurator: services => + { + configuratorWasCalled = true; + services.AddSingleton(_ => null!); // harmless extra registration + })); + + Assert.Null(exception); + Assert.True(configuratorWasCalled, "serviceConfigurator should have been invoked during host setup."); + TestOutput.WriteLine("async serviceConfigurator was invoked as expected."); + } + + [Fact] + public async Task RunAsync_WithFakeWorkspace_AndNonEmptyArgs_ShouldUseBenchmarkSwitcherPath() + { + var exception = await Record.ExceptionAsync(() => + BenchmarkProgram.RunAsync(new[] { "--list", "flat" }, setup: (Action)null)); + + Assert.Null(exception); + TestOutput.WriteLine("RunAsync with '--list flat' completed (BenchmarkSwitcher path)."); + } + + // --------------------------------------------------------------------------- + // Helper: build a minimal service provider for RunAsync tests + // --------------------------------------------------------------------------- + + private static ServiceProvider BuildServiceProvider( + BenchmarkWorkspaceOptions options, + IBenchmarkWorkspace workspace, + BenchmarkContext context) + { + return new ServiceCollection() + .AddSingleton(workspace) + .AddSingleton(options) + .AddSingleton(context) + .BuildServiceProvider(); + } + + // --------------------------------------------------------------------------- + // Test doubles + // --------------------------------------------------------------------------- + + /// + /// A workspace that returns an empty assembly array and tracks whether + /// was called. + /// + private sealed class EmptyWorkspace : IBenchmarkWorkspace + { + public bool PostProcessArtifactsCalled { get; private set; } + + public Assembly[] LoadBenchmarkAssemblies() => Array.Empty(); + + public void PostProcessArtifacts() => PostProcessArtifactsCalled = true; + } + + /// + /// A workspace that returns a specific assembly so that benchmark-type discovery + /// (filtering by types whose names end with "Benchmark") finds . + /// + private sealed class AssemblyWorkspace : IBenchmarkWorkspace + { + private readonly Assembly _assembly; + + public AssemblyWorkspace(Assembly assembly) => _assembly = assembly; + + public Assembly[] LoadBenchmarkAssemblies() => new[] { _assembly }; + + public void PostProcessArtifacts() { } + } +} + +/// +/// A placeholder type whose name ends with "Benchmark" so that +/// 's filter logic can discover it via +/// t.Name.EndsWith("Benchmark"). +/// +internal sealed class KnownBenchmark { } diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests.csproj b/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests.csproj new file mode 100644 index 0000000..eaab558 --- /dev/null +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.Console.FunctionalTests.csproj @@ -0,0 +1,11 @@ + + + + Codebelt.Extensions.BenchmarkDotNet.Console + + + + + + + diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Console.Tests/BenchmarkProgramTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.Console.Tests/BenchmarkProgramTest.cs index 0740d64..67a8964 100644 --- a/test/Codebelt.Extensions.BenchmarkDotNet.Console.Tests/BenchmarkProgramTest.cs +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Console.Tests/BenchmarkProgramTest.cs @@ -1,8 +1,9 @@ -using System; -using Codebelt.Extensions.Xunit; -using System.Reflection; -using System.Linq; -using Xunit; +using System; +using Codebelt.Extensions.Xunit; +using System.Reflection; +using System.Linq; +using System.Threading.Tasks; +using Xunit; namespace Codebelt.Extensions.BenchmarkDotNet.Console; @@ -328,10 +329,10 @@ public void Run_GenericMethod_ShouldExist() } [Fact] - public void Run_GenericMethod_ShouldHaveCorrectConstraints() - { - // Act - var method = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + public void Run_GenericMethod_ShouldHaveCorrectConstraints() + { + // Act + var method = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) .FirstOrDefault(m => m.Name == "Run" && m.IsGenericMethodDefinition && @@ -346,13 +347,80 @@ public void Run_GenericMethod_ShouldHaveCorrectConstraints() var constraints = typeParameter.GetGenericParameterConstraints(); Assert.Contains(constraints, c => c == typeof(IBenchmarkWorkspace)); Assert.True(typeParameter.GenericParameterAttributes.HasFlag(GenericParameterAttributes.ReferenceTypeConstraint)); - - TestOutput.WriteLine($"Generic type parameter '{typeParameter.Name}' has correct constraints: class, IBenchmarkWorkspace"); - } - - [Fact] - public void StaticProperties_ShouldBeReadOnly() - { + + TestOutput.WriteLine($"Generic type parameter '{typeParameter.Name}' has correct constraints: class, IBenchmarkWorkspace"); + } + + [Fact] + public void RunAsync_MethodWithDefaultWorkspace_ShouldExist() + { + // Act + var method = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + m.Name == "RunAsync" && + !m.IsGenericMethodDefinition && + m.GetParameters().Length == 2 && + m.GetParameters()[0].ParameterType == typeof(string[]) && + m.GetParameters()[1].ParameterType == typeof(Action)); + + // Assert + Assert.NotNull(method); + Assert.True(method.IsStatic); + Assert.True(method.IsPublic); + Assert.Equal(typeof(Task), method.ReturnType); + + TestOutput.WriteLine($"Found RunAsync method with default workspace: {method}"); + } + + [Fact] + public void RunAsync_GenericMethod_ShouldExist() + { + // Act + var method = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + m.Name == "RunAsync" && + m.IsGenericMethodDefinition && + m.GetParameters().Length == 2); + + // Assert + Assert.NotNull(method); + Assert.True(method.IsStatic); + Assert.True(method.IsPublic); + Assert.True(method.IsGenericMethodDefinition); + Assert.Equal(typeof(Task), method.ReturnType); + + var genericArguments = method.GetGenericArguments(); + Assert.Single(genericArguments); + + TestOutput.WriteLine($"Found generic RunAsync method: {method}"); + } + + [Fact] + public void RunAsync_GenericMethod_ShouldHaveCorrectConstraints() + { + // Act + var method = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + m.Name == "RunAsync" && + m.IsGenericMethodDefinition && + m.GetParameters().Length == 2); + + Assert.NotNull(method); + + var genericArguments = method.GetGenericArguments(); + var typeParameter = genericArguments[0]; + + // Assert + var constraints = typeParameter.GetGenericParameterConstraints(); + Assert.Contains(constraints, c => c == typeof(IBenchmarkWorkspace)); + Assert.True(typeParameter.GenericParameterAttributes.HasFlag(GenericParameterAttributes.ReferenceTypeConstraint)); + + TestOutput.WriteLine($"Generic type parameter '{typeParameter.Name}' has correct constraints: class, IBenchmarkWorkspace"); + } + + [Fact] + public void StaticProperties_ShouldBeReadOnly() + { // Act var buildConfigProperty = typeof(BenchmarkProgram).GetProperty("BuildConfiguration"); var isDebugBuildProperty = typeof(BenchmarkProgram).GetProperty("IsDebugBuild"); @@ -393,10 +461,10 @@ public void Run_Methods_ShouldHaveCorrectParameterNames() } [Fact] - public void Run_Methods_ShouldHaveOptionalSetupParameter() - { - // Act - Get the non-generic Run method explicitly - var nonGenericMethod = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + public void Run_Methods_ShouldHaveOptionalSetupParameter() + { + // Act - Get the non-generic Run method explicitly + var nonGenericMethod = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) .FirstOrDefault(m => m.Name == "Run" && !m.IsGenericMethodDefinition && @@ -410,7 +478,29 @@ public void Run_Methods_ShouldHaveOptionalSetupParameter() var setupParameter = nonGenericMethod.GetParameters()[1]; Assert.True(setupParameter.IsOptional); Assert.Null(setupParameter.DefaultValue); - - TestOutput.WriteLine($"Setup parameter is optional with default value: {setupParameter.DefaultValue ?? "null"}"); - } -} + + TestOutput.WriteLine($"Setup parameter is optional with default value: {setupParameter.DefaultValue ?? "null"}"); + } + + [Fact] + public void RunAsync_Methods_ShouldHaveOptionalSetupParameter() + { + // Act - Get the non-generic RunAsync method explicitly + var nonGenericMethod = typeof(BenchmarkProgram).GetMethods(BindingFlags.Public | BindingFlags.Static) + .FirstOrDefault(m => + m.Name == "RunAsync" && + !m.IsGenericMethodDefinition && + m.GetParameters().Length == 2 && + m.GetParameters()[0].ParameterType == typeof(string[]) && + m.GetParameters()[1].ParameterType == typeof(Action)); + + // Assert + Assert.NotNull(nonGenericMethod); + + var setupParameter = nonGenericMethod.GetParameters()[1]; + Assert.True(setupParameter.IsOptional); + Assert.Null(setupParameter.DefaultValue); + + TestOutput.WriteLine($"Setup parameter is optional with default value: {setupParameter.DefaultValue ?? "null"}"); + } +} diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkProgramTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkProgramTest.cs new file mode 100644 index 0000000..19128e7 --- /dev/null +++ b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkProgramTest.cs @@ -0,0 +1,145 @@ +using BenchmarkDotNet.Configs; +using Codebelt.Extensions.BenchmarkDotNet.Console; +using Codebelt.Extensions.Xunit; +using System; +using System.IO; +using Xunit; + +namespace Codebelt.Extensions.BenchmarkDotNet; + +/// +/// Functional tests for non-generic overloads and end-to-end host lifecycle +/// using the real against the existing tuning assemblies. +/// +public class BenchmarkProgramTest : Test +{ + // BenchmarkProgram.IsDebugBuild reflects whether the entry assembly was compiled in Debug mode, + // which is what BenchmarkWorkspace uses to locate tuning assemblies under bin/Debug or bin/Release. + private static readonly bool IsDebugBuild = BenchmarkProgram.IsDebugBuild; + + public BenchmarkProgramTest(ITestOutputHelper output) : base(output) + { + } + + /// + /// Exercises the non-generic Run(string[], Action<BenchmarkWorkspaceOptions>) overload (line 54) + /// and the non-generic Run(string[], Action<IServiceCollection>, Action<BenchmarkWorkspaceOptions>) + /// overload (line 68) which both ultimately delegate to + /// Run<BenchmarkWorkspace>(string[], Action<IServiceCollection>, Action<BenchmarkWorkspaceOptions>). + /// + /// Using "--list flat" makes BenchmarkSwitcher enumerate available benchmarks and return immediately + /// without executing any, which keeps the test fast and deterministic. + /// + [Fact] + public void Run_NonGeneric_WithListFlatArg_ShouldCompleteWithoutException() + { + var exception = Record.Exception(() => + BenchmarkProgram.Run( + new[] { "--list", "flat" }, + options => + { + options.AllowDebugBuild = IsDebugBuild; + options.Configuration = ManualConfig.CreateEmpty() + .WithOptions(ConfigOptions.DisableLogFile | ConfigOptions.DisableOptimizationsValidator); + })); + + Assert.Null(exception); + TestOutput.WriteLine("Non-generic BenchmarkProgram.Run completed via '--list flat'."); + } + + /// + /// Exercises the non-generic Run(string[], Action<IServiceCollection>, Action<BenchmarkWorkspaceOptions>) + /// overload with an explicit to ensure the three-argument + /// non-generic overload (line 68) is reached. + /// + [Fact] + public void Run_NonGenericWithServiceConfigurator_WithListFlatArg_ShouldCompleteWithoutException() + { + var configuratorCalled = false; + + var exception = Record.Exception(() => + BenchmarkProgram.Run( + new[] { "--list", "flat" }, + services => { configuratorCalled = true; }, + options => + { + options.AllowDebugBuild = IsDebugBuild; + options.Configuration = ManualConfig.CreateEmpty() + .WithOptions(ConfigOptions.DisableLogFile | ConfigOptions.DisableOptimizationsValidator); + })); + + Assert.Null(exception); + Assert.True(configuratorCalled); + TestOutput.WriteLine("Non-generic BenchmarkProgram.Run with service configurator completed."); + } + + /// + /// Verifies that loads the tuning assemblies + /// built for the current configuration and TFM, and that the assembly resolver hook is registered. + /// Running this test with the real workspace may cause the + /// event to fire for transitive dependencies discovered only through the tuning-folder DLL scan. + /// + [Fact] + public void BenchmarkWorkspace_LoadBenchmarkAssemblies_ShouldLoadTuningAssemblies() + { + // Arrange + var options = new BenchmarkWorkspaceOptions + { + AllowDebugBuild = IsDebugBuild, + TargetFrameworkMoniker = "net10.0" + }; + var workspace = new BenchmarkWorkspace(options); + + // Act + var assemblies = workspace.LoadBenchmarkAssemblies(); + + // Assert + Assert.NotNull(assemblies); + Assert.NotEmpty(assemblies); + + foreach (var assembly in assemblies) + { + TestOutput.WriteLine($"Loaded: {assembly.GetName().Name}"); + } + } + + /// + /// Verifies that behaves correctly when the + /// results directory already has files and then calls delete. This exercises the full cleanup path. + /// + [Fact] + public void BenchmarkWorkspace_PostProcessArtifacts_ShouldMoveFilesAndDeleteResultsDir() + { + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var resultsDir = Path.Combine(artifactsPath, "results"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + Directory.CreateDirectory(resultsDir); + + File.WriteAllText(Path.Combine(resultsDir, "SomeBenchmark-report.md"), "# report"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning" + }; + var workspace = new BenchmarkWorkspace(options); + + // Act + workspace.PostProcessArtifacts(); + + // Assert + Assert.False(Directory.Exists(resultsDir)); + Assert.True(File.Exists(Path.Combine(tuningDir, "SomeBenchmark-report.md"))); + } + finally + { + if (Directory.Exists(tempPath)) Directory.Delete(tempPath, true); + } + } +} diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkRunnerProgramTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkRunnerProgramTest.cs new file mode 100644 index 0000000..f048084 --- /dev/null +++ b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/BenchmarkRunnerProgramTest.cs @@ -0,0 +1,49 @@ +#if NET10_0_OR_GREATER +using BenchmarkDotNet.Environments; +using Codebelt.Extensions.BenchmarkDotNet.Console; +using Codebelt.Extensions.Xunit; +using Codebelt.Extensions.Xunit.Hosting; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.DependencyInjection.Extensions; +using Microsoft.Extensions.Hosting; +using System.Linq; +using Xunit; +using RunnerProgram = Codebelt.Extensions.BenchmarkDotNet.Runner.Program; + +namespace Codebelt.Extensions.BenchmarkDotNet; + +/// +/// Functional tests for the benchmark runner entry point. +/// +public class BenchmarkRunnerProgramTest : Test +{ + public BenchmarkRunnerProgramTest(ITestOutputHelper output) : base(output) + { + } + + /// + /// Verifies that the real runner entry point configures + /// as expected without executing benchmark workloads. + /// + [Fact] + public void Program_Main_ShouldConfigureBenchmarkWorkspaceOptions() + { + using var host = ApplicationHostFactory.Create( + builder => + { + builder.ConfigureServices(services => + { + services.RemoveAll(); + }); + }); + + var options = host.Services.GetRequiredService(); + var jobs = options.Configuration.GetJobs().ToArray(); + + Assert.Equal(BenchmarkProgram.IsDebugBuild, options.AllowDebugBuild); + Assert.True(options.SkipBenchmarksWithReports); + Assert.Contains(jobs, job => job.Environment.Runtime == CoreRuntime.Core90); + Assert.Contains(jobs, job => job.Environment.Runtime == CoreRuntime.Core10_0); + } +} +#endif diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests.csproj b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests.csproj new file mode 100644 index 0000000..c1c3f64 --- /dev/null +++ b/test/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests/Codebelt.Extensions.BenchmarkDotNet.FunctionalTests.csproj @@ -0,0 +1,13 @@ + + + + Codebelt.Extensions.BenchmarkDotNet + + + + + + + + + diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceOptionsTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceOptionsTest.cs index 5022c6f..f0d7697 100644 --- a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceOptionsTest.cs +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceOptionsTest.cs @@ -1,9 +1,10 @@ using BenchmarkDotNet.Configs; -using Codebelt.Extensions.Xunit; -using System; -using System.IO; -using System.Linq; -using Xunit; +using Codebelt.Extensions.Xunit; +using System; +using System.IO; +using System.Linq; +using System.Runtime.CompilerServices; +using Xunit; namespace Codebelt.Extensions.BenchmarkDotNet; @@ -655,4 +656,107 @@ public void SkipBenchmarksWithReports_ShouldBeSettable() // Assert Assert.True(options.SkipBenchmarksWithReports); } -} + + [Theory] + [InlineData(".NETFramework,Version=v1.1", "net11")] + [InlineData(".NETFramework,Version=v2.0", "net20")] + [InlineData(".NETFramework,Version=v3.5", "net35")] + [InlineData(".NETFramework,Version=v4.0", "net40")] + [InlineData(".NETFramework,Version=v4.0.3", "net403")] + [InlineData(".NETFramework,Version=v4.5.1", "net451")] + [InlineData(".NETFramework,Version=v4.7.2", "net472")] + [InlineData(".NETFramework,Version=v4.8", "net48")] + [InlineData(".NETFramework,Version=v4.8.1", "net481")] + public void ParseTargetFrameworkMoniker_ShouldReturnCorrectTfm_ForNETFramework(string frameworkName, string expectedTfm) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Equal(expectedTfm, result); + + TestOutput.WriteLine($"FrameworkName: {frameworkName} → TFM: {result}"); + } + + [Theory] + [InlineData(".NETStandard,Version=v1.0", "netstandard1.0")] + [InlineData(".NETStandard,Version=v1.6", "netstandard1.6")] + [InlineData(".NETStandard,Version=v2.0", "netstandard2.0")] + [InlineData(".NETStandard,Version=v2.1", "netstandard2.1")] + public void ParseTargetFrameworkMoniker_ShouldReturnCorrectTfm_ForNETStandard(string frameworkName, string expectedTfm) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Equal(expectedTfm, result); + + TestOutput.WriteLine($"FrameworkName: {frameworkName} → TFM: {result}"); + } + + [Theory] + [InlineData(".NETCoreApp,Version=v1.0", "netcoreapp1.0")] + [InlineData(".NETCoreApp,Version=v2.1", "netcoreapp2.1")] + [InlineData(".NETCoreApp,Version=v3.1", "netcoreapp3.1")] + public void ParseTargetFrameworkMoniker_ShouldReturnCorrectTfm_ForNETCoreApp(string frameworkName, string expectedTfm) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Equal(expectedTfm, result); + + TestOutput.WriteLine($"FrameworkName: {frameworkName} → TFM: {result}"); + } + + [Theory] + [InlineData(".NETCoreApp,Version=v5.0", "net5.0")] + [InlineData(".NETCoreApp,Version=v6.0", "net6.0")] + [InlineData(".NETCoreApp,Version=v8.0", "net8.0")] + [InlineData(".NETCoreApp,Version=v9.0", "net9.0")] + [InlineData(".NETCoreApp,Version=v10.0", "net10.0")] + public void ParseTargetFrameworkMoniker_ShouldReturnCorrectTfm_ForModernNet(string frameworkName, string expectedTfm) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Equal(expectedTfm, result); + + TestOutput.WriteLine($"FrameworkName: {frameworkName} → TFM: {result}"); + } + + [Theory] + [InlineData(null)] + [InlineData("")] + public void ParseTargetFrameworkMoniker_ShouldReturnNull_WhenFrameworkNameIsNullOrEmpty(string frameworkName) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Null(result); + } + + [Theory] + [InlineData("not-a-framework-name")] + [InlineData(".NETSomethingUnknown,Version=v1.0")] + public void ParseTargetFrameworkMoniker_ShouldReturnNull_WhenFrameworkNameIsUnrecognised(string frameworkName) + { + // Act + var result = ParseTargetFrameworkMoniker(frameworkName); + + // Assert + Assert.Null(result); + + TestOutput.WriteLine($"Unrecognised framework: {frameworkName} → null"); + } + + private static string ParseTargetFrameworkMoniker(string frameworkName) + { + return ParseTargetFrameworkMoniker(default(BenchmarkWorkspaceOptions), frameworkName); + } + + [UnsafeAccessor(UnsafeAccessorKind.StaticMethod, Name = "ParseTargetFrameworkMoniker")] + private static extern string ParseTargetFrameworkMoniker(BenchmarkWorkspaceOptions target, string frameworkName); +} diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceTest.cs index ab420a5..dd9e49a 100644 --- a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceTest.cs +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/BenchmarkWorkspaceTest.cs @@ -801,4 +801,201 @@ public void GetReportsResultsPath_AndGetReportsTuningPath_ShouldReturnDifferentP TestOutput.WriteLine($"Results path: {resultsPath}"); TestOutput.WriteLine($"Tuning path: {tuningPath}"); } + + [Fact] + public void LoadBenchmarkAssemblies_ShouldSkipEmptyFileName_WhenDllHasNoStem() + { + // Arrange – place a file whose stem is empty so that Path.GetFileNameWithoutExtension returns "" + // which exercises the IsNullOrEmpty(simpleName) guard in UpdateAssemblyLookup (lines 169-170). + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var buildConfig = IsDebugBuild ? "Debug" : "Release"; + try + { + Directory.CreateDirectory(tempPath); + var tuningDir = Path.Combine(tempPath, "tuning"); + var buildDir = Path.Combine(tuningDir, "bin", buildConfig, "net10.0"); + Directory.CreateDirectory(buildDir); + + // A file named ".dll" has an empty stem – Path.GetFileNameWithoutExtension(".dll") == "" + File.WriteAllText(Path.Combine(buildDir, ".dll"), "not-an-assembly"); + + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + RepositoryTuningFolder = "tuning", + BenchmarkProjectSuffix = "Benchmarks", + TargetFrameworkMoniker = "net10.0", + AllowDebugBuild = IsDebugBuild + }; + var workspace = new BenchmarkWorkspace(options); + + // Act – will throw because there are no real assemblies; that's expected + var exception = Record.Exception(() => workspace.LoadBenchmarkAssemblies()); + + // Assert – the important thing is that no unhandled exception from UpdateAssemblyLookup occurred + Assert.IsType(exception); + Assert.Contains("No assemblies were loaded", exception.Message); + + TestOutput.WriteLine("Empty-stem .dll file was silently skipped as expected."); + } + finally + { + if (Directory.Exists(tempPath)) + { + Directory.Delete(tempPath, true); + } + } + } + + [Fact] + public void PostProcessArtifacts_ShouldSkipBenchmarkRunFiles_WhenResultsDirContainsThem() + { + // Arrange + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + try + { + Directory.CreateDirectory(tempPath); + var artifactsPath = Path.Combine(tempPath, "artifacts"); + var resultsDir = Path.Combine(artifactsPath, "results"); + var tuningDir = Path.Combine(artifactsPath, "tuning"); + + Directory.CreateDirectory(resultsDir); + + // BenchmarkRun-prefixed files should NOT be moved + File.WriteAllText(Path.Combine(resultsDir, "BenchmarkRun-20240101-120000.log"), "run log"); + // Regular report files SHOULD be moved + File.WriteAllText(Path.Combine(resultsDir, "MyBenchmark-report.md"), "report"); + + var config = ManualConfig.CreateEmpty().WithArtifactsPath(artifactsPath); + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + Configuration = config, + RepositoryTuningFolder = "tuning" + }; + var workspace = new BenchmarkWorkspace(options); + + // Act + workspace.PostProcessArtifacts(); + + // Assert – the report file was moved but the BenchmarkRun log was NOT + Assert.False(Directory.Exists(resultsDir), "results directory should be deleted"); + Assert.True(File.Exists(Path.Combine(tuningDir, "MyBenchmark-report.md"))); + Assert.False(File.Exists(Path.Combine(tuningDir, "BenchmarkRun-20240101-120000.log"))); + + TestOutput.WriteLine("BenchmarkRun files correctly excluded from PostProcessArtifacts move."); + } + finally + { + if (Directory.Exists(tempPath)) + { + Directory.Delete(tempPath, true); + } + } + } + + [Fact] + public void LoadBenchmarkAssemblies_ShouldSkipDuplicateAssembly_WhenSameIdentityAppearsAtTwoPaths() + { + // This test exercises lines 141-142 (the duplicate-assembly-in-queue guard): + // + // if (assemblies.Any(a => AssemblyName.ReferenceMatchesDefinition(a.GetName(), candidateName))) + // { + // continue; // ← lines 141-142 + // } + // + // The guard fires when: + // 1. 'alreadyLoaded' (snapshot taken BEFORE the loop) does NOT contain the assembly, + // 2. the loop adds path1 of the assembly to 'assemblies', + // 3. the loop then encounters path2 with the SAME identity – + // 'alreadyLoaded' still has no entry (same snapshot), so the first `if` is skipped, + // but `assemblies.Any(...)` is TRUE → duplicate guard fires. + // + // To guarantee the assembly is never in the initial AppDomain snapshot we generate a + // brand-new assembly with a unique GUID-based name via PersistedAssemblyBuilder. + + var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString()); + var buildConfig = IsDebugBuild ? "Debug" : "Release"; + try + { + // Create two subdirectories that both match the build segment + var proj1Dir = Path.Combine(tempPath, "tuning", "proj1", "bin", buildConfig, "net10.0"); + var proj2Dir = Path.Combine(tempPath, "tuning", "proj2", "bin", buildConfig, "net10.0"); + Directory.CreateDirectory(proj1Dir); + Directory.CreateDirectory(proj2Dir); + + // Build a minimal in-memory assembly with a guaranteed-unique name + var uniqueId = Guid.NewGuid().ToString("N"); + var asmName = new AssemblyName($"Unique{uniqueId}.Benchmarks") { Version = new Version(1, 0, 0, 0) }; + + var persistedBuilder = new System.Reflection.Emit.PersistedAssemblyBuilder(asmName, typeof(object).Assembly); + persistedBuilder.DefineDynamicModule("main") + .DefineType("Placeholder", System.Reflection.TypeAttributes.Public | System.Reflection.TypeAttributes.Class) + .CreateType(); + + // Save to path1, then copy to path2 so both carry the SAME assembly identity + var fileName = $"Unique{uniqueId}.Benchmarks.dll"; + var path1 = Path.Combine(proj1Dir, fileName); + var path2 = Path.Combine(proj2Dir, fileName); + persistedBuilder.Save(path1); + File.Copy(path1, path2); + + var options = new BenchmarkWorkspaceOptions + { + RepositoryPath = tempPath, + RepositoryTuningFolder = "tuning", + BenchmarkProjectSuffix = "Benchmarks", + TargetFrameworkMoniker = "net10.0", + AllowDebugBuild = IsDebugBuild + }; + var workspace = new BenchmarkWorkspace(options); + + // Act – both paths match, one is loaded, the second hits the duplicate guard + var assemblies = workspace.LoadBenchmarkAssemblies(); + + // Assert – only ONE assembly loaded despite two matching paths + Assert.NotNull(assemblies); + Assert.Equal(1, assemblies.Length); + Assert.Contains(assemblies, a => a.GetName().Name == asmName.Name); + + TestOutput.WriteLine($"Loaded {assemblies.Length} assembly (duplicate suppressed): {assemblies[0].GetName().Name}"); + } + finally + { + // Unload assemblies and release file locks before cleanup on Windows + GC.Collect(); + GC.WaitForPendingFinalizers(); + GC.Collect(); + + // On Windows, loaded DLLs may still hold locks; retry deletion with delay + if (Directory.Exists(tempPath)) + { + const int maxRetries = 5; + const int delayMs = 200; + var lastException = (Exception)null; + for (int i = 0; i < maxRetries; i++) + { + try + { + Directory.Delete(tempPath, true); + break; + } + catch (UnauthorizedAccessException ex) + { + lastException = ex; + if (i < maxRetries - 1) + { + System.Threading.Thread.Sleep(delayMs); + } + } + } + // If we still can't delete after retries, log but don't fail the test + // The OS will clean up the temp directory eventually + if (Directory.Exists(tempPath)) + { + TestOutput.WriteLine($"Warning: Could not delete temp directory {tempPath}"); + } + } + } + } } diff --git a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/ServiceCollectionExtensionsTest.cs b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/ServiceCollectionExtensionsTest.cs index 86907e9..5f490e4 100644 --- a/test/Codebelt.Extensions.BenchmarkDotNet.Tests/ServiceCollectionExtensionsTest.cs +++ b/test/Codebelt.Extensions.BenchmarkDotNet.Tests/ServiceCollectionExtensionsTest.cs @@ -2,6 +2,7 @@ using System.Reflection; using Codebelt.Extensions.Xunit; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; using Xunit; namespace Codebelt.Extensions.BenchmarkDotNet @@ -46,6 +47,26 @@ public void AddBenchmarkWorkspace_GenericOverload_ShouldRegisterCustomImplementa Assert.Equal("repo-path", options.RepositoryPath); } + [Fact] + public void AddBenchmarkWorkspace_GenericOverload_ShouldUseNullCoalescingLambda_WhenSetupIsNull() + { + // When setup is null, the Configure call registers a no-op lambda (_ => {}). + // Resolving IOptions.Value causes the Options framework to + // invoke all registered configure-actions, which executes the null-coalescing lambda body. + var services = new ServiceCollection(); + services.AddBenchmarkWorkspace(setup: null); + using var sp = services.BuildServiceProvider(); + + // Resolving via IOptions triggers the registered configure action (the no-op lambda) + var optionsAccessor = sp.GetRequiredService>(); + var options = optionsAccessor.Value; + + Assert.NotNull(options); + Assert.IsType(sp.GetRequiredService()); + + TestOutput.WriteLine($"BenchmarkProjectSuffix: {options.BenchmarkProjectSuffix}"); + } + private sealed class FakeWorkspace : IBenchmarkWorkspace { public Assembly[] LoadBenchmarkAssemblies() => Array.Empty(); diff --git a/tuning/Codebelt.Extensions.BenchmarkDotNet.Console.Benchmarks/BenchmarkProgramBenchmark.cs b/tuning/Codebelt.Extensions.BenchmarkDotNet.Console.Benchmarks/BenchmarkProgramBenchmark.cs index de17aeb..f127580 100644 --- a/tuning/Codebelt.Extensions.BenchmarkDotNet.Console.Benchmarks/BenchmarkProgramBenchmark.cs +++ b/tuning/Codebelt.Extensions.BenchmarkDotNet.Console.Benchmarks/BenchmarkProgramBenchmark.cs @@ -2,7 +2,6 @@ using BenchmarkDotNet.Configs; using Cuemon; using Cuemon.Reflection; -using System; using System.Reflection; namespace Codebelt.Extensions.BenchmarkDotNet.Console;