Merge remote-tracking branch 'origin/main' into tests-out-of-repo

Author: radical

docs/updating-container-registry.md

@@ -0,0 +1,23 @@
+# Updating the test container registry
+
+Docker Hub has a rate limit on how often requests can be made from an IP. Because of this, we can't use Docker Hub in our build and automation. Instead, we have created a mirror container registry (netaspireci.azurecr.io) to mirror the public images used in our tests.
+
+Use the following guide to add a new image, or update an existing image with a new version.
+
+1. Install az cli - https://learn.microsoft.com/cli/azure/install-azure-cli
+2. Log in to the container registry (Assumes you are permitted to login to the registry. Contact an admin if you need permissions.)
+   1. `az login`
+   2. `az acr login --name netaspireci --expose-token --output tsv --query accessToken | docker login netaspireci.azurecr.io -u 00000000-0000-0000-0000-000000000000 --password-stdin`
+   3. See the following docs for more information
+      1. https://learn.microsoft.com/azure/container-registry/container-registry-get-started-docker-cli#log-in-to-a-registry
+      2. https://github.com/dotnet/dotnet-docker/blob/main/samples/push-image-to-acr.md#login-to-acr      
+3. Pull the image locally, tag it, and push it
+   1. `docker pull docker.io/library/redis:7.2`
+   2. `docker tag library/redis:7.2 netaspireci.azurecr.io/library/redis:7.2`
+   3. `docker push netaspireci.azurecr.io/library/redis:7.2`
+4. Alternatively, you can try the import command, but unless you have Docker Hub credentials it can fail due to rate limits.
+   1. `az acr import --name netaspireci --source docker.io/library/redis:7.2 --image library/redis:7.2 --username <Docker Hub user name> --password <Docker Hub token>`
+   2. See https://learn.microsoft.com/azure/container-registry/container-registry-import-images?tabs=azure-cli#import-from-docker-hub
+
+> [!IMPORTANT]
+> Note that the image name in netaspireci.azurecr.io needs to match exactly the name in docker.io or else the test won't be able to simply override the container registry.

src/Aspire.Dashboard/Components/Controls/Chart/ChartBase.cs

@@ -8,6 +8,7 @@
 using Aspire.Dashboard.Model;
 using Aspire.Dashboard.Otlp.Model;
 using Aspire.Dashboard.Otlp.Model.MetricValues;
+using Aspire.Dashboard.Otlp.Storage;
 using Aspire.Dashboard.Resources;
 using Aspire.Dashboard.Utils;
 using Microsoft.AspNetCore.Components;
@@ -30,18 +31,34 @@ public abstract class ChartBase : ComponentBase
     [Inject]
     public required IStringLocalizer<ControlsStrings> Loc { get; init; }
 
+    [Inject]
+    public required IStringLocalizer<Resources.Dialogs> DialogsLoc { get; init; }
+
     [Inject]
     public required IInstrumentUnitResolver InstrumentUnitResolver { get; init; }
 
     [Inject]
     public required BrowserTimeProvider TimeProvider { get; init; }
 
+    [Inject]
+    public required TelemetryRepository TelemetryRepository { get; init; }
+
     [Parameter, EditorRequired]
     public required InstrumentViewModel InstrumentViewModel { get; set; }
 
     [Parameter, EditorRequired]
     public required TimeSpan Duration { get; set; }
 
+    [Parameter]
+    public required List<OtlpApplication> Applications { get; set; }
+
+    // Stores a cache of the last set of spans returned as exemplars.
+    // This dictionary is replaced each time the chart is updated.
+    private Dictionary<SpanKey, OtlpSpan> _currentCache = new Dictionary<SpanKey, OtlpSpan>();
+    private Dictionary<SpanKey, OtlpSpan> _newCache = new Dictionary<SpanKey, OtlpSpan>();
+
+    private readonly record struct SpanKey(string TraceId, string SpanId);
+
     protected override void OnInitialized()
     {
         _currentDataStartTime = GetCurrentDataTime();
@@ -93,7 +110,7 @@ private Task OnInstrumentDataUpdate()
         return InvokeAsync(StateHasChanged);
     }
 
-    private (List<ChartTrace> Y, List<DateTimeOffset> X) CalculateHistogramValues(List<DimensionScope> dimensions, int pointCount, bool tickUpdate, DateTimeOffset inProgressDataTime, string yLabel)
+    private (List<ChartTrace> Y, List<DateTimeOffset> X, List<ChartExemplar> Exemplars) CalculateHistogramValues(List<DimensionScope> dimensions, int pointCount, bool tickUpdate, DateTimeOffset inProgressDataTime, string yLabel)
     {
         var pointDuration = Duration / pointCount;
         var traces = new Dictionary<int, ChartTrace>
@@ -103,8 +120,10 @@ private Task OnInstrumentDataUpdate()
             [99] = new() { Name = $"P99 {yLabel}", Percentile = 99 }
         };
         var xValues = new List<DateTimeOffset>();
+        var exemplars = new List<ChartExemplar>();
         var startDate = _currentDataStartTime;
         DateTimeOffset? firstPointEndTime = null;
+        DateTimeOffset? lastPointStartTime = null;
 
         // Generate the points in reverse order so that the chart is drawn from right to left.
         // Add a couple of extra points to the end so that the chart is drawn all the way to the right edge.
@@ -113,10 +132,11 @@ private Task OnInstrumentDataUpdate()
             var start = CalcOffset(pointIndex, startDate, pointDuration);
             var end = CalcOffset(pointIndex - 1, startDate, pointDuration);
             firstPointEndTime ??= end;
+            lastPointStartTime = start;
 
             xValues.Add(TimeProvider.ToLocalDateTimeOffset(end));
 
-            if (!TryCalculateHistogramPoints(dimensions, start, end, traces))
+            if (!TryCalculateHistogramPoints(dimensions, start, end, traces, exemplars))
             {
                 foreach (var trace in traces)
                 {
@@ -131,7 +151,7 @@ private Task OnInstrumentDataUpdate()
         }
         xValues.Reverse();
 
-        if (tickUpdate && TryCalculateHistogramPoints(dimensions, firstPointEndTime!.Value, inProgressDataTime, traces))
+        if (tickUpdate && TryCalculateHistogramPoints(dimensions, firstPointEndTime!.Value, inProgressDataTime, traces, exemplars))
         {
             xValues.Add(TimeProvider.ToLocalDateTimeOffset(inProgressDataTime));
         }
@@ -161,12 +181,15 @@ private Task OnInstrumentDataUpdate()
 
             previousValues = currentTrace;
         }
-        return (traces.Select(kvp => kvp.Value).ToList(), xValues);
+
+        exemplars = exemplars.Where(p => p.Start <= startDate && p.Start >= lastPointStartTime!.Value).OrderBy(p => p.Start).ToList();
+
+        return (traces.Select(kvp => kvp.Value).ToList(), xValues, exemplars);
     }
 
     private string FormatTooltip(string name, double yValue, DateTimeOffset xValue)
     {
-        return $"<b>{HttpUtility.HtmlEncode(InstrumentViewModel.Instrument?.Name)}</b><br />{HttpUtility.HtmlEncode(name)}: {FormatHelpers.FormatNumberWithOptionalDecimalPlaces(yValue, CultureInfo.CurrentCulture)}<br />Time: {FormatHelpers.FormatTime(TimeProvider, TimeProvider.ToLocal(xValue))}";
+        return $"<b>{HttpUtility.HtmlEncode(InstrumentViewModel.Instrument?.Name)}</b><br />{HttpUtility.HtmlEncode(name)}: {FormatHelpers.FormatNumberWithOptionalDecimalPlaces(yValue, maxDecimalPlaces: 6, CultureInfo.CurrentCulture)}<br />Time: {FormatHelpers.FormatTime(TimeProvider, TimeProvider.ToLocal(xValue))}";
     }
 
     private static HistogramValue GetHistogramValue(MetricValueBase metric)
@@ -179,7 +202,7 @@ private static HistogramValue GetHistogramValue(MetricValueBase metric)
         throw new InvalidOperationException("Unexpected metric type: " + metric.GetType());
     }
 
-    internal static bool TryCalculateHistogramPoints(List<DimensionScope> dimensions, DateTimeOffset start, DateTimeOffset end, Dictionary<int, ChartTrace> traces)
+    internal bool TryCalculateHistogramPoints(List<DimensionScope> dimensions, DateTimeOffset start, DateTimeOffset end, Dictionary<int, ChartTrace> traces, List<ChartExemplar> exemplars)
     {
         var hasValue = false;
 
@@ -199,6 +222,8 @@ internal static bool TryCalculateHistogramPoints(List<DimensionScope> dimensions
                 {
                     var histogramValue = GetHistogramValue(metric);
 
+                    AddExemplars(exemplars, metric);
+
                     // Only use the first recorded entry if it is the beginning of data.
                     // We can verify the first entry is the beginning of data by checking if the number of buckets equals the total count.
                     if (i == 0 && CountBuckets(histogramValue) != histogramValue.Count)
@@ -247,6 +272,57 @@ internal static bool TryCalculateHistogramPoints(List<DimensionScope> dimensions
         return hasValue;
     }
 
+    private void AddExemplars(List<ChartExemplar> exemplars, MetricValueBase metric)
+    {
+        if (metric.HasExemplars)
+        {
+            foreach (var exemplar in metric.Exemplars)
+            {
+                // TODO: Exemplars are duplicated on metrics in some scenarios.
+                // This is a quick fix to ensure a distinct collection of metrics are displayed in the UI.
+                // Investigation is needed into why there are duplicates.
+                var exists = false;
+                foreach (var existingExemplar in exemplars)
+                {
+                    if (exemplar.Start == existingExemplar.Start &&
+                        exemplar.Value == existingExemplar.Value &&
+                        exemplar.SpanId == existingExemplar.SpanId &&
+                        exemplar.TraceId == existingExemplar.TraceId)
+                    {
+                        exists = true;
+                        break;
+                    }
+                }
+                if (exists)
+                {
+                    continue;
+                }
+
+                // Try to find span the the local cache first.
+                // This is done to avoid scanning a potentially large trace collection in repository.
+                var key = new SpanKey(exemplar.TraceId, exemplar.SpanId);
+                if (!_currentCache.TryGetValue(key, out var span))
+                {
+                    span = GetSpan(exemplar.TraceId, exemplar.SpanId);
+                }
+                if (span != null)
+                {
+                    _newCache[key] = span;
+                }
+
+                var exemplarStart = TimeProvider.ToLocalDateTimeOffset(exemplar.Start);
+                exemplars.Add(new ChartExemplar
+                {
+                    Start = exemplarStart,
+                    Value = exemplar.Value,
+                    TraceId = exemplar.TraceId,
+                    SpanId = exemplar.SpanId,
+                    Span = span
+                });
+            }
+        }
+    }
+
     private static ulong CountBuckets(HistogramValue histogramValue)
     {
         ulong value = 0ul;
@@ -287,11 +363,12 @@ private static ulong CountBuckets(HistogramValue histogramValue)
         return explicitBounds[explicitBounds.Length - 1];
     }
 
-    private (List<ChartTrace> Y, List<DateTimeOffset> X) CalculateChartValues(List<DimensionScope> dimensions, int pointCount, bool tickUpdate, DateTimeOffset inProgressDataTime, string yLabel)
+    private (List<ChartTrace> Y, List<DateTimeOffset> X, List<ChartExemplar> Exemplars) CalculateChartValues(List<DimensionScope> dimensions, int pointCount, bool tickUpdate, DateTimeOffset inProgressDataTime, string yLabel)
     {
         var pointDuration = Duration / pointCount;
         var yValues = new List<double?>();
         var xValues = new List<DateTimeOffset>();
+        var exemplars = new List<ChartExemplar>();
         var startDate = _currentDataStartTime;
         DateTimeOffset? firstPointEndTime = null;
 
@@ -305,7 +382,7 @@ private static ulong CountBuckets(HistogramValue histogramValue)
 
             xValues.Add(TimeProvider.ToLocalDateTimeOffset(end));
 
-            if (TryCalculatePoint(dimensions, start, end, out var tickPointValue))
+            if (TryCalculatePoint(dimensions, start, end, exemplars, out var tickPointValue))
             {
                 yValues.Add(tickPointValue);
             }
@@ -318,7 +395,7 @@ private static ulong CountBuckets(HistogramValue histogramValue)
         yValues.Reverse();
         xValues.Reverse();
 
-        if (tickUpdate && TryCalculatePoint(dimensions, firstPointEndTime!.Value, inProgressDataTime, out var inProgressPointValue))
+        if (tickUpdate && TryCalculatePoint(dimensions, firstPointEndTime!.Value, inProgressDataTime, exemplars, out var inProgressPointValue))
         {
             yValues.Add(inProgressPointValue);
             xValues.Add(TimeProvider.ToLocalDateTimeOffset(inProgressDataTime));
@@ -343,10 +420,10 @@ private static ulong CountBuckets(HistogramValue histogramValue)
             }
         }
 
-        return ([trace], xValues);
+        return ([trace], xValues, exemplars);
     }
 
-    private static bool TryCalculatePoint(List<DimensionScope> dimensions, DateTimeOffset start, DateTimeOffset end, out double pointValue)
+    private bool TryCalculatePoint(List<DimensionScope> dimensions, DateTimeOffset start, DateTimeOffset end, List<ChartExemplar> exemplars, out double pointValue)
     {
         var hasValue = false;
         pointValue = 0d;
@@ -371,6 +448,8 @@ private static bool TryCalculatePoint(List<DimensionScope> dimensions, DateTimeO
                     dimensionValue = Math.Max(value, dimensionValue);
                     hasValue = true;
                 }
+
+                AddExemplars(exemplars, metric);
             }
 
             pointValue += dimensionValue;
@@ -406,16 +485,29 @@ private async Task UpdateChart(bool tickUpdate, DateTimeOffset inProgressDataTim
 
         List<ChartTrace> traces;
         List<DateTimeOffset> xValues;
+        List<ChartExemplar> exemplars;
         if (InstrumentViewModel.Instrument?.Type != OtlpInstrumentType.Histogram || InstrumentViewModel.ShowCount)
         {
-            (traces, xValues) = CalculateChartValues(InstrumentViewModel.MatchedDimensions, GraphPointCount, tickUpdate, inProgressDataTime, unit);
+            (traces, xValues, exemplars) = CalculateChartValues(InstrumentViewModel.MatchedDimensions, GraphPointCount, tickUpdate, inProgressDataTime, unit);
+
+            // TODO: Exemplars on non-histogram charts doesn't work well. Don't display for now.
+            exemplars.Clear();
         }
         else
         {
-            (traces, xValues) = CalculateHistogramValues(InstrumentViewModel.MatchedDimensions, GraphPointCount, tickUpdate, inProgressDataTime, unit);
+            (traces, xValues, exemplars) = CalculateHistogramValues(InstrumentViewModel.MatchedDimensions, GraphPointCount, tickUpdate, inProgressDataTime, unit);
         }
 
-        await OnChartUpdated(traces, xValues, tickUpdate, inProgressDataTime);
+        // Replace cache for next update.
+        _currentCache = _newCache;
+        _newCache = new Dictionary<SpanKey, OtlpSpan>();
+
+        await OnChartUpdated(traces, xValues, exemplars, tickUpdate, inProgressDataTime);
+    }
+
+    protected OtlpSpan? GetSpan(string traceId, string spanId)
+    {
+        return MetricsHelpers.GetSpan(TelemetryRepository, traceId, spanId);
     }
 
     private DateTimeOffset GetCurrentDataTime()
@@ -425,8 +517,8 @@ private DateTimeOffset GetCurrentDataTime()
 
     private string GetDisplayedUnit(OtlpInstrument instrument)
     {
-        return InstrumentUnitResolver.ResolveDisplayedUnit(instrument);
+        return InstrumentUnitResolver.ResolveDisplayedUnit(instrument, titleCase: true, pluralize: true);
     }
 
-    protected abstract Task OnChartUpdated(List<ChartTrace> traces, List<DateTimeOffset> xValues, bool tickUpdate, DateTimeOffset inProgressDataTime);
+    protected abstract Task OnChartUpdated(List<ChartTrace> traces, List<DateTimeOffset> xValues, List<ChartExemplar> exemplars, bool tickUpdate, DateTimeOffset inProgressDataTime);
 }

src/Aspire.Dashboard/Components/Controls/Chart/ChartContainer.razor

@@ -25,7 +25,7 @@ else
                        Label="@Loc[nameof(ControlsStrings.ChartContainerGraphTab)]"
                        Icon="@(new Icons.Regular.Size24.DataArea())">
                 <div class="metrics-chart-container metric-tab">
-                    <PlotlyChart InstrumentViewModel="_instrumentViewModel" Duration="Duration"/>
+                    <PlotlyChart InstrumentViewModel="_instrumentViewModel" Duration="Duration" Applications="Applications"/>
                     <ChartFilters InstrumentViewModel="_instrumentViewModel" Instrument="_instrument" ViewModel="_viewModel"/>
                 </div>
             </FluentTab>
@@ -34,7 +34,7 @@ else
                        Label="@Loc[nameof(ControlsStrings.ChartContainerTableTab)]"
                        Icon="@(new Icons.Regular.Size24.Table())">
                 <div class="metric-tab">
-                    <MetricTable InstrumentViewModel="_instrumentViewModel" Duration="Duration"/>
+                    <MetricTable InstrumentViewModel="_instrumentViewModel" Duration="Duration" Applications="Applications" />
                     <ChartFilters InstrumentViewModel="_instrumentViewModel" Instrument="_instrument" ViewModel="_viewModel"/>
                 </div>
             </FluentTab>
@@ -48,4 +48,7 @@ else
 
     [Parameter, EditorRequired]
     public required Func<Metrics.MetricViewKind, Task> OnViewChangedAsync { get; set; }
+
+    [Parameter]
+    public required List<OtlpApplication> Applications { get; set; }
 }

src/Aspire.Dashboard/Components/Controls/Chart/ChartExemplar.cs

@@ -0,0 +1,17 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+using System.Diagnostics;
+using Aspire.Dashboard.Otlp.Model;
+
+namespace Aspire.Dashboard.Components.Controls.Chart;
+
+[DebuggerDisplay("Start = {Start}, Value = {Value}, TraceId = {TraceId}, SpanId = {SpanId}")]
+public class ChartExemplar
+{
+    public required DateTimeOffset Start { get; init; }
+    public required double Value { get; init; }
+    public required string TraceId { get; init; }
+    public required string SpanId { get; init; }
+    public required OtlpSpan? Span { get; init; }
+}

src/Aspire.Dashboard/Components/Controls/Chart/ChartTrace.cs

@@ -1,4 +1,4 @@
-// Licensed to the .NET Foundation under one or more agreements.
+// Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
 namespace Aspire.Dashboard.Components.Controls.Chart;