Skip to content

Commit cf985fa

Browse files
aspire-repo-bot[bot]danegstaCopilot
authored
[release/13.4] Add proxyless endpoint on-demand allocation (#17859)
* Add proxyless endpoint on-demand allocation Allow dynamic proxyless container endpoints to allocate a target-port fallback when an endpoint reference requires an allocated endpoint before container creation. Disable the on-demand allocator once container ports are built so later resolution continues to use DCP service updates. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Log proxyless endpoint fallback allocation Log when a dynamic proxyless container endpoint is resolved before container creation and Aspire assigns the public port to match the target port. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Use atomic proxyless allocator cutoff Replace the endpoint allocation cutoff lock with an atomic exchange so BuildContainerPorts remains the point where on-demand proxyless endpoint allocation stops. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Remove proxyless allocator clear helper Use the atomic OnDemandAllocatedEndpointProvider setter directly at the BuildContainerPorts cutoff instead of a dedicated clear wrapper. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Simplify proxyless allocator provider storage Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Move proxyless allocator cutoff into port build Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Move on-demand endpoint allocation ownership to resource Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Clarify on-demand endpoint allocation lifecycle Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Simplify resource-owned endpoint allocation Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Move proxyless allocation cutoff after configuration Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: David Negstad <David.Negstad@microsoft.com> Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 62d6f98 commit cf985fa

7 files changed

Lines changed: 222 additions & 36 deletions

File tree

‎src/Aspire.Hosting/ApplicationModel/EndpointAnnotation.cs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4-
using System.Diagnostics;
4+
using System.Collections;
55
using System.Collections.Concurrent;
6+
using System.Diagnostics;
7+
using System.Diagnostics.CodeAnalysis;
68
using System.Net.Sockets;
7-
using System.Collections;
89

910
namespace Aspire.Hosting.ApplicationModel;
1011

@@ -454,6 +455,24 @@ public Task<AllocatedEndpoint> GetAllocatedEndpointAsync(NetworkIdentifier netwo
454455
return nes.Snapshot.GetValueAsync(cancellationToken);
455456
}
456457

458+
internal bool TryGetAllocatedEndpoint(NetworkIdentifier networkId, [NotNullWhen(true)] out AllocatedEndpoint? endpoint)
459+
{
460+
endpoint = null;
461+
462+
foreach (var endpointSnapshot in _snapshots)
463+
{
464+
if (!endpointSnapshot.NetworkID.Equals(networkId) || !endpointSnapshot.Snapshot.IsValueSet)
465+
{
466+
continue;
467+
}
468+
469+
endpoint = endpointSnapshot.Snapshot.GetValueAsync().GetAwaiter().GetResult();
470+
return true;
471+
}
472+
473+
return false;
474+
}
475+
457476
private NetworkEndpointSnapshot GetSnapshotFor(NetworkIdentifier networkId)
458477
{
459478
lock (_snapshots)

‎src/Aspire.Hosting/ApplicationModel/EndpointReference.cs‎

Lines changed: 24 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -221,6 +221,26 @@ public ReferenceExpression GetTlsValue(ReferenceExpression enabledValue, Referen
221221
GetAllocatedEndpoint()
222222
?? throw new InvalidOperationException($"The endpoint `{EndpointName}` is not allocated for the resource `{Resource.Name}`.");
223223

224+
internal Task<AllocatedEndpoint> GetAllocatedEndpointAsync(NetworkIdentifier networkId, CancellationToken cancellationToken = default)
225+
{
226+
var endpointAnnotation = EndpointAnnotation;
227+
if (endpointAnnotation.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var endpoint))
228+
{
229+
return Task.FromResult(endpoint);
230+
}
231+
232+
foreach (var allocationAnnotation in Resource.Annotations.OfType<OnDemandEndpointAllocationAnnotation>())
233+
{
234+
endpoint = allocationAnnotation.TryAllocate(endpointAnnotation, networkId);
235+
if (endpoint is not null)
236+
{
237+
return Task.FromResult(endpoint);
238+
}
239+
}
240+
241+
return endpointAnnotation.AllAllocatedEndpoints.GetAllocatedEndpointAsync(networkId, cancellationToken);
242+
}
243+
224244
private EndpointAnnotation? GetEndpointAnnotation()
225245
{
226246
if (_endpointAnnotation is not null)
@@ -242,20 +262,9 @@ public ReferenceExpression GetTlsValue(ReferenceExpression enabledValue, Referen
242262
return null;
243263
}
244264

245-
foreach (var nes in endpointAnnotation.AllAllocatedEndpoints)
246-
{
247-
if (string.Equals(nes.NetworkID.Value, (_contextNetworkId ?? KnownNetworkIdentifiers.LocalhostNetwork).Value, StringComparisons.NetworkId))
248-
{
249-
if (!nes.Snapshot.IsValueSet)
250-
{
251-
continue;
252-
}
253-
254-
return nes.Snapshot.GetValueAsync().GetAwaiter().GetResult();
255-
}
256-
}
257-
258-
return null;
265+
return endpointAnnotation.AllAllocatedEndpoints.TryGetAllocatedEndpoint(_contextNetworkId ?? KnownNetworkIdentifiers.LocalhostNetwork, out var allocatedEndpoint)
266+
? allocatedEndpoint
267+
: null;
259268
}
260269

261270
/// <summary>
@@ -382,8 +391,7 @@ public class EndpointReferenceExpression(EndpointReference endpointReference, En
382391

383392
async ValueTask<string?> ResolveValueWithAllocatedAddress()
384393
{
385-
var endpointSnapshots = Endpoint.EndpointAnnotation.AllAllocatedEndpoints;
386-
var allocatedEndpoint = await endpointSnapshots.GetAllocatedEndpointAsync(networkContext, cancellationToken).ConfigureAwait(false);
394+
var allocatedEndpoint = await Endpoint.GetAllocatedEndpointAsync(networkContext, cancellationToken).ConfigureAwait(false);
387395

388396
return Property switch
389397
{

‎src/Aspire.Hosting/ApplicationModel/OnDemandEndpointAllocationAnnotation.cs‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
4+
namespace Aspire.Hosting.ApplicationModel;
5+
6+
/// <summary>
7+
/// Stores a resource-owned endpoint allocator that can run before normal allocation completes.
8+
/// </summary>
9+
internal sealed class OnDemandEndpointAllocationAnnotation(Func<EndpointAnnotation, NetworkIdentifier, AllocatedEndpoint?> allocator) : IResourceAnnotation
10+
{
11+
private Func<EndpointAnnotation, NetworkIdentifier, AllocatedEndpoint?>? _allocator = allocator;
12+
13+
public AllocatedEndpoint? TryAllocate(EndpointAnnotation endpoint, NetworkIdentifier networkId)
14+
{
15+
var allocator = _allocator;
16+
17+
return allocator?.Invoke(endpoint, networkId);
18+
}
19+
20+
public void StopAllocating()
21+
{
22+
Interlocked.Exchange(ref _allocator, null);
23+
}
24+
}

‎src/Aspire.Hosting/Dcp/ContainerCreator.cs‎

Lines changed: 18 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -199,7 +199,7 @@ public IEnumerable<RenderedModelResource<Container>> PrepareObjects()
199199
}
200200

201201
var containerAppResource = new RenderedModelResource<Container>(container, ctr);
202-
DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get());
202+
DcpModelUtilities.AddServicesProducedInfo(containerAppResource, _appResources.Get(), _logger);
203203
_appResources.Add(containerAppResource);
204204
result.Add(containerAppResource);
205205
}
@@ -306,11 +306,6 @@ private async Task BuildAndCreateContainerAsync(RenderedModelResource<Container>
306306

307307
var spec = dcpContainer.Spec;
308308

309-
if (cr.ServicesProduced.Count > 0)
310-
{
311-
spec.Ports = BuildContainerPorts(cr);
312-
}
313-
314309
spec.VolumeMounts = BuildContainerMounts(cr.ModelResource);
315310

316311
var (runArgs, failedToApplyRunArgs) = await BuildRunArgsAsync(logger, cr.ModelResource, cToken).ConfigureAwait(false);
@@ -321,11 +316,24 @@ private async Task BuildAndCreateContainerAsync(RenderedModelResource<Container>
321316
spec.RunArgs = runArgs;
322317

323318
var (configuration, pemCertificates, createFiles) = await BuildContainerConfiguration(cr, logger, cToken).ConfigureAwait(false);
319+
// Configuration callbacks are the last pre-creation point where on-demand allocation can run.
320+
cr.ModelResource.Annotations
321+
.OfType<OnDemandEndpointAllocationAnnotation>()
322+
.SingleOrDefault()
323+
?.StopAllocating();
324+
324325
if (configuration.Exception is not null)
325326
{
326327
throw new FailedToApplyEnvironmentException($"Failed to apply configuration to container {cr.ModelResource.Name}", configuration.Exception);
327328
}
328329

330+
// Environment callbacks can resolve proxyless endpoint ports and commit a fallback host port,
331+
// so build ports afterward.
332+
if (cr.ServicesProduced.Count > 0)
333+
{
334+
spec.Ports = BuildContainerPorts(cr);
335+
}
336+
329337
var args = configuration.Arguments.ToList();
330338
if (modelContainer is ContainerResource { ShellExecution: true })
331339
{
@@ -987,10 +995,11 @@ private static List<ContainerPortSpec> BuildContainerPorts(RenderedModelResource
987995

988996
if (!ea.IsProxied && ea.SpecifiedPort is int hostPort)
989997
{
998+
sp.Service.Spec.Port ??= hostPort;
990999
portSpec.HostPort = hostPort;
9911000
}
9921001

993-
switch (sp.EndpointAnnotation.Protocol)
1002+
switch (ea.Protocol)
9941003
{
9951004
case ProtocolType.Tcp:
9961005
portSpec.Protocol = PortProtocol.TCP;
@@ -1000,9 +1009,9 @@ private static List<ContainerPortSpec> BuildContainerPorts(RenderedModelResource
10001009
break;
10011010
}
10021011

1003-
if (sp.EndpointAnnotation.TargetHost != KnownHostNames.Localhost)
1012+
if (ea.TargetHost != KnownHostNames.Localhost)
10041013
{
1005-
portSpec.HostIP = sp.EndpointAnnotation.TargetHost;
1014+
portSpec.HostIP = ea.TargetHost;
10061015
}
10071016

10081017
ports.Add(portSpec);

‎src/Aspire.Hosting/Dcp/DcpModelUtilities.cs‎

Lines changed: 54 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
using System.Net;
88
using Aspire.Hosting.ApplicationModel;
99
using Aspire.Hosting.Dcp.Model;
10+
using Microsoft.Extensions.Logging;
1011

1112
namespace Aspire.Hosting.Dcp;
1213

@@ -33,7 +34,8 @@ internal static bool ShouldDeferCreateForExplicitStart(IResource modelResource,
3334
/// </summary>
3435
internal static void AddServicesProducedInfo<TDcpResource>(
3536
RenderedModelResource<TDcpResource> appResource,
36-
IEnumerable<IAppResource> appResources)
37+
IEnumerable<IAppResource> appResources,
38+
ILogger? logger = null)
3739
where TDcpResource : CustomResource, IKubernetesStaticMetadata
3840
{
3941
var modelResource = appResource.ModelResource;
@@ -87,6 +89,16 @@ internal static void AddServicesProducedInfo<TDcpResource>(
8789
appResource.ServicesProduced.Add(sp);
8890
}
8991

92+
if (appResource.ServicesProduced.Any(sp => IsDynamicProxylessContainerEndpoint(appResource, sp)) &&
93+
!modelResource.Annotations.OfType<OnDemandEndpointAllocationAnnotation>().Any())
94+
{
95+
// These endpoints normally get their host port during container creation. If a
96+
// reference needs the allocated endpoint while building the container configuration,
97+
// commit the fallback port before waiting would deadlock resource creation.
98+
modelResource.Annotations.Add(new OnDemandEndpointAllocationAnnotation(
99+
(endpoint, networkId) => TryAllocateDynamicProxylessContainerEndpoint(appResource, endpoint, networkId, logger)));
100+
}
101+
90102
static bool HasMultipleReplicas(CustomResource resource)
91103
{
92104
if (resource is Executable exe && exe.Metadata.Annotations.TryGetValue(CustomResource.ResourceReplicaCount, out var value) && int.TryParse(value, CultureInfo.InvariantCulture, out var replicas) && replicas > 1)
@@ -161,9 +173,10 @@ internal static bool TryApplyServiceAddressToEndpoint(Service observedService, I
161173
return isDynamicProxylessContainerEndpoint && AreResourceEndpointsAllocated(modelResource);
162174
}
163175

164-
private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp, bool allowPending)
176+
private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp, bool allowPending, int? fallbackPort = null)
165177
{
166178
var svc = sp.DcpResource;
179+
var allocatedPort = svc.AllocatedPort ?? fallbackPort;
167180

168181
if (sp.EndpointAnnotation.AllocatedEndpoint is not null)
169182
{
@@ -182,7 +195,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp
182195
throw new InvalidDataException($"Service {svc.Metadata.Name} should have valid address at this point");
183196
}
184197

185-
if (!sp.EndpointAnnotation.IsProxied && svc.AllocatedPort is null)
198+
if (!sp.EndpointAnnotation.IsProxied && allocatedPort is null)
186199
{
187200
if (allowPending)
188201
{
@@ -192,7 +205,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp
192205
throw new InvalidOperationException($"Service '{svc.Metadata.Name}' needs to specify a port for endpoint '{sp.EndpointAnnotation.Name}' since it isn't using a proxy.");
193206
}
194207

195-
if (!svc.HasCompleteAddress)
208+
if (allocatedPort is null || string.IsNullOrEmpty(svc.AllocatedAddress))
196209
{
197210
if (allowPending)
198211
{
@@ -207,7 +220,7 @@ private static bool TryAddLocalhostAllocatedEndpoint(ServiceWithModelResource sp
207220
sp.EndpointAnnotation.AllocatedEndpoint = new AllocatedEndpoint(
208221
sp.EndpointAnnotation,
209222
targetHost,
210-
(int)svc.AllocatedPort!,
223+
allocatedPort.Value,
211224
bindingMode,
212225
targetPortExpression: $$$"""{{- portForServing "{{{svc.Metadata.Name}}}" -}}""",
213226
KnownNetworkIdentifiers.LocalhostNetwork);
@@ -278,6 +291,42 @@ private static bool IsDynamicProxylessContainerEndpoint<TDcpResource>(RenderedMo
278291
sp.EndpointAnnotation.SpecifiedPort is null;
279292
}
280293

294+
private static AllocatedEndpoint? TryAllocateDynamicProxylessContainerEndpoint<TDcpResource>(
295+
RenderedModelResource<TDcpResource> resource,
296+
EndpointAnnotation endpoint,
297+
NetworkIdentifier networkId,
298+
ILogger? logger)
299+
where TDcpResource : CustomResource, IKubernetesStaticMetadata
300+
{
301+
var sp = resource.ServicesProduced.SingleOrDefault(sp =>
302+
ReferenceEquals(sp.EndpointAnnotation, endpoint) &&
303+
IsDynamicProxylessContainerEndpoint(resource, sp));
304+
if (sp is null)
305+
{
306+
return null;
307+
}
308+
309+
Debug.Assert(endpoint.TargetPort is not null);
310+
311+
var targetPort = endpoint.TargetPort.Value;
312+
endpoint.Port = targetPort;
313+
logger?.LogInformation(
314+
"Endpoint '{EndpointName}' on container resource '{ResourceName}' was resolved before the container was created, so Aspire is assigning public port {PublicPort} to match target port {TargetPort} for proxyless access.",
315+
endpoint.Name,
316+
sp.ModelResource.Name,
317+
targetPort,
318+
targetPort);
319+
320+
if (TryAddLocalhostAllocatedEndpoint(sp, allowPending: false, fallbackPort: targetPort))
321+
{
322+
AddContainerNetworkAllocatedEndpoint(resource, sp);
323+
}
324+
325+
return endpoint.AllAllocatedEndpoints.TryGetAllocatedEndpoint(networkId, out var allocatedEndpoint)
326+
? allocatedEndpoint
327+
: null;
328+
}
329+
281330
internal static void AddContainerTunnelAllocatedEndpoints(
282331
IEnumerable<IResource> affectedResources,
283332
DcpAppResourceStore allAppResources,

0 commit comments

Comments
 (0)
⚡