Skip to content

Commit 920afb2

Browse files
scbeddsemick-dev
scbedd
and
semick-dev
authored
Add proxy audit log and other assorted fixes (#8781)
* Use a unified lock for all access to a recording. sanitizers or recording entries. * Creating an audit log so we can see raw associated information for important events about a specific recording. * obtain a lock before stopping playback, ensuring that ongoing writes in other threads won't be broken --------- Co-authored-by: semick-dev <sbeddall@gmail.com>
1 parent 3c8f30f commit 920afb2

14 files changed

Lines changed: 306 additions & 57 deletions

File tree

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/PlaybackTests.cs‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -151,10 +151,16 @@ public async void TestStopPlaybackSimple()
151151
};
152152
await controller.Start();
153153
var targetRecordingId = httpContext.Response.Headers["x-recording-id"].ToString();
154-
154+
155155
httpContext.Request.Headers["x-recording-id"] = new string[] { targetRecordingId };
156-
controller.Stop();
156+
await controller.Stop();
157+
158+
var auditSession = testRecordingHandler.AuditSessions[targetRecordingId];
159+
160+
Assert.NotNull(auditSession);
161+
var auditResults = TestHelpers.ExhaustQueue<AuditLogItem>(auditSession);
157162

163+
Assert.Equal(2, auditResults.Count);
158164
Assert.False(testRecordingHandler.PlaybackSessions.ContainsKey(targetRecordingId));
159165
}
160166

@@ -189,7 +195,7 @@ public async void TestStopPlaybackInMemory()
189195
await playbackController.Start();
190196
var targetRecordingId = playbackContext.Response.Headers["x-recording-id"].ToString();
191197
playbackContext.Request.Headers["x-recording-id"] = new string[] { targetRecordingId };
192-
playbackController.Stop();
198+
await playbackController.Stop();
193199

194200
testRecordingHandler.InMemorySessions.ContainsKey(targetRecordingId);
195201
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordSessionTests.cs‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,12 @@ public async Task CanRoundTripDockerDigest()
165165
session.Session.Entries.Add(testEntry);
166166
await handler.StopRecording(recordingId);
167167

168+
// ensure that we audited properly
169+
var auditSession = handler.AuditSessions[recordingId];
170+
var auditItems = TestHelpers.ExhaustQueue<AuditLogItem>(auditSession);
171+
172+
Assert.Equal(2, auditItems.Count);
173+
168174
// now load it, did we avoid mangling it?
169175
var sessionFromDisk = TestHelpers.LoadRecordSession(Path.Combine(testFolder, testName));
170176
var targetEntry = sessionFromDisk.Session.Entries[0];

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordingHandlerTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -273,7 +273,7 @@ public async Task TestInMemoryPurgesSucessfully()
273273

274274
await recordingHandler.StartPlaybackAsync(key, httpContext.Response, Common.RecordingType.InMemory);
275275
var playbackSession = httpContext.Response.Headers["x-recording-id"];
276-
recordingHandler.StopPlayback(playbackSession, true);
276+
await recordingHandler.StopPlayback(playbackSession, true);
277277

278278
Assert.True(0 == recordingHandler.InMemorySessions.Count);
279279
}
@@ -287,7 +287,7 @@ public async Task TestInMemoryDoesntPurgeErroneously()
287287

288288
await recordingHandler.StartPlaybackAsync(key, httpContext.Response, Common.RecordingType.InMemory);
289289
var playbackSession = httpContext.Response.Headers["x-recording-id"];
290-
recordingHandler.StopPlayback(playbackSession, false);
290+
await recordingHandler.StopPlayback(playbackSession, false);
291291

292292
Assert.True(1 == recordingHandler.InMemorySessions.Count);
293293
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
using Xunit;
1313
using System.Threading.Tasks;
1414
using System.Security.Cryptography;
15+
using System.Collections.Concurrent;
1516

1617
namespace Azure.Sdk.Tools.TestProxy.Tests
1718
{
@@ -35,6 +36,18 @@ public static class TestHelpers
3536
{
3637
public static readonly string DisableBranchCleanupEnvVar = "DISABLE_INTEGRATION_BRANCH_CLEANUP";
3738

39+
public static List<T> ExhaustQueue<T>(ConcurrentQueue<T> queue)
40+
{
41+
List<T> results = new List<T>();
42+
43+
while (queue.TryDequeue(out var item))
44+
{
45+
results.Add(item);
46+
}
47+
48+
return results;
49+
}
50+
3851
public static string GetValueFromCertificateFile(string certName)
3952
{
4053
var path = Path.Join(Directory.GetCurrentDirectory(), "Test.Certificates", certName);

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs‎

Lines changed: 4 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -158,22 +158,11 @@ public async Task AddSanitizers()
158158
throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items.");
159159
}
160160

161-
var registeredSanitizers = new List<string>();
161+
// we need check if a recording id is present BEFORE the loop, as we want to encapsulate the entire
162+
// sanitizer add operation in a single lock, rather than gathering and releasing a sanitizer lock
163+
// for the session/recording on _each_ sanitizer addition.
162164

163-
// register them all
164-
foreach(var sanitizer in workload)
165-
{
166-
if (recordingId != null)
167-
{
168-
var registeredId = await _recordingHandler.RegisterSanitizer(sanitizer, recordingId);
169-
registeredSanitizers.Add(registeredId);
170-
}
171-
else
172-
{
173-
var registeredId = await _recordingHandler.RegisterSanitizer(sanitizer);
174-
registeredSanitizers.Add(registeredId);
175-
}
176-
}
165+
var registeredSanitizers = await _recordingHandler.RegisterSanitizers(workload, recordingId);
177166

178167
if (recordingId != null)
179168
{

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Audit.cs‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
2+
// Licensed under the MIT License.
3+
4+
using Azure.Sdk.Tools.TestProxy.Common;
5+
using Azure.Sdk.Tools.TestProxy.Common.Exceptions;
6+
using Azure.Sdk.Tools.TestProxy.Store;
7+
using Microsoft.AspNetCore.Http;
8+
using Microsoft.AspNetCore.Mvc;
9+
using Microsoft.Extensions.Logging;
10+
using System;
11+
using System.Collections.Generic;
12+
using System.Net;
13+
using System.Text;
14+
using System.Threading.Tasks;
15+
16+
namespace Azure.Sdk.Tools.TestProxy
17+
{
18+
[ApiController]
19+
[Route("[controller]/[action]")]
20+
public sealed class Audit : ControllerBase
21+
{
22+
private readonly RecordingHandler _recordingHandler;
23+
24+
public Audit(RecordingHandler recordingHandler)
25+
{
26+
_recordingHandler = recordingHandler;
27+
}
28+
29+
[HttpGet]
30+
public async Task Logs()
31+
{
32+
33+
var allAuditSessions = _recordingHandler.RetrieveOngoingAuditLogs();
34+
allAuditSessions.AddRange(_recordingHandler.AuditSessions.Values);
35+
36+
StringBuilder stringBuilder = new StringBuilder();
37+
38+
foreach (var auditLogQueue in allAuditSessions) {
39+
while (auditLogQueue.TryDequeue(out var logItem))
40+
{
41+
stringBuilder.Append(logItem.ToCsvString() + Environment.NewLine);
42+
}
43+
}
44+
45+
Response.ContentType = "text/plain";
46+
47+
await Response.WriteAsync(stringBuilder.ToString());
48+
}
49+
}
50+
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/AuditLogItem.cs‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
using System;
2+
3+
namespace Azure.Sdk.Tools.TestProxy.Common
4+
{
5+
public class AuditLogItem
6+
{
7+
public string RecordingId { get; set; }
8+
9+
public DateTime Timestamp { get; set; }
10+
11+
public string Uri { get; set; }
12+
13+
public string Verb { get; set; }
14+
15+
public string Message { get; set; }
16+
17+
public AuditLogItem(string recordingId, string requestUri, string requestMethod) {
18+
RecordingId = recordingId;
19+
Timestamp = DateTime.UtcNow;
20+
21+
Uri = requestUri;
22+
Verb = requestMethod;
23+
}
24+
25+
public string ToCsvString()
26+
{
27+
return $"{RecordingId},{Timestamp.ToString("o")},{Verb},{Uri},{Message}";
28+
}
29+
30+
public AuditLogItem(string recordingId, string message)
31+
{
32+
RecordingId = recordingId;
33+
Timestamp = DateTime.UtcNow;
34+
35+
Message = message;
36+
}
37+
38+
}
39+
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/DebugLogger.cs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
using System.Collections.Generic;
66
using System.Text.Json;
77
using Microsoft.AspNetCore.Http.Extensions;
8+
using System.Collections.Concurrent;
89

910
namespace Azure.Sdk.Tools.TestProxy.Common
1011
{

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ModifiableRecordSession.cs‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
using System;
2+
using System.Collections.Concurrent;
23
using System.Collections.Generic;
34
using System.Linq;
45
using System.Net.Http;
@@ -9,13 +10,13 @@ namespace Azure.Sdk.Tools.TestProxy.Common
910
{
1011
public class ModifiableRecordSession
1112
{
12-
public RecordMatcher CustomMatcher { get; set;}
13+
public RecordMatcher CustomMatcher { get; set; }
1314

1415
public RecordSession Session { get; }
1516

1617
public ModifiableRecordSession(SanitizerDictionary sanitizerRegistry, string sessionId)
1718
{
18-
lock(sanitizerRegistry.SessionSanitizerLock)
19+
lock (sanitizerRegistry.SessionSanitizerLock)
1920
{
2021
this.AppliedSanitizers = sanitizerRegistry.SessionSanitizers.ToList();
2122
}
@@ -42,18 +43,17 @@ public ModifiableRecordSession(RecordSession session, SanitizerDictionary saniti
4243

4344
public List<ResponseTransform> AdditionalTransforms { get; } = new List<ResponseTransform>();
4445

45-
public SemaphoreSlim SanitizerLock = new SemaphoreSlim(1);
46-
4746
public List<string> AppliedSanitizers { get; set; } = new List<string>();
4847
public List<string> ForRemoval { get; } = new List<string>();
4948

5049
public string SourceRecordingId { get; set; }
5150

5251
public int PlaybackResponseTime { get; set; }
5352

53+
public ConcurrentQueue<AuditLogItem> AuditLog { get; set; } = new ConcurrentQueue<AuditLogItem>();
5454
public async void ResetExtensions(SanitizerDictionary sanitizerDictionary)
5555
{
56-
await SanitizerLock.WaitAsync();
56+
await Session.EntryLock.WaitAsync();
5757
try
5858
{
5959
AdditionalTransforms.Clear();
@@ -66,7 +66,7 @@ public async void ResetExtensions(SanitizerDictionary sanitizerDictionary)
6666
}
6767
finally
6868
{
69-
SanitizerLock.Release();
69+
Session.EntryLock.Release();
7070
}
7171
}
7272
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/RecordSession.cs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,6 @@
33

44
using System;
55
using System.Collections.Generic;
6-
using System.IO;
7-
using System.Linq;
86
using System.Text.Json;
97
using System.Threading;
108
using System.Threading.Tasks;

0 commit comments

Comments
 (0)
⚡