Skip to content

Commit 3e0781e

Browse files
scbeddscbedd
authored
Removable Sanitizers (#8120)
* Sanitizers are now removable via API Route * Added route Admin/GetSanitizers to retrieve applied sanitizers for the session level * optionally honors x-recording-id header to retrieve applied sanitizers for a specific recording/playback session * Added route Admin/RemoveSanitizers to * Accepts an object of form { Sanitizers: [ "id1", "id2" ] } and attempts to remove the sanitizers there * optionally honors x-recording-id header to remove applied sanitizers for a specific recording/playback session * Updated Admin/AddSanitizer and Admin/AddSanitizers to return the ids that have been registered
1 parent 793837c commit 3e0781e

14 files changed

Lines changed: 727 additions & 110 deletions

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs‎

Lines changed: 264 additions & 37 deletions
Large diffs are not rendered by default.

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/InfoTests.cs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -47,8 +47,8 @@ public void TestReflectionModelWithAdvancedType()
4747
{
4848
RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
4949
var httpContext = new DefaultHttpContext();
50-
testRecordingHandler.Sanitizers.Clear();
51-
testRecordingHandler.Sanitizers.Add(new GeneralRegexSanitizer(value: "A new value", condition: new ApplyCondition() { UriRegex= ".+/Tables" }));
50+
testRecordingHandler.SanitizerRegistry.Clear();
51+
testRecordingHandler.SanitizerRegistry.Register(new GeneralRegexSanitizer(value: "A new value", condition: new ApplyCondition() { UriRegex= ".+/Tables" }));
5252

5353
var controller = new Info(testRecordingHandler)
5454
{
@@ -72,8 +72,8 @@ public async Task TestReflectionModelWithTargetRecordSession()
7272

7373
var recordingId = httpContext.Response.Headers["x-recording-id"].ToString();
7474

75-
testRecordingHandler.AddSanitizerToRecording(recordingId, new UriRegexSanitizer(regex: "ABC123"));
76-
testRecordingHandler.AddSanitizerToRecording(recordingId, new BodyRegexSanitizer(regex: ".+?"));
75+
testRecordingHandler.RegisterSanitizer(new UriRegexSanitizer(regex: "ABC123"), recordingId);
76+
testRecordingHandler.RegisterSanitizer(new BodyRegexSanitizer(regex: ".+?"), recordingId);
7777
testRecordingHandler.SetMatcherForRecording(recordingId, new CustomDefaultMatcher(compareBodies: false, excludedHeaders: "an-excluded-header"));
7878

7979
var model = new ActiveMetadataModel(testRecordingHandler, recordingId);

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/LoggingTests.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ public async Task PlaybackLogsSanitizedRequest()
4747
Assert.True(testRecordingHandler.PlaybackSessions.ContainsKey(recordingId));
4848
var entry = testRecordingHandler.PlaybackSessions[recordingId].Session.Entries[0];
4949
HttpRequest request = TestHelpers.CreateRequestFromEntry(entry);
50-
request.Headers["Authorization"] = "fake-auth-header";
50+
request.Headers["Authorization"] = "Sanitized";
5151

5252
HttpResponse response = new DefaultHttpContext().Response;
5353
await testRecordingHandler.HandlePlaybackRequest(recordingId, request, response);

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordingHandlerTests.cs‎

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
using System.Runtime.InteropServices;
2424
using Azure.Sdk.Tools.TestProxy.Common.Exceptions;
2525
using Azure.Sdk.Tools.TestProxy.Store;
26+
using Microsoft.VisualStudio.TestPlatform.ObjectModel.Adapter;
2627

2728
namespace Azure.Sdk.Tools.TestProxy.Tests
2829
{
@@ -96,10 +97,11 @@ private void _checkDefaultExtensions(RecordingHandler handlerForTest, CheckSkips
9697

9798
if (skipsToCheck.HasFlag(CheckSkips.IncludeSanitizers))
9899
{
99-
Assert.Equal(3, handlerForTest.Sanitizers.Count);
100-
Assert.IsType<RecordedTestSanitizer>(handlerForTest.Sanitizers[0]);
101-
Assert.IsType<BodyKeySanitizer>(handlerForTest.Sanitizers[1]);
102-
Assert.IsType<BodyKeySanitizer>(handlerForTest.Sanitizers[2]);
100+
var sessionSanitizers = handlerForTest.SanitizerRegistry.GetSanitizers();
101+
Assert.Equal(3, sessionSanitizers.Count);
102+
Assert.IsType<RecordedTestSanitizer>(sessionSanitizers[0]);
103+
Assert.IsType<BodyKeySanitizer>(sessionSanitizers[1]);
104+
Assert.IsType<BodyKeySanitizer>(sessionSanitizers[2]);
103105
}
104106
}
105107
#endregion
@@ -166,7 +168,7 @@ public void TestResetAfterAddition()
166168
RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
167169

168170
// act
169-
testRecordingHandler.Sanitizers.Add(new BodyRegexSanitizer("sanitized", ".*"));
171+
testRecordingHandler.SanitizerRegistry.Register(new BodyRegexSanitizer("sanitized", ".*"));
170172
testRecordingHandler.Matcher = new BodilessMatcher();
171173
testRecordingHandler.Transforms.Add(new ApiVersionTransform());
172174
testRecordingHandler.SetDefaultExtensions();
@@ -182,7 +184,7 @@ public void TestResetAfterRemoval()
182184
RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
183185

184186
// act
185-
testRecordingHandler.Sanitizers.Clear();
187+
testRecordingHandler.SanitizerRegistry.Clear();
186188
testRecordingHandler.Matcher = null;
187189
testRecordingHandler.Transforms.Clear();
188190
testRecordingHandler.SetDefaultExtensions();
@@ -199,18 +201,19 @@ public async Task TestResetTargetsRecordingOnly()
199201
await testRecordingHandler.StartRecordingAsync("recordingings/cool.json", httpContext.Response);
200202
var recordingId = httpContext.Response.Headers["x-recording-id"].ToString();
201203

202-
203-
testRecordingHandler.Sanitizers.Clear();
204-
testRecordingHandler.Sanitizers.Add(new BodyRegexSanitizer("sanitized", ".*"));
205-
testRecordingHandler.AddSanitizerToRecording(recordingId, new GeneralRegexSanitizer("sanitized", ".*"));
204+
testRecordingHandler.SanitizerRegistry.Clear();
205+
testRecordingHandler.SanitizerRegistry.Register(new BodyRegexSanitizer("sanitized", ".*"));
206+
testRecordingHandler.RegisterSanitizer(new GeneralRegexSanitizer("sanitized", ".*"), recordingId);
206207
testRecordingHandler.SetDefaultExtensions(recordingId);
207208
var session = testRecordingHandler.RecordingSessions.First().Value;
209+
var recordingSanitizers = testRecordingHandler.SanitizerRegistry.GetSanitizers(session);
210+
var sessionSanitizers = testRecordingHandler.SanitizerRegistry.GetSanitizers();
208211

209212
// session sanitizer is still set to a single one
210-
Assert.Single(testRecordingHandler.Sanitizers);
211-
Assert.IsType<BodyRegexSanitizer>(testRecordingHandler.Sanitizers[0]);
213+
Assert.Single(sessionSanitizers);
214+
Assert.IsType<BodyRegexSanitizer>(sessionSanitizers[0]);
212215
_checkDefaultExtensions(testRecordingHandler, CheckSkips.IncludeMatcher | CheckSkips.IncludeTransforms);
213-
Assert.Empty(session.AdditionalSanitizers);
216+
Assert.Equal(session.AppliedSanitizers, testRecordingHandler.SanitizerRegistry.SessionSanitizers);
214217
Assert.Empty(session.AdditionalTransforms);
215218
Assert.Null(session.CustomMatcher);
216219
}
@@ -223,22 +226,22 @@ public async Task TestResetTargetsSessionOnly()
223226
await testRecordingHandler.StartRecordingAsync("recordingings/cool.json", httpContext.Response);
224227
var recordingId = httpContext.Response.Headers["x-recording-id"].ToString();
225228

226-
testRecordingHandler.Sanitizers.Clear();
227-
testRecordingHandler.Sanitizers.Add(new BodyRegexSanitizer("sanitized", ".*"));
229+
testRecordingHandler.SanitizerRegistry.Clear();
230+
testRecordingHandler.SanitizerRegistry.Register(new BodyRegexSanitizer("sanitized", ".*"));
228231
testRecordingHandler.Transforms.Clear();
229-
testRecordingHandler.AddSanitizerToRecording(recordingId, new GeneralRegexSanitizer("sanitized", ".*"));
232+
testRecordingHandler.RegisterSanitizer(new GeneralRegexSanitizer("sanitized", ".*"), recordingId);
230233
testRecordingHandler.SetDefaultExtensions(recordingId);
231234
var session = testRecordingHandler.RecordingSessions.First().Value;
232235

233236
// check that the individual session had reset sanitizers
234-
Assert.Empty(session.AdditionalSanitizers);
237+
Assert.Equal(testRecordingHandler.SanitizerRegistry.GetSanitizers(), testRecordingHandler.SanitizerRegistry.GetSanitizers(session));
235238

236239
// stop the recording to clear out the session cache
237240
testRecordingHandler.StopRecording(recordingId);
238241

239242
// then verify that the session level is NOT reset.
240-
Assert.Single(testRecordingHandler.Sanitizers);
241-
Assert.IsType<BodyRegexSanitizer>(testRecordingHandler.Sanitizers.First());
243+
Assert.Single(testRecordingHandler.SanitizerRegistry.GetSanitizers());
244+
Assert.IsType<BodyRegexSanitizer>(testRecordingHandler.SanitizerRegistry.GetSanitizers().First());
242245
}
243246

244247
[Fact]

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ public async Task RegexEntrySanitizerCreatesOverAPI(string body)
125125
{
126126

127127
RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
128-
testRecordingHandler.Sanitizers.Clear();
128+
testRecordingHandler.SanitizerRegistry.Clear();
129129
var httpContext = new DefaultHttpContext();
130130
httpContext.Request.Headers["x-abstraction-identifier"] = "RegexEntrySanitizer";
131131
httpContext.Request.Body = TestHelpers.GenerateStreamRequestBody(body);
@@ -142,7 +142,7 @@ public async Task RegexEntrySanitizerCreatesOverAPI(string body)
142142
};
143143

144144
await controller.AddSanitizer();
145-
var sanitizer = testRecordingHandler.Sanitizers[0];
145+
var sanitizer = testRecordingHandler.SanitizerRegistry.GetSanitizers()[0];
146146
Assert.True(sanitizer is RegexEntrySanitizer);
147147

148148

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/TestHelpers.cs‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -83,15 +83,15 @@ public static ModifiableRecordSession LoadRecordSession(string path)
8383
using var stream = System.IO.File.OpenRead(path);
8484
using var doc = JsonDocument.Parse(stream);
8585

86-
return new ModifiableRecordSession(RecordSession.Deserialize(doc.RootElement));
86+
return new ModifiableRecordSession(RecordSession.Deserialize(doc.RootElement), new SanitizerDictionary(), Guid.NewGuid().ToString());
8787
}
8888

8989
public static RecordingHandler LoadRecordSessionIntoInMemoryStore(string path)
9090
{
9191
using var stream = System.IO.File.OpenRead(path);
9292
using var doc = JsonDocument.Parse(stream);
9393
var guid = Guid.NewGuid().ToString();
94-
var session = new ModifiableRecordSession(RecordSession.Deserialize(doc.RootElement));
94+
var session = new ModifiableRecordSession(RecordSession.Deserialize(doc.RootElement), new SanitizerDictionary(), Guid.NewGuid().ToString());
9595

9696
RecordingHandler handler = new RecordingHandler(Directory.GetCurrentDirectory());
9797
handler.InMemorySessions.TryAdd(guid, session);
@@ -516,5 +516,24 @@ public static bool CheckExistenceOfTag(Assets assets, string workingDirectory)
516516
CommandResult result = GitHandler.Run($"ls-remote {cloneUrl} --tags {assets.Tag}", workingDirectory);
517517
return result.StdOut.Trim().Length > 0;
518518
}
519+
520+
public static List<T> EnumerateArray<T>(JsonElement element)
521+
{
522+
List<T> values = new List<T>();
523+
524+
if (element.ValueKind.ToString() != "Array")
525+
{
526+
throw new Exception("This test helper is intended for array members only");
527+
}
528+
else
529+
{
530+
foreach(var item in element.EnumerateArray())
531+
{
532+
values.Add(item.Deserialize<T>());
533+
}
534+
}
535+
536+
return values;
537+
}
519538
}
520539
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Admin.cs‎

Lines changed: 90 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,13 @@
44
using Azure.Sdk.Tools.TestProxy.Common;
55
using Azure.Sdk.Tools.TestProxy.Common.Exceptions;
66
using Azure.Sdk.Tools.TestProxy.Store;
7+
using Microsoft.AspNetCore.Http;
78
using Microsoft.AspNetCore.Mvc;
89
using Microsoft.Extensions.Logging;
910
using System;
1011
using System.Collections.Generic;
1112
using System.Linq;
13+
using System.Linq.Expressions;
1214
using System.Net;
1315
using System.Text.Json;
1416
using System.Threading.Tasks;
@@ -63,6 +65,73 @@ public async Task AddTransform()
6365
}
6466
}
6567

68+
[HttpPost]
69+
public async Task RemoveSanitizers([FromBody]RemoveSanitizerList sanitizerList)
70+
{
71+
DebugLogger.LogAdminRequestDetails(_logger, Request);
72+
var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true);
73+
74+
var removedSanitizers = new List<string>();
75+
var exceptionsList = new List<string>();
76+
77+
if (sanitizerList.Sanitizers.Count == 0)
78+
{
79+
throw new HttpException(HttpStatusCode.BadRequest, "At least one sanitizerId for removal must be provided.");
80+
}
81+
82+
foreach(var sanitizerId in sanitizerList.Sanitizers) {
83+
try
84+
{
85+
var removedId = _recordingHandler.UnregisterSanitizer(sanitizerId, recordingId);
86+
removedSanitizers.Add(sanitizerId);
87+
}
88+
catch (HttpException ex) {
89+
exceptionsList.Add(ex.Message);
90+
}
91+
}
92+
93+
if (exceptionsList.Count > 0)
94+
{
95+
var varExceptionMessage = $"Unable to remove {exceptionsList.Count} sanitizer{(exceptionsList.Count > 1 ? 's' : string.Empty)}. Detailed list follows: \n"
96+
+ string.Join("\n", exceptionsList);
97+
throw new HttpException(HttpStatusCode.BadRequest, varExceptionMessage);
98+
}
99+
else
100+
{
101+
var json = JsonSerializer.Serialize(new { Removed = removedSanitizers });
102+
103+
Response.ContentType = "application/json";
104+
Response.ContentLength = json.Length;
105+
106+
await Response.WriteAsync(json);
107+
}
108+
}
109+
110+
[HttpGet]
111+
public async Task GetSanitizers()
112+
{
113+
DebugLogger.LogAdminRequestDetails(_logger, Request);
114+
var recordingId = RecordingHandler.GetHeader(Request, "x-recording-id", allowNulls: true);
115+
116+
List<RegisteredSanitizer> sanitizers;
117+
118+
if (!string.IsNullOrEmpty(recordingId))
119+
{
120+
var session = _recordingHandler.GetActiveSession(recordingId);
121+
sanitizers = _recordingHandler.SanitizerRegistry.GetRegisteredSanitizers(session);
122+
}
123+
else
124+
{
125+
sanitizers = _recordingHandler.SanitizerRegistry.GetRegisteredSanitizers();
126+
}
127+
128+
var json = JsonSerializer.Serialize(new { Sanitizers = sanitizers });
129+
Response.ContentType = "application/json";
130+
Response.ContentLength = json.Length;
131+
132+
await Response.WriteAsync(json);
133+
}
134+
66135
[HttpPost]
67136
public async Task AddSanitizer()
68137
{
@@ -72,14 +141,22 @@ public async Task AddSanitizer()
72141

73142
RecordedTestSanitizer s = (RecordedTestSanitizer)GetSanitizer(sName, await HttpRequestInteractions.GetBody(Request));
74143

144+
string registeredSanitizerId;
145+
75146
if (recordingId != null)
76147
{
77-
_recordingHandler.AddSanitizerToRecording(recordingId, s);
148+
registeredSanitizerId = _recordingHandler.RegisterSanitizer(s, recordingId);
78149
}
79150
else
80151
{
81-
_recordingHandler.Sanitizers.Add(s);
152+
registeredSanitizerId = _recordingHandler.RegisterSanitizer(s);
82153
}
154+
155+
var json = JsonSerializer.Serialize(new { Sanitizer = registeredSanitizerId });
156+
Response.ContentType = "application/json";
157+
Response.ContentLength = json.Length;
158+
159+
await Response.WriteAsync(json);
83160
}
84161

85162
[HttpPost]
@@ -96,19 +173,28 @@ public async Task AddSanitizers()
96173
throw new HttpException(HttpStatusCode.BadRequest, "When bulk adding sanitizers, ensure there is at least one sanitizer added in each batch. Received 0 work items.");
97174
}
98175

176+
var registeredSanitizers = new List<string>();
177+
99178
// register them all
100179
foreach(var sanitizer in workload)
101180
{
102181
if (recordingId != null)
103182
{
104-
_recordingHandler.AddSanitizerToRecording(recordingId, sanitizer);
183+
var registeredId = _recordingHandler.RegisterSanitizer(sanitizer, recordingId);
184+
registeredSanitizers.Add(registeredId);
185+
Response.Headers.Add("x-recording-id", recordingId);
105186
}
106187
else
107188
{
108-
_recordingHandler.Sanitizers.Add(sanitizer);
189+
var registeredId = _recordingHandler.RegisterSanitizer(sanitizer);
190+
registeredSanitizers.Add(registeredId);
109191
}
110192
}
193+
var json = JsonSerializer.Serialize(new { Sanitizers = registeredSanitizers });
194+
Response.ContentType = "application/json";
195+
Response.ContentLength = json.Length;
111196

197+
await Response.WriteAsync(json);
112198
}
113199

114200

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/ModifiableRecordSession.cs‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,27 +12,41 @@ public class ModifiableRecordSession
1212

1313
public RecordSession Session { get; }
1414

15-
public ModifiableRecordSession(RecordSession session)
15+
public ModifiableRecordSession(SanitizerDictionary sanitizerRegistry, string sessionId)
16+
{
17+
this.AppliedSanitizers = sanitizerRegistry.SessionSanitizers.ToList();
18+
this.SessionId = sessionId;
19+
}
20+
21+
public ModifiableRecordSession(RecordSession session, SanitizerDictionary sanitizerRegistry, string sessionId)
1622
{
1723
Session = session;
24+
this.AppliedSanitizers = sanitizerRegistry.SessionSanitizers.ToList();
25+
this.SessionId = sessionId;
1826
}
1927

28+
public string SessionId;
29+
2030
public string Path { get; set; }
2131

2232
public HttpClient Client { get; set; }
2333

2434
public List<ResponseTransform> AdditionalTransforms { get; } = new List<ResponseTransform>();
2535

26-
public List<RecordedTestSanitizer> AdditionalSanitizers { get; }= new List<RecordedTestSanitizer>();
36+
public List<string> AppliedSanitizers { get; set; } = new List<string>();
37+
public List<string> ForRemoval { get; } = new List<string>();
2738

2839
public string SourceRecordingId { get; set; }
2940

3041
public int PlaybackResponseTime { get; set; }
3142

32-
public void ResetExtensions()
43+
public void ResetExtensions(SanitizerDictionary sanitizerDictionary)
3344
{
3445
AdditionalTransforms.Clear();
35-
AdditionalSanitizers.Clear();
46+
AppliedSanitizers = new List<string>();
47+
AppliedSanitizers.AddRange(sanitizerDictionary.SessionSanitizers);
48+
ForRemoval.Clear();
49+
3650
CustomMatcher = null;
3751
Client = null;
3852
}

‎tools/test-proxy/Azure.Sdk.Tools.TestProxy/Common/RecordedTestSanitizer.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// Copyright (c) Microsoft Corporation. All rights reserved.
1+
// Copyright (c) Microsoft Corporation. All rights reserved.
22
// Licensed under the MIT License.
33

44
using Azure.Core;
@@ -21,7 +21,6 @@ public class RecordedTestSanitizer
2121

2222
public ApplyCondition Condition { get; protected set; } = null;
2323

24-
2524
/// This is just a temporary workaround to avoid breaking tests that need to be re-recorded
2625
// when updating the JsonPathSanitizer logic to avoid changing date formats when deserializing requests.
2726
// this property will be removed in the future.

0 commit comments

Comments
 (0)
⚡