[TestProxy] Common Sanitizers (#8038)

- Each recording or playback session now gets a modifiable list of sanitizers that are applied.
 - Session sanitizers are now cut at playback or record start. Any further session sanitizer addition will not directly affect the recording that has been started.
- As a consequence of all these additional sanitizers, stored recordings will now be sanitized. The way this will work is
  - Client starts playback session, gets a recordingId back
  - Client uses recordingId to add a couple additional sanitizers
  - Client starts firing actual requests from the test
- The proxy will sanitize the loaded recording according to which sanitizers are active when the first real request for the playback session occurs.
 - Upon calling /Admin/AddSanitizers or Admin/AddSanitizer, an object or list of objects representing the registered sanitizers is now returned to the calling client. These objects - contain the base sanitizer properties + the id of the registered sanitizer.
 - Added route /Admin/RemoveSanitizers which can apply to either the session-level or an individual playback/record session.

Author: scbedd

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/AdminTests.cs

@@ -1026,7 +1026,7 @@ public async Task RemoveSanitizerSucceedsForExistingSessionSanitizer()
             };
 
             var expectedSanitizerCount = testRecordingHandler.SanitizerRegistry.GetSanitizers().Count;
-            await controller.RemoveSanitizers(new RemoveSanitizerList() { Sanitizers = new List<string>() { "AZSDK001" } });
+            await controller.RemoveSanitizers(new RemoveSanitizerList() { Sanitizers = new List<string>() { "AZSDK0000" } });
 
             httpContext.Response.Body.Seek(0, SeekOrigin.Begin);
             var response = await JsonDocument.ParseAsync(httpContext.Response.Body, options: new JsonDocumentOptions() { AllowTrailingCommas = true });
@@ -1036,7 +1036,7 @@ public async Task RemoveSanitizerSucceedsForExistingSessionSanitizer()
             var returnedSanitizerIds = TestHelpers.EnumerateArray<string>(prop);
 
             Assert.Single(returnedSanitizerIds);
-            Assert.Equal("AZSDK001", returnedSanitizerIds.First());
+            Assert.Equal("AZSDK0000", returnedSanitizerIds.First());
 
             Assert.Equal(expectedSanitizerCount - 1, testRecordingHandler.SanitizerRegistry.GetSanitizers().Count);
         }

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/InfoTests.cs

@@ -18,6 +18,8 @@ namespace Azure.Sdk.Tools.TestProxy.Tests
 {
     public class InfoTests
     {
+        private int DefaultExtensionCount { get { return new RecordingHandler(null).SanitizerRegistry.GetSanitizers().Count; } }
+
         [Fact]
         public void TestReflectionModelBuild()
         {
@@ -79,17 +81,17 @@ public async Task TestReflectionModelWithTargetRecordSession()
             var model = new ActiveMetadataModel(testRecordingHandler, recordingId);
             var descriptions = model.Descriptions.ToList();
 
-            // we should have exactly 6 if we're counting all the customizations appropriately
-            Assert.True(descriptions.Count == 6);
+            // we should have exactly DefaultExtensionCount + 2 if we're counting all the customizations appropriately
+            Assert.True(descriptions.Count == DefaultExtensionCount + 3);
             Assert.True(model.Matchers.Count() == 1);
-            Assert.True(model.Sanitizers.Count() == 5);
+            Assert.True(model.Sanitizers.Count() == DefaultExtensionCount + 2);
 
             // confirm that the overridden matcher is showing up
-            Assert.True(descriptions[3].ConstructorDetails.Arguments[1].Item2 == "\"ABC123\"");
-            Assert.True(descriptions[4].ConstructorDetails.Arguments[1].Item2 == "\".+?\"");
+            Assert.True(descriptions[DefaultExtensionCount].ConstructorDetails.Arguments[1].Item2 == "\"ABC123\"");
+            Assert.True(descriptions[DefaultExtensionCount + 1].ConstructorDetails.Arguments[1].Item2 == "\".+?\"");
 
             // and finally confirm our sanitizers are what we expect
-            Assert.True(descriptions[5].Name == "CustomDefaultMatcher");
+            Assert.True(descriptions[DefaultExtensionCount + 2].Name == "CustomDefaultMatcher");
         }
     }
 }

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/LoggingTests.cs

@@ -52,7 +52,7 @@ public async Task PlaybackLogsSanitizedRequest()
                 HttpResponse response = new DefaultHttpContext().Response;
                 await testRecordingHandler.HandlePlaybackRequest(recordingId, request, response);
 
-                AssertLogs(logger, 4, 8);
+                AssertLogs(logger, 4, 8, 12);
             }
             finally
             {
@@ -92,7 +92,7 @@ public async Task RecordingHandlerLogsSanitizedRequests()
 
             try
             {
-                AssertLogs(logger, 2, 8);
+                AssertLogs(logger, 2, 8, 12);
             }
             finally
             {
@@ -101,7 +101,7 @@ public async Task RecordingHandlerLogsSanitizedRequests()
             }
         }
 
-        private static void AssertLogs(TestLogger logger, int offset, int expectedLength)
+        private static void AssertLogs(TestLogger logger, int offset, int expectedLength, int expectedContentLength)
         {
             Assert.Equal(expectedLength, logger.Logs.Count);
             Assert.Equal(
@@ -116,10 +116,10 @@ private static void AssertLogs(TestLogger logger, int offset, int expectedLength
             Assert.Equal(
                 "Request Body Content{\"key\":\"Location\",\"value\":\"https://fakeazsdktestaccount.table.core.windows.net/Tables\"}",
                 logger.Logs[2 + offset].ToString());
-            Assert.Equal("URI: [ https://fakeazsdktestaccount.table.core.windows.net/Tables]" +
+            Assert.Equal("URI: [ https://Sanitized.table.core.windows.net/Tables]" +
                          Environment.NewLine + "Headers: [{\"Accept\":[\"application/json;odata=minimalmetadata\"],\"Accept-Encoding\":[\"gzip, deflate\"],\"Authorization\":[\"Sanitized\"],\"Connection\":[\"keep-alive\"]," +
-                         "\"Content-Length\":[\"12\"],\"Content-Type\":[\"application/octet-stream\"],\"DataServiceVersion\":[\"3.0\"],\"Date\":[\"Tue, 18 May 2021 23:27:42 GMT\"]," +
-                         "\"User-Agent\":[\"azsdk-python-data-tables/12.0.0b7 Python/3.8.6 (Windows-10-10.0.19041-SP0)\"],\"x-ms-client-request-id\":[\"a4c24b7a-b830-11eb-a05e-10e7c6392c5a\"]," +
+                         "\"Content-Length\":[\"" + expectedContentLength + "\"],\"Content-Type\":[\"application/octet-stream\"],\"DataServiceVersion\":[\"3.0\"],\"Date\":[\"Tue, 18 May 2021 23:27:42 GMT\"]," +
+                         "\"User-Agent\":[\"azsdk-python-data-tables/12.0.0b7 Python/3.8.6 (Windows-10-10.0.19041-SP0)\"],\"x-ms-client-request-id\":[\"Sanitized\"]," +
                          "\"x-ms-date\":[\"Tue, 18 May 2021 23:27:42 GMT\"],\"x-ms-version\":[\"2019-02-02\"]}]" + Environment.NewLine,
                 logger.Logs[3 + offset].ToString());
         }

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/MatcherTests.cs

@@ -1,4 +1,4 @@
-using Azure.Sdk.Tools.TestProxy.Common;
+using Azure.Sdk.Tools.TestProxy.Common;
 using Azure.Sdk.Tools.TestProxy.Matchers;
 using Microsoft.AspNetCore.Http;
 using Microsoft.AspNetCore.Http.Features;
@@ -299,6 +299,50 @@ public async Task CustomMatcherMatchesDifferentUriOrder()
             await testRecordingHandler.HandlePlaybackRequest(recordingId, playbackContext.Request, playbackContext.Response);
             Assert.Equal("WESTUS:20210909T204819Z:f9a33867-6efc-4748-b322-303b2b933466", playbackContext.Response.Headers["x-ms-routing-request-id"].ToString());
         }
+
+
+        [Fact]
+        public async Task EncodedUriAmpersandWorksCrossplat()
+        {
+            RecordingHandler testRecordingHandler = new RecordingHandler(Directory.GetCurrentDirectory());
+            testRecordingHandler.Matcher = new CustomDefaultMatcher(ignoreQueryOrdering: true);
+            var playbackContext = new DefaultHttpContext();
+            var targetFile = "Test.RecordEntries/request_with_encoded_ampersand.json";
+            var body = "{\"x-recording-file\":\"" + targetFile + "\"}";
+            playbackContext.Request.Body = TestHelpers.GenerateStreamRequestBody(body);
+            playbackContext.Request.ContentLength = body.Length;
+
+            var controller = new Playback(testRecordingHandler, new NullLoggerFactory())
+            {
+                ControllerContext = new ControllerContext()
+                {
+                    HttpContext = playbackContext
+                }
+            };
+            await controller.Start();
+            var recordingId = playbackContext.Response.Headers["x-recording-id"].ToString();
+
+            playbackContext.Request.Headers.Clear();
+            playbackContext.Response.Headers.Clear();
+            playbackContext.Request.Method = "GET";
+
+            var requestHeaders = new Dictionary<string, string>(){
+                { "x-recording-id", recordingId },
+                { "x-recording-upstream-base-uri", "https://REDACTED" }
+            };
+            foreach (var kvp in requestHeaders)
+            {
+                playbackContext.Request.Headers.Add(kvp.Key, kvp.Value);
+            }
+            var queryString = "?api-version=1.0&year=2023&basinId=AL&govId=5";
+            var path = "/weather/tropical/storms/json";
+            playbackContext.Request.Host = new HostString("https://localhost:5001");
+            playbackContext.Features.Get<IHttpRequestFeature>().RawTarget = path + queryString;
+            await testRecordingHandler.HandlePlaybackRequest(recordingId, playbackContext.Request, playbackContext.Response);
+
+            Assert.Equal("Ref A: 980665086A12483993E2782EDFC9F29A Ref B: STBEDGE0106 Ref C: 2023-07-19T22:52:17Z", playbackContext.Response.Headers["X-MSEdge-Ref"].ToString());
+            Assert.Equal(200, playbackContext.Response.StatusCode);
+        }
     }
 }
 

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/RecordingHandlerTests.cs

@@ -29,6 +29,8 @@ namespace Azure.Sdk.Tools.TestProxy.Tests
 {
     public class RecordingHandlerTests
     {
+        private int DefaultExtensionCount { get { return new RecordingHandler(null).SanitizerRegistry.DefaultSanitizerList.Count; } }
+
         #region helpers and private test fields
         private HttpContext GenerateHttpRequestContext(string[] headerValueStrings)
         {
@@ -97,11 +99,14 @@ private void _checkDefaultExtensions(RecordingHandler handlerForTest, CheckSkips
 
             if (skipsToCheck.HasFlag(CheckSkips.IncludeSanitizers))
             {
-                var sessionSanitizers = handlerForTest.SanitizerRegistry.GetSanitizers();
-                Assert.Equal(3, sessionSanitizers.Count);
-                Assert.IsType<RecordedTestSanitizer>(sessionSanitizers[0]);
-                Assert.IsType<BodyKeySanitizer>(sessionSanitizers[1]);
-                Assert.IsType<BodyKeySanitizer>(sessionSanitizers[2]);
+                
+                var sanitizers = handlerForTest.SanitizerRegistry.GetSanitizers();
+
+                Assert.Equal(DefaultExtensionCount, sanitizers.Count);
+                Assert.IsType<RecordedTestSanitizer>(sanitizers[0]);
+                Assert.IsType<GeneralRegexSanitizer>(sanitizers[1]);
+                Assert.IsType<GeneralRegexSanitizer>(sanitizers[2]);
+                Assert.IsType<BodyKeySanitizer>(sanitizers[108]);
             }
         }
         #endregion
@@ -445,7 +450,7 @@ public async Task TestCanSkipRecordingEntireRequestResponse()
                 var record = RecordSession.Deserialize(doc.RootElement);
                 Assert.Single(record.Entries);
                 var entry = record.Entries.First();
-                Assert.Equal("value", JsonDocument.Parse(entry.Request.Body).RootElement.GetProperty("key").GetString());
+                Assert.Equal("Sanitized", JsonDocument.Parse(entry.Request.Body).RootElement.GetProperty("key").GetString());
                 Assert.Equal(MockHttpHandler.DefaultResponse, Encoding.UTF8.GetString(entry.Response.Body));
             }
             finally

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/SanitizerTests.cs

@@ -36,6 +36,33 @@ public void OauthResponseSanitizerCleansV2AuthRequest()
             Assert.Empty(session.Session.Entries);
         }
 
+        [Fact]
+        public void SanitizerDecodesUnicodeAmpersandSanitizesClientIdAndSecret()
+        {
+            var session = TestHelpers.LoadRecordSession("Test.RecordEntries/request_with_encoding.json");
+
+            var clientSan = new BodyRegexSanitizer(regex: "(client_id=)(?<cid>[^&\\\"]+)", groupForReplace: "cid");
+            var secretSan = new BodyRegexSanitizer(regex: "client_secret=(?<secret>[^&\\\"]+)", groupForReplace: "secret");
+
+            session.Session.Sanitize(clientSan);
+            session.Session.Sanitize(secretSan);
+
+            Assert.Equal("client_id=Sanitized&grant_type=client_credentials&client_info=1&client_secret=Sanitized&claims=%7B%22access_token=blahblah", Encoding.UTF8.GetString(session.Session.Entries[0].Request.Body));
+        }
+
+        [Fact]
+        public void EnsureSASCleanupDoesntOverrunInXML()
+        {
+            var sanitizerDictionary = new SanitizerDictionary();
+            var sessionwithXmlBody = TestHelpers.LoadRecordSession("Test.RecordEntries/xml_body_with_sas_present.json");
+
+            Assert.True(sanitizerDictionary.Sanitizers.TryGetValue("AZSDK1007", out RegisteredSanitizer SASURISanitizer));
+
+            sessionwithXmlBody.Session.Sanitize(SASURISanitizer.Sanitizer);
+
+            Assert.Contains("<CopyProgress>1024/1024</CopyProgress>", Encoding.UTF8.GetString(sessionwithXmlBody.Session.Entries[0].Response.Body));
+        }
+
         [Fact]
         public void OauthResponseSanitizerCleansNonV2AuthRequest()
         {

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/request_with_encoded_ampersand.json

@@ -0,0 +1,39 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://REDACTED/weather/tropical/storms/json?api-version=1.0\u0026year=2023\u0026basinId=AL\u0026govId=5",
+      "RequestMethod": "GET",
+      "RequestHeaders": {
+      },
+      "RequestBody": null,
+      "StatusCode": 200,
+      "ResponseHeaders": {
+        "Cache-Control": "public, max-age=600",
+        "Content-Length": "123",
+        "Content-Type": "application/json; charset=utf-8",
+        "Date": "Wed, 19 Jul 2023 22:52:17 GMT",
+        "Expires": "Wed, 19 Jul 2023 23:02:17 GMT",
+        "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+        "Vary": "Accept-Encoding",
+        "X-Cache": "CONFIG_NOCACHE",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-azuremaps-region": "West US 2",
+        "X-MSEdge-Ref": "Ref A: 980665086A12483993E2782EDFC9F29A Ref B: STBEDGE0106 Ref C: 2023-07-19T22:52:17Z"
+      },
+      "ResponseBody": {
+        "results": [
+          {
+            "year": "2023",
+            "basinId": "AL",
+            "govId": 5,
+            "name": "Don",
+            "isActive": true,
+            "isRetired": false,
+            "isSubtropical": false
+          }
+        ]
+      }
+    }
+  ],
+  "Variables": {}
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/request_with_encoding.json

@@ -0,0 +1,54 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://login.microsoftonline.com/00000000-0000-0000-0000-000000000000/oauth2/v2.0/token",
+      "RequestMethod": "POST",
+      "RequestHeaders": {
+        "Accept": "application/json",
+        "Accept-Encoding": "gzip, deflate",
+        "client-request-id": "76ea37cc-3d67-45ec-9bf5-9f7c0cb8400e",
+        "Connection": "keep-alive",
+        "Content-Length": "284",
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Cookie": "fpc=gibberishblahblah; stsservicecookie=estsfd; x-ms-gateway-slice=estsfd",
+        "User-Agent": "azsdk-python-identity/1.11.0b4 Python/3.8.6 (Windows-10-10.0.22000-SP0)",
+        "x-client-cpu": "x64",
+        "x-client-current-telemetry": "4|730,0|",
+        "x-client-last-telemetry": "4|0|||",
+        "x-client-os": "win32",
+        "x-client-sku": "MSAL.Python",
+        "x-client-ver": "1.18.0",
+        "x-ms-lib-capability": "retry-after, h429"
+      },
+      "RequestBody": "client_id=00000000-0000-0000-0000-000000000000\u0026grant_type=client_credentials\u0026client_info=1\u0026client_secret=a_secret~value_\u0026claims=%7B%22access_token=blahblah",
+      "StatusCode": 200,
+      "ResponseHeaders": {
+        "Cache-Control": "no-store, no-cache",
+        "client-request-id": "76ea37cc-3d67-45ec-9bf5-9f7c0cb8400e",
+        "Content-Length": "111",
+        "Content-Type": "application/json; charset=utf-8",
+        "Date": "Wed, 31 Aug 2022 21:01:03 GMT",
+        "Expires": "-1",
+        "P3P": "CP=\u0022DSP CUR OTPi IND OTRi ONL FIN\u0022",
+        "Pragma": "no-cache",
+        "Set-Cookie": [
+          "fpc=gibberishblahblah; expires=Fri, 30-Sep-2022 21:01:04 GMT; path=/; secure; HttpOnly; SameSite=None",
+          "x-ms-gateway-slice=estsfd; path=/; secure; samesite=none; httponly",
+          "stsservicecookie=estsfd; path=/; secure; samesite=none; httponly"
+        ],
+        "Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+        "X-Content-Type-Options": "nosniff",
+        "x-ms-clitelem": "1,0,0,,",
+        "x-ms-ests-server": "2.1.13622.4 - WUS2 ProdSlices",
+        "X-XSS-Protection": "0"
+      },
+      "ResponseBody": {
+        "token_type": "Bearer",
+        "expires_in": 86399,
+        "ext_expires_in": 86399,
+        "refresh_in": 43199,
+        "access_token": "Sanitized"
+      }
+    }
+  ]
+}

tools/test-proxy/Azure.Sdk.Tools.TestProxy.Tests/Test.RecordEntries/xml_body_with_sas_present.json

@@ -0,0 +1,30 @@
+{
+  "Entries": [
+    {
+      "RequestUri": "https://REDACTED.blob.core.windows.net/blockblobclienttestsynccopyfromuri?comp=list&include=copy&prefix=destS72lXCxNKX&restype=container",
+      "RequestMethod": "GET",
+      "RequestHeaders": {
+        "Authorization": "Sanitized",
+        "User-Agent": "azsdk-cpp-storage-blobs/12.11.0-beta.2 (Windows 10 Enterprise 6.3 22631 22621.1.amd64fre.ni_release.220506-1250)",
+        "x-ms-client-request-id": "Sanitized",
+        "x-ms-date": "Sun, 28 Apr 2024 06:31:46 GMT",
+        "x-ms-version": "2023-11-03"
+      },
+      "RequestBody": null,
+      "StatusCode": 200,
+      "ResponseHeaders": {
+        "Content-Type": "application/xml",
+        "Date": "Sun, 28 Apr 2024 06:31:45 GMT",
+        "Server": [
+          "Windows-Azure-Blob/1.0",
+          "Microsoft-HTTPAPI/2.0"
+        ],
+        "Transfer-Encoding": "chunked",
+        "x-ms-client-request-id": "Sanitized",
+        "x-ms-request-id": "Sanitized",
+        "x-ms-version": "2023-11-03"
+      },
+      "ResponseBody": "\uFEFF<?xml version=\"1.0\" encoding=\"utf-8\"?><EnumerationResults ServiceEndpoint=\"https://afakeblobname.blob.core.windows.net/\" ContainerName=\"blockblobclienttestsynccopyfromuri\"><Prefix>destS72lXCxNKX</Prefix><Blobs><Blob><Name>destS72lXCxNKX</Name><VersionId>2024-04-28T06:31:45.7176515Z</VersionId><IsCurrentVersion>true</IsCurrentVersion><Properties><Creation-Time>Sun, 28 Apr 2024 06:31:45 GMT</Creation-Time><Last-Modified>Sun, 28 Apr 2024 06:31:45 GMT</Last-Modified><Etag>0x8DC674CE03A6C6F</Etag><Content-Length>1024</Content-Length><Content-Type>application/octet-stream</Content-Type><Content-Encoding /><Content-Language /><Content-CRC64>iu0B5sKKllE=</Content-CRC64><Content-MD5>7Fnk5RibvrmdtnSTCz3FWA==</Content-MD5><Cache-Control /><Content-Disposition /><BlobType>BlockBlob</BlobType><AccessTier>Hot</AccessTier><AccessTierInferred>true</AccessTierInferred><LeaseStatus>unlocked</LeaseStatus><LeaseState>available</LeaseState><CopyId>d49efb17-b054-4515-aac3-6e301ee771ce</CopyId><CopySource>https://afakeblobname.blob.core.windows.net:443/blockblobclienttestsynccopyfromuri/sourceC4GbldEmXi?se=2024-04-19T09%3a42%3a44Z&amp;sp=xxx&amp;spr=https%2chttp&amp;sr=c&amp;sv=2023-11-03</CopySource><CopyStatus>success</CopyStatus><CopyProgress>1024/1024</CopyProgress><CopyCompletionTime>Fri, 26 Apr 2024 09:42:44 GMT</CopyCompletionTime><ServerEncrypted>true</ServerEncrypted></Properties><OrMetadata /></Blob></Blobs><NextMarker /></EnumerationResults>"
+    }
+  ]
+}