fix: harden secrets detection benchmark behavior

Signed-off-by: lucarlig <luca.carlig@ibm.com>

Author: lucarlig

plugins/secrets_detection/README.md

@@ -24,9 +24,9 @@ Configuration (example)
       generic_api_key_assignment: false  # Broad heuristic; enable only if you want generic header/assignment coverage
       slack_token: true
       private_key_block: true
-      jwt_like: true
-      hex_secret_32: true
-      base64_24: true
+      jwt_like: false  # Broad heuristic; keep disabled by default unless you explicitly want warning/block coverage for JWT-shaped tokens
+      hex_secret_32: false  # Broad heuristic; keep disabled by default unless you explicitly want coverage for generic hex strings
+      base64_24: false  # Broad heuristic; keep disabled by default unless you explicitly want coverage for generic base64-like strings
     redact: false                # replace matches with redaction_text
     redaction_text: "***REDACTED***"
     block_on_detection: true

plugins_rust/secrets_detection/README.md

@@ -52,9 +52,9 @@ secrets_detection:
       generic_api_key_assignment: false  # Broad heuristic; useful for X-API-Key/api_key=... style coverage, but can increase false positives
       slack_token: true
       private_key_block: true
-      jwt_like: true
-      hex_secret_32: true
-      base64_24: true
+      jwt_like: false
+      hex_secret_32: false
+      base64_24: false
     redact: false                    # Replace secrets with redaction_text
     redaction_text: "***REDACTED***"
     block_on_detection: true         # Block requests containing secrets

tests/loadtest/locustfile_secret_detection.py

@@ -23,6 +23,8 @@
 from locust import between, task
 from locust.contrib.fasthttp import FastHttpUser
 
+from tests.loadtest.secret_detection_benchmark_utils import is_secret_detection_blocked
+
 
 class SecretsDetectionUser(FastHttpUser):
     """User that sends prompts with and without secrets to test detection performance."""
@@ -96,15 +98,13 @@ def get_prompt_with_secret_expect_block(self):
             name="/rpc prompts/get [secret-blocked]",
             catch_response=True,
         ) as response:
-            # Expect 403 when secrets are detected and blocked
-            if response.status_code == 403:
+            if is_secret_detection_blocked(response):
                 self.secrets_blocked_count += 1
                 response.success()
             elif response.status_code == 200:
                 self.secrets_not_blocked_count += 1
-                response.failure("Secret-bearing prompt unexpectedly succeeded with HTTP 200")
+                response.failure("Secret-bearing payload was accepted without a secrets-detection violation")
+                if self.secrets_not_blocked_count % 10 == 0:
+                    print(f"⚠️  Warning: {self.secrets_not_blocked_count} secrets not blocked (may indicate detection disabled)")
             else:
                 response.failure(f"Unexpected status {response.status_code}")
-
-
-# Made with Bob

tests/loadtest/secret_detection_benchmark_utils.py

@@ -0,0 +1,29 @@
+"""Helpers for the secrets-detection benchmark."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def is_secret_detection_blocked(response: Any) -> bool:
+    """Return True when the response indicates the secrets plugin blocked the request."""
+    if getattr(response, "status_code", None) == 403:
+        return True
+
+    try:
+        payload = response.json()
+    except Exception:
+        return False
+
+    if not isinstance(payload, dict):
+        return False
+
+    error = payload.get("error")
+    if not isinstance(error, dict):
+        return False
+
+    data = error.get("data")
+    if isinstance(data, dict) and data.get("plugin_error_code") == "SECRETS_DETECTED":
+        return True
+
+    return False

tests/unit/plugins/test_secrets_detection.py

@@ -12,6 +12,7 @@
 from mcpgateway.plugins.framework import PluginConfig, ResourceHookType
 from mcpgateway.services.resource_service import ResourceService
 from plugins.secrets_detection.secrets_detection import SecretsDetectionPlugin
+from tests.loadtest.secret_detection_benchmark_utils import is_secret_detection_blocked
 
 # Try to import Rust implementation
 try:
@@ -568,3 +569,40 @@ def test_plugin_warns_when_broad_patterns_enabled(caplog):
 
     assert "Broad secrets heuristics enabled" in caplog.text
     assert "generic_api_key_assignment" in caplog.text
+
+
+def test_benchmark_helper_treats_json_rpc_secret_violation_as_blocked():
+    """Load-test helper should recognize JSON-RPC secret violations even on HTTP 200."""
+
+    class DummyResponse:
+        status_code = 200
+
+        @staticmethod
+        def json():
+            return {
+                "error": {
+                    "message": "Plugin Violation: secrets detected",
+                    "data": {
+                        "plugin_error_code": "SECRETS_DETECTED",
+                    },
+                }
+            }
+
+    assert is_secret_detection_blocked(DummyResponse()) is True
+
+
+def test_benchmark_helper_does_not_treat_success_payload_as_blocked():
+    """Load-test helper should not flag ordinary successful responses as blocked."""
+
+    class DummyResponse:
+        status_code = 200
+
+        @staticmethod
+        def json():
+            return {
+                "result": {
+                    "messages": [],
+                }
+            }
+
+    assert is_secret_detection_blocked(DummyResponse()) is False

tests/unit/test_locustfile_secret_detection.py

@@ -26,6 +26,8 @@ def test_secret_request_marks_http_200_as_failure():
         for node in ast.walk(target_function)
         if isinstance(node, ast.Call) and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Name) and node.func.value.id == "response" and node.func.attr == "success"
     ]
+    helper_calls = [node for node in ast.walk(target_function) if isinstance(node, ast.Call) and isinstance(node.func, ast.Name) and node.func.id == "is_secret_detection_blocked"]
 
-    assert any(call.args and isinstance(call.args[0], ast.Constant) and call.args[0].value == "Secret-bearing prompt unexpectedly succeeded with HTTP 200" for call in failure_calls)
+    assert helper_calls
+    assert any(call.args and isinstance(call.args[0], ast.Constant) and call.args[0].value == "Secret-bearing payload was accepted without a secrets-detection violation" for call in failure_calls)
     assert len(success_calls) == 1