chore: added honeybeepf-llm for the benchmark test

Author: jundorok

.github/workflows/benchmark.yml

@@ -33,6 +33,37 @@ jobs:
           source .venv/bin/activate
           uv pip install -r requirements.txt
 
+      - name: Install Rust toolchain (stable + nightly + rust-src)
+        run: |
+          rustup install stable
+          rustup install nightly
+          rustup component add rust-src --toolchain nightly
+        working-directory: .
+
+      - name: Cache cargo
+        uses: Swatinem/rust-cache@v2
+        with:
+          workspaces: honeybeepf-llm
+
+      - name: Install bpf-linker
+        run: cargo install bpf-linker
+        working-directory: .
+
+      - name: Build honeybeepf-llm agent
+        run: cargo +nightly build --release -p honeybeepf-llm
+        working-directory: honeybeepf-llm
+
+      - name: setcap eBPF capabilities
+        run: |
+          sudo setcap cap_bpf,cap_sys_admin,cap_perfmon,cap_net_admin=eip honeybeepf-llm/target/release/honeybeepf-llm
+          getcap honeybeepf-llm/target/release/honeybeepf-llm
+        working-directory: .
+
+      - name: Create otel-output directory
+        run: |
+          mkdir -p otel-output
+          chmod 777 otel-output
+
       - name: Start services
         run: docker compose up -d --wait
 
@@ -42,11 +73,23 @@ jobs:
           python run_benchmark.py \
             --scenario ${{ inputs.scenario }} \
             --output results.json \
-            --markdown summary.md
+            --markdown summary.md \
+            --ebpf-binary ../../honeybeepf-llm/target/release/honeybeepf-llm
 
       - name: Post results to Job Summary
         run: cat summary.md >> "$GITHUB_STEP_SUMMARY"
 
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: benchmark-output
+          path: |
+            tests/benchmark/results.json
+            tests/benchmark/summary.md
+            tests/benchmark/otel-output/otel.jsonl
+          retention-days: 14
+
       - name: Teardown
         if: always()
         run: docker compose down

.gitignore

@@ -67,5 +67,8 @@ __pycache__/
 *.pyc
 .venv/
 
+# Benchmark OTel collector output (runtime artifacts)
+tests/benchmark/otel-output/
+
 # Superpowers docs
 docs/superpowers/

tests/benchmark/Dockerfile.mock

@@ -1,6 +1,17 @@
 FROM python:3.12-slim
 WORKDIR /app
+RUN apt-get update && \
+    apt-get install -y --no-install-recommends openssl && \
+    rm -rf /var/lib/apt/lists/*
 RUN pip install --no-cache-dir fastapi uvicorn
+RUN openssl req -x509 -newkey rsa:2048 \
+    -keyout /app/key.pem -out /app/cert.pem \
+    -days 3650 -nodes \
+    -subj '/CN=mock-llm' \
+    -addext "subjectAltName=DNS:mock-llm,DNS:localhost,IP:127.0.0.1"
 COPY mock_server.py .
-EXPOSE 8080
-CMD ["uvicorn", "mock_server:app", "--host", "0.0.0.0", "--port", "8080", "--log-level", "warning"]
+EXPOSE 8443
+CMD ["uvicorn", "mock_server:app", \
+     "--host", "0.0.0.0", "--port", "8443", \
+     "--ssl-keyfile=/app/key.pem", "--ssl-certfile=/app/cert.pem", \
+     "--log-level", "warning"]

tests/benchmark/config.py

@@ -2,7 +2,7 @@
 
 from load_generator import LoadProfile
 
-DIRECT_URL = "http://localhost:8080"
+DIRECT_URL = "https://localhost:8443"
 PROXY_URL = "http://localhost:4000"
 CONTAINER_NAMES = ["benchmark-mock-llm-1", "benchmark-litellm-1"]
 

tests/benchmark/docker-compose.yml

@@ -4,12 +4,13 @@ services:
       context: .
       dockerfile: Dockerfile.mock
     ports:
-      - "8080:8080"
+      - "8443:8443"
     environment:
       - MOCK_RESPONSE_DELAY_MS=50
       - MOCK_COMPLETION_TOKENS=30
     healthcheck:
-      test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:8080/health')"]
+      test: ["CMD", "python", "-c",
+        "import urllib.request, ssl; ctx=ssl._create_unverified_context(); urllib.request.urlopen('https://localhost:8443/health', context=ctx)"]
       interval: 2s
       timeout: 2s
       retries: 5
@@ -24,11 +25,25 @@ services:
     depends_on:
       mock-llm:
         condition: service_healthy
+      otel-collector:
+        condition: service_started
     environment:
       - LITELLM_LOG=ERROR
+      - OTEL_EXPORTER=otlp_grpc
+      - OTEL_ENDPOINT=http://otel-collector:4317
+      - OTEL_SERVICE_NAME=litellm
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://localhost:4000/health/liveliness')"]
       interval: 2s
       timeout: 3s
       retries: 5
       start_period: 30s
+
+  otel-collector:
+    image: otel/opentelemetry-collector-contrib:0.110.0
+    command: ["--config=/etc/otel-collector.yaml"]
+    volumes:
+      - ./otel-collector.yaml:/etc/otel-collector.yaml
+      - ./otel-output:/output
+    ports:
+      - "4317:4317"

tests/benchmark/litellm_config.yaml

@@ -2,17 +2,21 @@ model_list:
   - model_name: gpt-4o-mini
     litellm_params:
       model: openai/gpt-4o-mini
-      api_base: http://mock-llm:8080/v1
+      api_base: https://mock-llm:8443/v1
       api_key: fake-key-for-benchmark
 
   - model_name: claude-sonnet-4-20250514
     litellm_params:
       model: openai/claude-sonnet-4-20250514
-      api_base: http://mock-llm:8080/v1
+      api_base: https://mock-llm:8443/v1
       api_key: fake-key-for-benchmark
 
   - model_name: gemini-2.0-flash
     litellm_params:
       model: openai/gemini-2.0-flash
-      api_base: http://mock-llm:8080/v1
+      api_base: https://mock-llm:8443/v1
       api_key: fake-key-for-benchmark
+
+litellm_settings:
+  callbacks: ["otel"]
+  ssl_verify: false

tests/benchmark/load_generator.py

@@ -65,7 +65,7 @@ async def worker(client: httpx.AsyncClient):
 
         t_start = time.monotonic()
 
-        async with httpx.AsyncClient(timeout=self._timeout) as client:
+        async with httpx.AsyncClient(timeout=self._timeout, verify=False) as client:
             if profile.rate_rps > 0 and profile.duration_secs > 0:
                 interval = 1.0 / profile.rate_rps
                 tasks = []

tests/benchmark/otel-collector.yaml

@@ -0,0 +1,26 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+
+exporters:
+  debug:
+    verbosity: detailed
+  file:
+    path: /output/otel.jsonl
+
+service:
+  telemetry:
+    logs:
+      level: info
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug, file]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug, file]
+    logs:
+      receivers: [otlp]
+      exporters: [debug, file]

tests/benchmark/report.py

@@ -16,18 +16,35 @@ def _overhead(key: str, b: float, p: float) -> str:
     return "-"
 
 
-def format_report(baseline: dict, proxy: dict) -> str:
-    header = f"{'Metric':<25} {'Baseline':>12} {'LiteLLM Proxy':>14} {'Overhead':>12}"
+def format_report(baseline: dict, proxy: dict, ebpf: dict | None = None) -> str:
+    if ebpf is None:
+        header = f"{'Metric':<25} {'Baseline':>12} {'LiteLLM Proxy':>14} {'Overhead':>12}"
+        sep = "-" * len(header)
+        lines = [header, sep]
+        for row in ROWS:
+            label, key, fmt = row[0], row[1], row[2]
+            mult = row[3] if len(row) > 3 else 1
+            b = baseline.get(key, 0) * mult
+            p = proxy.get(key, 0) * mult
+            lines.append(f"{label:<25} {format(b, fmt):>12} {format(p, fmt):>14} {_overhead(key, b, p):>12}")
+        return "\n".join(lines)
+
+    header = (
+        f"{'Metric':<25} {'Baseline':>12} {'LiteLLM Proxy':>14} {'honeybeepf-llm':>15} "
+        f"{'Proxy Overhead':>14} {'eBPF Overhead':>14}"
+    )
     sep = "-" * len(header)
     lines = [header, sep]
-
     for row in ROWS:
         label, key, fmt = row[0], row[1], row[2]
         mult = row[3] if len(row) > 3 else 1
         b = baseline.get(key, 0) * mult
         p = proxy.get(key, 0) * mult
-        lines.append(f"{label:<25} {format(b, fmt):>12} {format(p, fmt):>14} {_overhead(key, b, p):>12}")
-
+        e = ebpf.get(key, 0) * mult
+        lines.append(
+            f"{label:<25} {format(b, fmt):>12} {format(p, fmt):>14} {format(e, fmt):>15} "
+            f"{_overhead(key, b, p):>14} {_overhead(key, b, e):>14}"
+        )
     return "\n".join(lines)
 
 
@@ -40,24 +57,38 @@ def format_markdown(results: dict) -> str:
 
         baseline = data.get("baseline", {})
         proxy = data.get("proxy", {})
+        ebpf = data.get("ebpf")
 
         lines.append(f"### {scenario_name}")
         lines.append("")
-        lines.append("| Metric | Baseline | LiteLLM Proxy | Overhead |")
-        lines.append("|--------|----------|---------------|----------|")
-
-        for row in ROWS:
-            label, key, fmt = row[0], row[1], row[2]
-            mult = row[3] if len(row) > 3 else 1
-            b = baseline.get(key, 0) * mult
-            p = proxy.get(key, 0) * mult
-            lines.append(f"| {label} | {format(b, fmt)} | {format(p, fmt)} | {_overhead(key, b, p)} |")
 
+        if ebpf is None:
+            lines.append("| Metric | Baseline | LiteLLM Proxy | Overhead |")
+            lines.append("|--------|----------|---------------|----------|")
+            for row in ROWS:
+                label, key, fmt = row[0], row[1], row[2]
+                mult = row[3] if len(row) > 3 else 1
+                b = baseline.get(key, 0) * mult
+                p = proxy.get(key, 0) * mult
+                lines.append(f"| {label} | {format(b, fmt)} | {format(p, fmt)} | {_overhead(key, b, p)} |")
+        else:
+            lines.append("| Metric | Baseline | LiteLLM Proxy | honeybeepf-llm | Proxy Overhead | eBPF Overhead |")
+            lines.append("|--------|----------|---------------|----------------|----------------|----------------|")
+            for row in ROWS:
+                label, key, fmt = row[0], row[1], row[2]
+                mult = row[3] if len(row) > 3 else 1
+                b = baseline.get(key, 0) * mult
+                p = proxy.get(key, 0) * mult
+                e = ebpf.get(key, 0) * mult
+                lines.append(
+                    f"| {label} | {format(b, fmt)} | {format(p, fmt)} | {format(e, fmt)} "
+                    f"| {_overhead(key, b, p)} | {_overhead(key, b, e)} |"
+                )
         lines.append("")
 
     if "kill" in results:
         k = results["kill"]
-        lines.append("### Kill Test")
+        lines.append("### Kill Test (LiteLLM Proxy only — single-arm, no baseline/eBPF analog)")
         lines.append("")
         lines.append(f"- Proxy killed mid-load: **{k['errors']}** requests failed ({k['error_rate']:.0%} error rate)")
         lines.append("")