Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion packages/sample-app/src/sample_experiment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import type {
TaskInput,
TaskOutput,
} from "@traceloop/node-server-sdk";

import fs from "fs";
import "dotenv/config";

const main = async () => {
Expand Down Expand Up @@ -160,6 +160,15 @@ const main = async () => {
console.log(` - Experiment ID: ${results2.experimentId}`);
console.log("Evaluation Results:", results2.evaluations);
}

console.log("\n📊 Exporting experiment results...");

// Export to CSV and JSON using last run's experiment slug and run ID
const csvData = await client.experiment.toCsvString();
fs.writeFileSync("experiment_results.csv", csvData);

const jsonData = await client.experiment.toJsonString();
fs.writeFileSync("experiment_results.json", jsonData);
} catch (error) {
console.error(
"❌ Error in experiment operations:",
Expand Down
98 changes: 98 additions & 0 deletions packages/traceloop-sdk/src/lib/client/experiment/experiment.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,8 @@ export class Experiment {
private client: TraceloopClient;
private evaluator: Evaluator;
private datasets: Datasets;
private _lastExperimentSlug?: string;
private _lastRunId?: string;

constructor(client: TraceloopClient) {
this.client = client;
Expand Down Expand Up @@ -167,6 +169,11 @@ export class Experiment {
const evalResults = evaluationResults.map(
(evaluation) => evaluation.result,
);

// Track last experiment slug and run ID for export methods
this._lastExperimentSlug = experimentSlug;
this._lastRunId = experimentResponse.run.id;

return {
taskResults: taskResults,
errors: taskErrors,
Expand Down Expand Up @@ -477,11 +484,102 @@ export class Experiment {
);
const data = await this.handleResponse(response);

// Track last experiment slug and run ID for export methods
this._lastExperimentSlug = data.experimentSlug;
this._lastRunId = data.runId;

return data;
} catch (error) {
throw new Error(
`GitHub experiment execution failed: ${error instanceof Error ? error.message : "Unknown error"}`,
);
}
}

/**
* Resolve export parameters by falling back to last used values
*/
private resolveExportParams(
experimentSlug?: string,
runId?: string,
): { slug: string; runId: string } {
const slug = experimentSlug || this._lastExperimentSlug;
const rid = runId || this._lastRunId;

if (!slug) {
throw new Error("experiment_slug is required");
}
if (!rid) {
throw new Error("run_id is required");
}

return { slug, runId: rid };
}

/**
* Export experiment results as CSV string
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
* @param runId - Optional run ID (uses last run if not provided)
* @returns CSV string of experiment results
*/
async toCsvString(experimentSlug?: string, runId?: string): Promise<string> {
const { slug, runId: rid } = this.resolveExportParams(
experimentSlug,
runId,
);

const response = await this.client.get(
`/v2/experiments/${slug}/runs/${rid}/export/csv`,
);

if (!response.ok) {
throw new Error(
`Failed to export CSV for experiment '${slug}' run '${rid}'`,
);
}

const result = await this.handleResponse(response);

if (result === null || result === undefined) {
throw new Error(
`Failed to export CSV for experiment '${slug}' run '${rid}'`,
);
}

return String(result);
}

/**
* Export experiment results as JSON string
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
* @param runId - Optional run ID (uses last run if not provided)
* @returns JSON string of experiment results
*/
async toJsonString(experimentSlug?: string, runId?: string): Promise<string> {
const { slug, runId: rid } = this.resolveExportParams(
experimentSlug,
runId,
);

const response = await this.client.get(
`/v2/experiments/${slug}/runs/${rid}/export/json`,
);

if (!response.ok) {
throw new Error(
`Failed to export JSON for experiment '${slug}' run '${rid}'`,
);
}

const result = await this.handleResponse(response);

if (result === null || result === undefined) {
throw new Error(
`Failed to export JSON for experiment '${slug}' run '${rid}'`,
);
}

// If result is already a string, return it; otherwise stringify it
return typeof result === "string" ? result : JSON.stringify(result);
}
}
147 changes: 147 additions & 0 deletions packages/traceloop-sdk/test/experiment-export.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,147 @@
import { Polly, setupMocha as setupPolly } from "@pollyjs/core";
import NodeHttpAdapter from "@pollyjs/adapter-node-http";
import FetchAdapter from "@pollyjs/adapter-fetch";
import FSPersister from "@pollyjs/persister-fs";
import * as traceloop from "../src";
import * as assert from "assert";

// Register adapters and persisters
Polly.register(NodeHttpAdapter);
Polly.register(FetchAdapter);
Polly.register(FSPersister);

describe("Experiment Export Tests", () => {
let polly: Polly;
let client: traceloop.TraceloopClient;
let experimentSlug: string;
let runId: string;

setupPolly({
adapters: ["node-http", "fetch"],
persister: "fs",
recordIfMissing: process.env.RECORD_MODE === "NEW",
recordFailedRequests: true,
mode: process.env.RECORD_MODE === "NEW" ? "record" : "replay",
matchRequestsBy: {
headers: false,
url: {
protocol: true,
hostname: true,
pathname: true,
query: false,
},
},
logging: true,
});

before(async function () {
const apiKey =
process.env.RECORD_MODE === "NEW"
? process.env.TRACELOOP_API_KEY!
: "test-key";
const baseUrl =
process.env.RECORD_MODE === "NEW"
? process.env.TRACELOOP_BASE_URL!
: "https://api-staging.traceloop.com";

client = new traceloop.TraceloopClient({
appName: "experiment_export_test",
apiKey,
baseUrl,
});
});

beforeEach(function () {
const { server } = this.polly as Polly;
server.any().on("beforePersist", (_req, recording) => {
recording.request.headers = recording.request.headers.filter(
({ name }: { name: string }) =>
!["authorization"].includes(name.toLowerCase()),
);
});
});

describe("Export Methods", () => {
it("should export experiment results as CSV with explicit parameters", async function () {
// Skip this test unless valid Polly recordings exist
if (process.env.RECORD_MODE !== "NEW") {
this.skip();
return;
}

// Use known experiment slug and run ID for testing
experimentSlug = "test-experiment-slug";
runId = "test-run-id";

const csvData = await client.experiment.toCsvString(
experimentSlug,
runId,
);

assert.ok(csvData);
assert.strictEqual(typeof csvData, "string");
console.log(`✓ Exported CSV data: ${csvData.length} characters`);
});
Comment thread
nina-kollman marked this conversation as resolved.

it("should export experiment results as JSON with explicit parameters", async function () {
// Skip this test unless valid Polly recordings exist
if (process.env.RECORD_MODE !== "NEW") {
this.skip();
return;
}

experimentSlug = "test-experiment-slug";
runId = "test-run-id";

const jsonData = await client.experiment.toJsonString(
experimentSlug,
runId,
);

assert.ok(jsonData);
assert.strictEqual(typeof jsonData, "string");
// Verify it's valid JSON
JSON.parse(jsonData);
console.log(`✓ Exported JSON data: ${jsonData.length} characters`);
});
Comment thread
nina-kollman marked this conversation as resolved.

it("should throw error when exporting CSV without experiment slug", async function () {
try {
await client.experiment.toCsvString();
assert.fail("Should have thrown an error");
} catch (error) {
assert.ok(error instanceof Error);
assert.ok(error.message.includes("experiment_slug is required"));
console.log("✓ Correctly threw error for missing experiment slug");
}
});

it("should throw error when exporting JSON without run ID", async function () {
try {
await client.experiment.toJsonString("some-slug");
assert.fail("Should have thrown an error");
} catch (error) {
assert.ok(error instanceof Error);
assert.ok(error.message.includes("run_id is required"));
console.log("✓ Correctly threw error for missing run ID");
}
});

it("should use last run values when not provided", async function () {
// This test would require running an actual experiment first
// For now, we'll just verify the error handling
try {
await client.experiment.toCsvString();
assert.fail("Should have thrown an error");
} catch (error) {
assert.ok(error instanceof Error);
// Should fail because no last run exists
assert.ok(
error.message.includes("experiment_slug is required") ||
error.message.includes("run_id is required"),
);
console.log("✓ Correctly handled missing last run values");
}
});
});
});