Skip to content

Commit 114d2b8

Browse files
nina-kollmannina-kollman
authored
fix(traceloop-sdk): Add csv and json support to experiment (#864)
1 parent 534f198 commit 114d2b8

File tree

3 files changed

+255
-1
lines changed

3 files changed

+255
-1
lines changed

‎packages/sample-app/src/sample_experiment.ts‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import type {
77
TaskInput,
88
TaskOutput,
99
} from "@traceloop/node-server-sdk";
10-
10+
import fs from "fs";
1111
import "dotenv/config";
1212

1313
const main = async () => {
@@ -160,6 +160,15 @@ const main = async () => {
160160
console.log(` - Experiment ID: ${results2.experimentId}`);
161161
console.log("Evaluation Results:", results2.evaluations);
162162
}
163+
164+
console.log("\n📊 Exporting experiment results...");
165+
166+
// Export to CSV and JSON using last run's experiment slug and run ID
167+
const csvData = await client.experiment.toCsvString();
168+
fs.writeFileSync("experiment_results.csv", csvData);
169+
170+
const jsonData = await client.experiment.toJsonString();
171+
fs.writeFileSync("experiment_results.json", jsonData);
163172
} catch (error) {
164173
console.error(
165174
"❌ Error in experiment operations:",

‎packages/traceloop-sdk/src/lib/client/experiment/experiment.ts‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ export class Experiment {
2222
private client: TraceloopClient;
2323
private evaluator: Evaluator;
2424
private datasets: Datasets;
25+
private _lastExperimentSlug?: string;
26+
private _lastRunId?: string;
2527

2628
constructor(client: TraceloopClient) {
2729
this.client = client;
@@ -167,6 +169,11 @@ export class Experiment {
167169
const evalResults = evaluationResults.map(
168170
(evaluation) => evaluation.result,
169171
);
172+
173+
// Track last experiment slug and run ID for export methods
174+
this._lastExperimentSlug = experimentSlug;
175+
this._lastRunId = experimentResponse.run.id;
176+
170177
return {
171178
taskResults: taskResults,
172179
errors: taskErrors,
@@ -477,11 +484,102 @@ export class Experiment {
477484
);
478485
const data = await this.handleResponse(response);
479486

487+
// Track last experiment slug and run ID for export methods
488+
this._lastExperimentSlug = data.experimentSlug;
489+
this._lastRunId = data.runId;
490+
480491
return data;
481492
} catch (error) {
482493
throw new Error(
483494
`GitHub experiment execution failed: ${error instanceof Error ? error.message : "Unknown error"}`,
484495
);
485496
}
486497
}
498+
499+
/**
500+
* Resolve export parameters by falling back to last used values
501+
*/
502+
private resolveExportParams(
503+
experimentSlug?: string,
504+
runId?: string,
505+
): { slug: string; runId: string } {
506+
const slug = experimentSlug || this._lastExperimentSlug;
507+
const rid = runId || this._lastRunId;
508+
509+
if (!slug) {
510+
throw new Error("experiment_slug is required");
511+
}
512+
if (!rid) {
513+
throw new Error("run_id is required");
514+
}
515+
516+
return { slug, runId: rid };
517+
}
518+
519+
/**
520+
* Export experiment results as CSV string
521+
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
522+
* @param runId - Optional run ID (uses last run if not provided)
523+
* @returns CSV string of experiment results
524+
*/
525+
async toCsvString(experimentSlug?: string, runId?: string): Promise<string> {
526+
const { slug, runId: rid } = this.resolveExportParams(
527+
experimentSlug,
528+
runId,
529+
);
530+
531+
const response = await this.client.get(
532+
`/v2/experiments/${slug}/runs/${rid}/export/csv`,
533+
);
534+
535+
if (!response.ok) {
536+
throw new Error(
537+
`Failed to export CSV for experiment '${slug}' run '${rid}'`,
538+
);
539+
}
540+
541+
const result = await this.handleResponse(response);
542+
543+
if (result === null || result === undefined) {
544+
throw new Error(
545+
`Failed to export CSV for experiment '${slug}' run '${rid}'`,
546+
);
547+
}
548+
549+
return String(result);
550+
}
551+
552+
/**
553+
* Export experiment results as JSON string
554+
* @param experimentSlug - Optional experiment slug (uses last run if not provided)
555+
* @param runId - Optional run ID (uses last run if not provided)
556+
* @returns JSON string of experiment results
557+
*/
558+
async toJsonString(experimentSlug?: string, runId?: string): Promise<string> {
559+
const { slug, runId: rid } = this.resolveExportParams(
560+
experimentSlug,
561+
runId,
562+
);
563+
564+
const response = await this.client.get(
565+
`/v2/experiments/${slug}/runs/${rid}/export/json`,
566+
);
567+
568+
if (!response.ok) {
569+
throw new Error(
570+
`Failed to export JSON for experiment '${slug}' run '${rid}'`,
571+
);
572+
}
573+
574+
const result = await this.handleResponse(response);
575+
576+
if (result === null || result === undefined) {
577+
throw new Error(
578+
`Failed to export JSON for experiment '${slug}' run '${rid}'`,
579+
);
580+
}
581+
582+
// If result is already a string, return it; otherwise stringify it
583+
return typeof result === "string" ? result : JSON.stringify(result);
584+
}
487585
}

‎packages/traceloop-sdk/test/experiment-export.test.ts‎

Lines changed: 147 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,147 @@
1+
import { Polly, setupMocha as setupPolly } from "@pollyjs/core";
2+
import NodeHttpAdapter from "@pollyjs/adapter-node-http";
3+
import FetchAdapter from "@pollyjs/adapter-fetch";
4+
import FSPersister from "@pollyjs/persister-fs";
5+
import * as traceloop from "../src";
6+
import * as assert from "assert";
7+
8+
// Register adapters and persisters
9+
Polly.register(NodeHttpAdapter);
10+
Polly.register(FetchAdapter);
11+
Polly.register(FSPersister);
12+
13+
describe("Experiment Export Tests", () => {
14+
let polly: Polly;
15+
let client: traceloop.TraceloopClient;
16+
let experimentSlug: string;
17+
let runId: string;
18+
19+
setupPolly({
20+
adapters: ["node-http", "fetch"],
21+
persister: "fs",
22+
recordIfMissing: process.env.RECORD_MODE === "NEW",
23+
recordFailedRequests: true,
24+
mode: process.env.RECORD_MODE === "NEW" ? "record" : "replay",
25+
matchRequestsBy: {
26+
headers: false,
27+
url: {
28+
protocol: true,
29+
hostname: true,
30+
pathname: true,
31+
query: false,
32+
},
33+
},
34+
logging: true,
35+
});
36+
37+
before(async function () {
38+
const apiKey =
39+
process.env.RECORD_MODE === "NEW"
40+
? process.env.TRACELOOP_API_KEY!
41+
: "test-key";
42+
const baseUrl =
43+
process.env.RECORD_MODE === "NEW"
44+
? process.env.TRACELOOP_BASE_URL!
45+
: "https://api-staging.traceloop.com";
46+
47+
client = new traceloop.TraceloopClient({
48+
appName: "experiment_export_test",
49+
apiKey,
50+
baseUrl,
51+
});
52+
});
53+
54+
beforeEach(function () {
55+
const { server } = this.polly as Polly;
56+
server.any().on("beforePersist", (_req, recording) => {
57+
recording.request.headers = recording.request.headers.filter(
58+
({ name }: { name: string }) =>
59+
!["authorization"].includes(name.toLowerCase()),
60+
);
61+
});
62+
});
63+
64+
describe("Export Methods", () => {
65+
it("should export experiment results as CSV with explicit parameters", async function () {
66+
// Skip this test unless valid Polly recordings exist
67+
if (process.env.RECORD_MODE !== "NEW") {
68+
this.skip();
69+
return;
70+
}
71+
72+
// Use known experiment slug and run ID for testing
73+
experimentSlug = "test-experiment-slug";
74+
runId = "test-run-id";
75+
76+
const csvData = await client.experiment.toCsvString(
77+
experimentSlug,
78+
runId,
79+
);
80+
81+
assert.ok(csvData);
82+
assert.strictEqual(typeof csvData, "string");
83+
console.log(`✓ Exported CSV data: ${csvData.length} characters`);
84+
});
85+
86+
it("should export experiment results as JSON with explicit parameters", async function () {
87+
// Skip this test unless valid Polly recordings exist
88+
if (process.env.RECORD_MODE !== "NEW") {
89+
this.skip();
90+
return;
91+
}
92+
93+
experimentSlug = "test-experiment-slug";
94+
runId = "test-run-id";
95+
96+
const jsonData = await client.experiment.toJsonString(
97+
experimentSlug,
98+
runId,
99+
);
100+
101+
assert.ok(jsonData);
102+
assert.strictEqual(typeof jsonData, "string");
103+
// Verify it's valid JSON
104+
JSON.parse(jsonData);
105+
console.log(`✓ Exported JSON data: ${jsonData.length} characters`);
106+
});
107+
108+
it("should throw error when exporting CSV without experiment slug", async function () {
109+
try {
110+
await client.experiment.toCsvString();
111+
assert.fail("Should have thrown an error");
112+
} catch (error) {
113+
assert.ok(error instanceof Error);
114+
assert.ok(error.message.includes("experiment_slug is required"));
115+
console.log("✓ Correctly threw error for missing experiment slug");
116+
}
117+
});
118+
119+
it("should throw error when exporting JSON without run ID", async function () {
120+
try {
121+
await client.experiment.toJsonString("some-slug");
122+
assert.fail("Should have thrown an error");
123+
} catch (error) {
124+
assert.ok(error instanceof Error);
125+
assert.ok(error.message.includes("run_id is required"));
126+
console.log("✓ Correctly threw error for missing run ID");
127+
}
128+
});
129+
130+
it("should use last run values when not provided", async function () {
131+
// This test would require running an actual experiment first
132+
// For now, we'll just verify the error handling
133+
try {
134+
await client.experiment.toCsvString();
135+
assert.fail("Should have thrown an error");
136+
} catch (error) {
137+
assert.ok(error instanceof Error);
138+
// Should fail because no last run exists
139+
assert.ok(
140+
error.message.includes("experiment_slug is required") ||
141+
error.message.includes("run_id is required"),
142+
);
143+
console.log("✓ Correctly handled missing last run values");
144+
}
145+
});
146+
});
147+
});

0 commit comments

Comments
 (0)
⚡