evaluator slug optional

Author: Nina Kollman

packages/sample-app/src/sample_experiment.ts

@@ -87,16 +87,16 @@ const main = async () => {
       datasetSlug: "medical-q",
       datasetVersion: "v1",
       evaluators: [{ name: "medical_advice" }],
-      experimentSlug: "medical-advice-exp-ts",
+      // experimentSlug: "medical-advice-exp-ts",
       stopOnError: false,
       waitForResults: true,
     });
 
     console.log(`✅ Completed refuse advice experiment:`);
-    console.log(`   - Results: ${results1.results.length}`);
+    console.log(`   - Results: ${results1.taskResults.length}`);
     console.log(`   - Errors: ${results1.errors.length}`);
     console.log(`   - Experiment ID: ${results1.experimentId}`);
-    console.log("Results:", results1.results);
+    console.log("Evaluation Results:", results1.evaluations);
 
     console.log("\n🧪 Running experiment with comprehensive medical info prompt...");
 
@@ -117,7 +117,7 @@ const main = async () => {
     // Compare results
     console.log("\n📊 Experiment Comparison:");
     console.log("Refuse Advice Strategy:");
-    results1.results.slice(0, 2).forEach((result: TaskResponse, i: number) => {
+    results1.taskResults.slice(0, 2).forEach((result: TaskResponse, i: number) => {
       console.log(`  Sample ${i + 1}:`);
       console.log(`    Question: ${result.input?.question || 'N/A'}`);
       console.log(`    Response: ${result.output?.completion?.substring(0, 100) || 'N/A'}...`);

packages/traceloop-sdk/src/lib/client/experiment/experiment.ts

@@ -25,6 +25,15 @@ export class Experiment {
     this.datasets = new Datasets(client);
   }
 
+  /**
+   * Generate a unique experiment slug
+   */
+  private generateExperimentSlug(): string {
+    const timestamp = Date.now().toString(36);
+    const random = Math.random().toString(36).substring(2, 7);
+    return `exp-${timestamp}${random}`.substring(0, 15);
+  }
+
   private async handleResponse(response: Response) {
     if (!response.ok) {
       let errorMessage = `HTTP ${response.status}: ${response.statusText}`;
@@ -66,28 +75,24 @@ export class Experiment {
       datasetSlug,
       datasetVersion,
       evaluators = [],
-      experimentSlug,
       waitForResults = true,
     } = options;
 
-    // Validate inputs
+    // When experimentSlug is not provided a random one is generated
+    let { experimentSlug } = options;
+    if (!experimentSlug) {
+      experimentSlug = this.client.experimentSlug || this.generateExperimentSlug();
+    }
+
     this.validateRunOptions(task, options);
 
     try {
-      if (!experimentSlug) {
-        throw new Error('Experiment slug is required'); // TODO nina
-      }
-
-      // 1. Initialize experiment
-      console.log(`🔧 Step 1: Initializing experiment with slug: ${experimentSlug}`);
       const experimentResponse = await this.initializeExperiment({
-        slug: experimentSlug || 'default-experiment',
+        slug: experimentSlug,
         datasetSlug,
         datasetVersion,
       });
       console.log(`✅ Step 1: Experiment initialized with ID: ${experimentResponse.experiment.id}`);
-
-      // 2. Get dataset rows
       console.log(`🔧 Step 2: Getting dataset rows for: ${datasetSlug}, version: ${datasetVersion}`);
       const rows = await this.getDatasetRows(datasetSlug, datasetVersion);
       console.log(`✅ Step 2: Retrieved ${rows.length} rows from dataset`);
@@ -111,10 +116,8 @@ export class Experiment {
           timestamp: Date.now()
         };
 
-        // Add to results array
         taskResults.push(taskResponse);
 
-        // Create task
         const response = await this.createTask(
           experimentSlug,
           experimentResponse.run.id,
@@ -139,12 +142,14 @@ export class Experiment {
         }
       }
 
+      const evalResults = evaluationResults.map((evaluation) => evaluation.result);
+      console.log("evalResults", evalResults);
       return {
-        results: taskResults,
+        taskResults: taskResults,
         errors: taskErrors,
         experimentId: experimentResponse.experiment.id,
         runId: experimentResponse.run.id,
-        evaluations: evaluationResults
+        evaluations: evalResults
       };
 
     } catch (error) {
@@ -204,7 +209,6 @@ export class Experiment {
     return data;
   }
 
-
   /**
    * Parse JSONL string into list of {col_name: col_value} dictionaries
    * Skips the first line (columns definition)
@@ -231,7 +235,6 @@ export class Experiment {
     return rows;
   }
 
-
   /**
    * Get dataset rows for experiment execution
    */

packages/traceloop-sdk/src/lib/client/traceloop-client.ts

@@ -23,6 +23,12 @@ export class TraceloopClient {
   public appName: string;
   private baseUrl: string;
   private apiKey: string;
+  public experimentSlug?: string;
+
+  public userFeedback: UserFeedback;
+  public datasets: Datasets;
+  public experiment: Experiment;
+  public evaluator: Evaluator;
 
   /**
    * Creates a new instance of the TraceloopClient.
@@ -36,12 +42,13 @@ export class TraceloopClient {
       options.baseUrl ||
       process.env.TRACELOOP_BASE_URL ||
       "https://api.traceloop.com";
-  }
+    this.experimentSlug = options.experimentSlug;
 
-  userFeedback = new UserFeedback(this);
-  datasets = new Datasets(this);
-  experiment = new Experiment(this);
-  evaluator = new Evaluator(this);
+    this.userFeedback = new UserFeedback(this);
+    this.datasets = new Datasets(this);
+    this.experiment = new Experiment(this);
+    this.evaluator = new Evaluator(this);
+  }
 
   async post(path: string, body: Record<string, unknown> | any) {
     return await fetch(`${this.baseUrl}${path}`, {

packages/traceloop-sdk/src/lib/configuration/index.ts

@@ -39,6 +39,9 @@ export const initialize = (options: InitializeOptions = {}) => {
   if (!options.appName) {
     options.appName = process.env.npm_package_name;
   }
+  if (!options.experimentSlug) {
+    options.experimentSlug = process.env.TRACELOOP_EXP_SLUG;
+  }
 
   if (options.traceloopSyncEnabled === undefined) {
     if (process.env.TRACELOOP_SYNC_ENABLED !== undefined) {
@@ -96,6 +99,7 @@ export const initialize = (options: InitializeOptions = {}) => {
       apiKey: options.apiKey,
       baseUrl: options.baseUrl,
       appName: options.appName!,
+      experimentSlug: options.experimentSlug,
     });
     return _client;
   }

packages/traceloop-sdk/src/lib/interfaces/experiment.interface.ts

@@ -31,11 +31,11 @@ export interface TaskResponse {
 }
 
 export interface ExperimentRunResult {
-  results: TaskResponse[];
+  taskResults: TaskResponse[];
   errors: string[];
   experimentId?: string;
   runId?: string;
-  evaluations?: ExecutionResponse[];
+  evaluations?: Record<string, any>[];
 }
 
 export interface InitExperimentRequest {

packages/traceloop-sdk/src/lib/interfaces/index.ts

@@ -5,8 +5,3 @@ export * from "./traceloop-client.interface";
 export * from "./dataset.interface";
 export * from "./experiment.interface";
 export * from "./evaluator.interface";
-export interface TraceloopClientOptions {
-  apiKey: string;
-  appName: string;
-  baseUrl?: string;
-}

packages/traceloop-sdk/src/lib/interfaces/initialize-options.interface.ts

@@ -135,4 +135,10 @@ export interface InitializeOptions {
    * Defaults to true.
    */
   tracingEnabled?: boolean;
+
+  /**
+   * The experiment slug to use when running experiments. Optional.
+   * Defaults to the TRACELOOP_EXP_SLUG environment variable.
+   */
+  experimentSlug?: string;
 }

packages/traceloop-sdk/src/lib/interfaces/traceloop-client.interface.ts

@@ -2,4 +2,5 @@ export interface TraceloopClientOptions {
   apiKey: string;
   appName: string;
   baseUrl?: string;
+  experimentSlug?: string;
 }