support evaluator name + verison

Author: Nina Kollman

packages/sample-app/src/sample_experiment.ts

@@ -86,8 +86,8 @@ const main = async () => {
     experiment.run(medicalTaskRefuseAdvice, {
       datasetSlug: "medical-q",
       datasetVersion: "v1",
-      evaluators: [{ name: "medical_advice" }],
-      // experimentSlug: "medical-advice-exp-ts",
+      evaluators: ["medical_advice"],
+      experimentSlug: "medical-advice-exp-ts",
       stopOnError: false,
       waitForResults: true,
     });

packages/traceloop-sdk/src/lib/client/evaluator/evaluator.ts

@@ -67,21 +67,25 @@ export class Evaluator extends BaseDatasetEntity {
       throw new Error('experimentId, evaluator, and taskResult are required');
     }
 
-    if (!evaluator.name) {
-      throw new Error('evaluator.name is required');
+    // Handle both string and object evaluator types
+    const evaluatorName = typeof evaluator === 'string' ? evaluator : evaluator.name;
+    const evaluatorVersion = typeof evaluator === 'string' ? undefined : evaluator.version;
+
+    if (!evaluatorName) {
+      throw new Error('evaluator name is required');
     }
 
     const inputSchemaMapping = this.createInputSchemaMapping(taskResult);
 
     const payload = {
       experiment_id: experimentId,  
       experiment_run_id: experimentRunId,
-      evaluator_version: evaluator.version,
+      evaluator_version: evaluatorVersion,
       task_id: taskId,
       input_schema_mapping: inputSchemaMapping,
     };
 
-    const response = await this.client.post(`/v2/evaluators/slug/${evaluator.name}/execute`, payload);
+    const response = await this.client.post(`/v2/evaluators/slug/${evaluatorName}/execute`, payload);
     const data = await this.handleResponse(response);
 
     return {
@@ -169,9 +173,15 @@ export class Evaluator extends BaseDatasetEntity {
       throw new Error('At least one task result must be provided');
     }
 
-    // Validate each evaluator
-    if (!evaluator.name || typeof evaluator.name !== 'string') {
-      throw new Error(`Evaluator must have a valid name`);
+    // Validate evaluator based on its type
+    if (typeof evaluator === 'string') {
+      if (!evaluator.trim()) {
+        throw new Error('Evaluator name cannot be empty');
+      }
+    } else {
+      if (!evaluator.name || typeof evaluator.name !== 'string' || !evaluator.name.trim()) {
+        throw new Error('Evaluator must have a valid name');
+      }
     }
 
     // Validate each task result

packages/traceloop-sdk/src/lib/client/experiment/experiment.ts

@@ -272,8 +272,20 @@ export class Experiment {
 
     if (options.evaluators) {
       options.evaluators.forEach((evaluator, index) => {
-        if (!evaluator.name || typeof evaluator.name !== 'string') {
-          throw new Error(`Evaluator at index ${index} must have a valid name`);
+        if (typeof evaluator === 'string') {
+          if (!evaluator.trim()) {
+            throw new Error(`Evaluator at index ${index} cannot be an empty string`);
+          }
+        } else {
+          if (!evaluator || typeof evaluator !== 'object') {
+            throw new Error(`Evaluator at index ${index} must be a string or object with name and version`);
+          }
+          if (!evaluator.name || typeof evaluator.name !== 'string' || !evaluator.name.trim()) {
+            throw new Error(`Evaluator at index ${index} must have a valid non-empty name`);
+          }
+          if (!evaluator.version || typeof evaluator.version !== 'string' || !evaluator.version.trim()) {
+            throw new Error(`Evaluator at index ${index} must have a valid non-empty version`);
+          }
         }
       });
     }

packages/traceloop-sdk/src/lib/interfaces/experiment.interface.ts

@@ -2,9 +2,11 @@ export interface ExperimentTaskFunction<TInput = Record<string, any>, TOutput =
   (input: TInput): Promise<TOutput> | TOutput;
 }
 
-export interface EvaluatorDetails {
+export type EvaluatorDetails  = string | EvaluatorWithVersion;
+
+export interface EvaluatorWithVersion {
   name: string;
-  version?: string;
+  version: string;
 }
 
 export interface ExperimentRunOptions {