Merge pull request #1192 from Kiln-AI/KIL-491/spec-eval-name-sync

Sync eval name when spec name is updated

Author: sfierro

app/desktop/studio_server/copilot_api.py

@@ -303,7 +303,7 @@ async def create_spec_with_copilot(
         models_to_save: list[Eval | EvalConfig | TaskRun | Spec] = []
 
         # 1. Create the Eval
-        eval_model = Eval(
+        eval = Eval(
             parent=task,
             name=request.name,
             description=None,
@@ -315,11 +315,11 @@ async def create_spec_with_copilot(
             template_properties=None,
             evaluation_data_type=evaluation_data_type,
         )
-        models_to_save.append(eval_model)
+        models_to_save.append(eval)
 
         # 2. Create judge eval config
         eval_config = EvalConfig(
-            parent=eval_model,
+            parent=eval,
             name=generate_memorable_name(),
             config_type=EvalConfigType.llm_as_judge,
             model_name=request.judge_info.task_metadata.model_name,
@@ -332,7 +332,7 @@ async def create_spec_with_copilot(
         models_to_save.append(eval_config)
 
         # Set as default config after ID is assigned
-        eval_model.current_config_id = eval_config.id
+        eval.current_config_id = eval_config.id
 
         # 3. Generate examples via copilot API
         api_key = get_copilot_api_key()
@@ -379,7 +379,7 @@ async def create_spec_with_copilot(
             priority=Priority.p1,
             status=SpecStatus.active,
             tags=[],
-            eval_id=eval_model.id,
+            eval_id=eval.id,
             task_sample=request.task_sample,
             synthetic_data_generation_session_config=SyntheticDataGenerationSessionConfig(
                 topic_generation_config=SyntheticDataGenerationStepConfig(
@@ -405,8 +405,8 @@ async def create_spec_with_copilot(
         # Save everything, with cleanup on failure.
         saved_models: list[Eval | EvalConfig | TaskRun | Spec] = []
         try:
-            eval_model.save_to_file()
-            saved_models.append(eval_model)
+            eval.save_to_file()
+            saved_models.append(eval)
 
             eval_config.save_to_file()
             saved_models.append(eval_config)

app/desktop/studio_server/test_copilot_api.py

@@ -13,6 +13,8 @@
 from app.desktop.studio_server.copilot_api import connect_copilot_api
 from fastapi import FastAPI
 from fastapi.testclient import TestClient
+from kiln_ai.datamodel import Project, Task
+from kiln_ai.datamodel.spec_properties import SpecType
 from kiln_server.custom_errors import connect_custom_errors
 
 
@@ -349,3 +351,96 @@ def test_generate_batch_validation_error(
             )
             assert response.status_code == 422
             assert "Validation error from server" in response.json()["message"]
+
+
+class TestCreateSpecWithCopilot:
+    @pytest.fixture
+    def project_and_task(self, tmp_path):
+        project_path = tmp_path / "test_project" / "project.kiln"
+        project_path.parent.mkdir()
+        project = Project(name="Test Project", path=project_path)
+        project.save_to_file()
+        task = Task(
+            name="Test Task",
+            instruction="Test instruction",
+            description="Test task",
+            parent=project,
+        )
+        task.save_to_file()
+        return project, task
+
+    @pytest.fixture
+    def copilot_request_data(self):
+        step_config = {
+            "task_metadata": {
+                "model_name": "gpt-4",
+                "model_provider_name": "openai",
+            },
+            "prompt": "Test prompt",
+        }
+        return {
+            "name": "Test Spec",
+            "definition": "The system should respond politely",
+            "properties": {
+                "spec_type": SpecType.tone.value,
+                "core_requirement": "Be polite",
+                "tone_description": "Professional and friendly",
+            },
+            "judge_info": step_config,
+            "sdg_session_config": {
+                "topic_generation_config": step_config,
+                "input_generation_config": step_config,
+                "output_generation_config": step_config,
+            },
+            "task_description": "Test task",
+            "task_prompt_with_example": "Test prompt",
+        }
+
+    def test_create_spec_with_copilot_success(
+        self, client, project_and_task, copilot_request_data
+    ):
+        project, task = project_and_task
+
+        with (
+            patch(
+                "app.desktop.studio_server.copilot_api.task_from_id",
+                return_value=task,
+            ),
+            patch(
+                "app.desktop.studio_server.copilot_api.get_copilot_api_key",
+                return_value="test_key",
+            ),
+            patch(
+                "app.desktop.studio_server.copilot_api.generate_copilot_examples",
+                new_callable=AsyncMock,
+                return_value={},
+            ),
+            patch(
+                "app.desktop.studio_server.copilot_api.create_dataset_task_runs",
+                return_value=[],
+            ),
+            patch(
+                "app.desktop.studio_server.copilot_api.generate_memorable_name",
+                return_value="test-config-name",
+            ),
+        ):
+            response = client.post(
+                f"/api/projects/{project.id}/tasks/{task.id}/spec_with_copilot",
+                json=copilot_request_data,
+            )
+
+        assert response.status_code == 200
+        res = response.json()
+        assert res["name"] == "Test Spec"
+        assert res["definition"] == "The system should respond politely"
+        assert res["eval_id"] is not None
+
+        # Verify models were saved
+        evals = task.evals()
+        assert len(evals) == 1
+        assert evals[0].name == "Test Spec"
+        assert evals[0].current_config_id is not None
+
+        specs = task.specs()
+        assert len(specs) == 1
+        assert specs[0].eval_id == evals[0].id

libs/server/kiln_server/spec_api.py

@@ -112,7 +112,7 @@ async def create_spec(
             spec_type, spec_data.evaluate_full_trace
         )
 
-        eval_model = Eval(
+        eval = Eval(
             parent=task,
             name=spec_data.name,
             description=None,
@@ -133,15 +133,15 @@ async def create_spec(
             priority=spec_data.priority,
             status=spec_data.status,
             tags=spec_data.tags,
-            eval_id=eval_model.id,
+            eval_id=eval.id,
             task_sample=spec_data.task_sample,
         )
 
-        eval_model.save_to_file()
+        eval.save_to_file()
         try:
             spec.save_to_file()
         except Exception:
-            eval_model.delete()
+            eval.delete()
             raise
 
         return spec
@@ -217,7 +217,30 @@ async def update_spec(
         if request.tags is not None:
             spec.tags = request.tags
 
-        spec.save_to_file()
+        # Sync eval name when spec name changes
+        eval: Eval | None = None
+        previous_eval_name: str | None = None
+        if request.name is not None and spec.eval_id:
+            parent_task = task_from_id(project_id, task_id)
+            eval = Eval.from_id_and_parent_path(spec.eval_id, parent_task.path)
+            if eval and eval.name != request.name:
+                previous_eval_name = eval.name
+                eval.name = request.name
+                eval.save_to_file()
+
+        try:
+            spec.save_to_file()
+        except Exception:
+            if eval is not None and previous_eval_name is not None:
+                try:
+                    eval.name = previous_eval_name
+                    eval.save_to_file()
+                except Exception:
+                    logger.exception(
+                        "Failed to roll back eval name after spec save failure"
+                    )
+            raise
+
         return spec
 
     @app.delete(
@@ -241,7 +264,9 @@ async def delete_spec(
         # Delete associated eval if it exists
         if spec.eval_id:
             parent_task = task_from_id(project_id, task_id)
-            eval = Eval.from_id_and_parent_path(spec.eval_id, parent_task.path)
+            eval: Eval | None = Eval.from_id_and_parent_path(
+                spec.eval_id, parent_task.path
+            )
             if eval:
                 eval.delete()