Merge branch 'main' into KIL-506/task-warn

Author: sfierro

app/desktop/studio_server/eval_api.py

@@ -25,6 +25,7 @@
 from kiln_ai.datamodel.json_schema import string_to_json_key
 from kiln_ai.datamodel.prompt_id import is_frozen_prompt
 from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties
+from kiln_ai.datamodel.spec import SpecStatus
 from kiln_ai.datamodel.task import RunConfigProperties, TaskRunConfig
 from kiln_ai.datamodel.task_output import normalize_rating
 from kiln_ai.utils.name_generator import generate_memorable_name
@@ -1271,11 +1272,15 @@ async def get_run_config_eval_scores(
         task_run_config_from_id(project_id, task_id, run_config_id)
 
         # Build a mapping from eval_id to spec_id for evals that are associated with specs
+        # Also track which eval_ids belong to archived specs so we can exclude them
         specs = task.specs()
         eval_id_to_spec_id: Dict[str, str] = {}
+        archived_eval_ids: set[str] = set()
         for spec in specs:
             if spec.eval_id and spec.id:
                 eval_id_to_spec_id[spec.eval_id] = spec.id
+                if spec.status == SpecStatus.archived:
+                    archived_eval_ids.add(spec.eval_id)
 
         evals = task.evals()
         eval_results: List[RunConfigEvalResult] = []
@@ -1292,6 +1297,10 @@ async def get_run_config_eval_scores(
         total_eval_runs = 0
 
         for eval in evals:
+            # Skip evals associated with archived specs
+            if eval.id and eval.id in archived_eval_ids:
+                continue
+
             # Get the dataset size for this eval
             expected_dataset_ids = dataset_ids_in_filter(
                 task, eval.eval_set_filter_id, readonly=True

app/desktop/studio_server/test_eval_api.py

@@ -47,7 +47,7 @@
 )
 from kiln_ai.datamodel.prompt import BasePrompt
 from kiln_ai.datamodel.run_config import KilnAgentRunConfigProperties
-from kiln_ai.datamodel.spec import Spec
+from kiln_ai.datamodel.spec import Spec, SpecStatus
 from kiln_ai.datamodel.spec_properties import DesiredBehaviourProperties, SpecType
 from kiln_ai.datamodel.task import TaskRunConfig
 from kiln_ai.datamodel.task_run import Usage
@@ -2362,6 +2362,132 @@ async def test_get_run_config_eval_scores_includes_spec_id(
     assert legacy_eval_result["spec_id"] is None
 
 
+@pytest.mark.asyncio
+async def test_get_run_config_eval_scores_excludes_archived_specs(
+    client, mock_task, mock_eval, mock_eval_config, mock_run_config
+):
+    """Test that get_run_config_eval_scores excludes evals associated with archived specs"""
+
+    # Create an active spec
+    active_spec = Spec(
+        id="active_spec1",
+        name="Active Spec",
+        definition="Active spec definition",
+        properties=DesiredBehaviourProperties(
+            spec_type=SpecType.desired_behaviour,
+            core_requirement="test instruction",
+            desired_behaviour_description="test desired behaviour",
+        ),
+        eval_id=mock_eval.id,
+        status=SpecStatus.active,
+        parent=mock_task,
+    )
+    active_spec.save_to_file()
+
+    # Create an archived spec with its own eval
+    archived_eval = Eval(
+        id="archived_eval1",
+        name="Archived Eval",
+        description="Eval for archived spec",
+        template=None,
+        eval_set_filter_id="tag::archived_eval_set",
+        eval_configs_filter_id="tag::archived_golden",
+        output_scores=[
+            EvalOutputScore(
+                name="score1",
+                instruction="desc1",
+                type=TaskOutputRatingType.five_star,
+            ),
+        ],
+        parent=mock_task,
+    )
+    archived_eval.save_to_file()
+
+    archived_eval_config = EvalConfig(
+        id="archived_eval_config1",
+        name="Archived Eval Config",
+        config_type=EvalConfigType.g_eval,
+        properties={"eval_steps": ["step1"]},
+        parent=archived_eval,
+        model_name="gpt-4",
+        model_provider="openai",
+    )
+    archived_eval_config.save_to_file()
+    archived_eval.current_config_id = archived_eval_config.id
+    archived_eval.save_to_file()
+
+    archived_spec = Spec(
+        id="archived_spec1",
+        name="Archived Spec",
+        definition="Archived spec definition",
+        properties=DesiredBehaviourProperties(
+            spec_type=SpecType.desired_behaviour,
+            core_requirement="test instruction",
+            desired_behaviour_description="test desired behaviour",
+        ),
+        eval_id=archived_eval.id,
+        status=SpecStatus.archived,
+        parent=mock_task,
+    )
+    archived_spec.save_to_file()
+
+    # Build mock eval objects with explicit attributes
+    mock_eval_config_for_api = MagicMock()
+    mock_eval_config_for_api.id = mock_eval_config.id
+    mock_eval_config_for_api.runs.return_value = []
+
+    mock_eval_for_api = MagicMock()
+    mock_eval_for_api.id = mock_eval.id
+    mock_eval_for_api.name = mock_eval.name
+    mock_eval_for_api.eval_set_filter_id = mock_eval.eval_set_filter_id
+    mock_eval_for_api.output_scores = mock_eval.output_scores
+    mock_eval_for_api.current_config_id = mock_eval_config.id
+    mock_eval_for_api.configs.return_value = [mock_eval_config_for_api]
+
+    archived_eval_config_for_api = MagicMock()
+    archived_eval_config_for_api.id = archived_eval_config.id
+    archived_eval_config_for_api.runs.return_value = []
+
+    archived_eval_for_api = MagicMock()
+    archived_eval_for_api.id = archived_eval.id
+    archived_eval_for_api.name = archived_eval.name
+    archived_eval_for_api.eval_set_filter_id = archived_eval.eval_set_filter_id
+    archived_eval_for_api.output_scores = archived_eval.output_scores
+    archived_eval_for_api.current_config_id = archived_eval_config.id
+    archived_eval_for_api.configs.return_value = [archived_eval_config_for_api]
+
+    mock_task_for_api = MagicMock()
+    mock_task_for_api.evals.return_value = [mock_eval_for_api, archived_eval_for_api]
+    mock_task_for_api.specs.return_value = [active_spec, archived_spec]
+
+    with (
+        patch(
+            "app.desktop.studio_server.eval_api.task_from_id"
+        ) as mock_task_from_id_patch,
+        patch(
+            "app.desktop.studio_server.eval_api.task_run_config_from_id"
+        ) as mock_task_run_config_from_id_patch,
+        patch(
+            "app.desktop.studio_server.eval_api.dataset_ids_in_filter"
+        ) as mock_dataset_ids_in_filter,
+    ):
+        mock_task_from_id_patch.return_value = mock_task_for_api
+        mock_task_run_config_from_id_patch.return_value = mock_run_config
+        mock_dataset_ids_in_filter.return_value = set()
+
+        response = client.get(
+            f"/api/projects/project1/tasks/task1/run_configs/{mock_run_config.id}/eval_scores"
+        )
+
+    assert response.status_code == 200
+    data = response.json()
+
+    # Only the active spec's eval should be present, not the archived one
+    assert len(data["eval_results"]) == 1
+    assert data["eval_results"][0]["eval_name"] == "Test Eval"
+    assert data["eval_results"][0]["spec_id"] == "active_spec1"
+
+
 @pytest.mark.asyncio
 async def test_get_run_configs_includes_finetunes_with_run_config(
     client, mock_task_from_id, mock_task

app/web_ui/src/lib/types.ts

@@ -36,6 +36,7 @@ export type DockerModelRunnerConnection =
   components["schemas"]["DockerModelRunnerConnection"]
 export type RunSummary = components["schemas"]["RunSummary"]
 export type PromptResponse = components["schemas"]["PromptResponse"]
+export type ApiPrompt = components["schemas"]["ApiPrompt"]
 export type ChatStrategy = components["schemas"]["ChatStrategy"]
 export type EvalOutputScore = components["schemas"]["EvalOutputScore"]
 export type EvalTemplateId = components["schemas"]["EvalTemplateId"]

app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/+page.svelte

@@ -11,7 +11,7 @@
     prompts_by_task_composite_id,
   } from "$lib/stores/prompts_store"
   import { onMount } from "svelte"
-  import type { Task } from "$lib/types"
+  import type { Task, ApiPrompt } from "$lib/types"
   import { createKilnError, KilnError } from "$lib/utils/error_handlers"
   import { getPromptType } from "./prompt_generators/prompt_generators"
   import InfoTooltip from "$lib/ui/info_tooltip.svelte"
@@ -88,6 +88,12 @@
     goto(`/prompts/${project_id}/${task_id}/edit_base_prompt`)
   }
 
+  function handleClonePrompt(prompt: ApiPrompt) {
+    goto(
+      `/prompts/${project_id}/${task_id}/clone/${encodeURIComponent(prompt.id)}`,
+    )
+  }
+
   type TableColumn = {
     key: string
     label: string
@@ -303,6 +309,13 @@
                                   Set as Base Prompt
                                 </button>
                               </li>
+                              <li>
+                                <button
+                                  on:click={() => handleClonePrompt(prompt)}
+                                >
+                                  Clone
+                                </button>
+                              </li>
                             </ul>
                           </Float>
                         </div>

app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/clone/[prompt_id]/+page.svelte

@@ -0,0 +1,85 @@
+<script lang="ts">
+  import AppPage from "../../../../../app_page.svelte"
+  import { page } from "$app/stores"
+  import { KilnError, createKilnError } from "$lib/utils/error_handlers"
+  import {
+    load_task_prompts,
+    prompts_by_task_composite_id,
+  } from "$lib/stores/prompts_store"
+  import { get_task_composite_id } from "$lib/stores"
+  import { onMount } from "svelte"
+  import PromptForm from "../../prompt_form.svelte"
+
+  $: project_id = $page.params.project_id!
+  $: task_id = $page.params.task_id!
+  $: prompt_id = $page.params.prompt_id!
+
+  let initial_prompt_name = ""
+  let initial_prompt = ""
+  let initial_chain_of_thought_instructions: string | null = null
+  let loading = true
+  let loading_error: KilnError | null = null
+
+  onMount(async () => {
+    try {
+      await load_task_prompts(project_id, task_id)
+      const task_prompts =
+        $prompts_by_task_composite_id[
+          get_task_composite_id(project_id, task_id)
+        ]
+      const source_prompt = task_prompts?.prompts.find(
+        (p) => p.id === prompt_id,
+      )
+
+      if (!source_prompt) {
+        throw new KilnError("Source prompt not found.")
+      }
+
+      initial_prompt_name = `Copy of ${source_prompt.name}`
+      initial_prompt = source_prompt.prompt
+      initial_chain_of_thought_instructions =
+        source_prompt.chain_of_thought_instructions || null
+    } catch (e) {
+      loading_error = createKilnError(e)
+    } finally {
+      loading = false
+    }
+  })
+</script>
+
+<div class="max-w-[1400px]">
+  <AppPage
+    title="Clone Prompt"
+    sub_subtitle="Read the Docs"
+    sub_subtitle_link="https://docs.kiln.tech/docs/prompts"
+    breadcrumbs={[
+      {
+        label: "Optimize",
+        href: `/optimize/${project_id}/${task_id}`,
+      },
+      {
+        label: "Prompts",
+        href: `/prompts/${project_id}/${task_id}`,
+      },
+    ]}
+  >
+    {#if loading}
+      <div class="w-full min-h-[50vh] flex justify-center items-center">
+        <div class="loading loading-spinner loading-lg"></div>
+      </div>
+    {:else if loading_error}
+      <div class="text-error text-sm">
+        {loading_error.getMessage() || "An unknown error occurred"}
+      </div>
+    {:else}
+      <PromptForm
+        {project_id}
+        {task_id}
+        clone_mode={true}
+        {initial_prompt_name}
+        {initial_prompt}
+        {initial_chain_of_thought_instructions}
+      />
+    {/if}
+  </AppPage>
+</div>

app/web_ui/src/routes/(app)/prompts/[project_id]/[task_id]/clone/[prompt_id]/+page.ts

@@ -0,0 +1 @@
+export const prerender = false