Require more APIs to use permission (#1199)

* gate more apis

* Update libs/server/kiln_server/document_api.py

allow agent to see progress

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

* Update libs/server/kiln_server/utils/agent_checks/annotations/post_api_projects_project_id_rag_configs_progress.json

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

---------

Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>

Author: tawnymanticore

app/desktop/studio_server/copilot_api.py

@@ -75,7 +75,7 @@
     RefineSpecApiOutput,
     SubmitAnswersRequest,
 )
-from kiln_server.utils.agent_checks.policy import ALLOW_AGENT
+from kiln_server.utils.agent_checks.policy import agent_policy_require_approval
 from pydantic import BaseModel, Field
 
 logger = logging.getLogger(__name__)
@@ -115,7 +115,11 @@ class CreateSpecWithCopilotRequest(BaseModel):
 
 
 def connect_copilot_api(app: FastAPI):
-    @app.post("/api/copilot/clarify_spec", tags=["Copilot"], openapi_extra=ALLOW_AGENT)
+    @app.post(
+        "/api/copilot/clarify_spec",
+        tags=["Copilot"],
+        openapi_extra=agent_policy_require_approval("Run Copilot spec clarification?"),
+    )
     async def clarify_spec(input: ClarifySpecApiInput) -> ClarifySpecApiOutput:
         api_key = get_copilot_api_key()
         client = get_authenticated_client(api_key)
@@ -141,7 +145,11 @@ async def clarify_spec(input: ClarifySpecApiInput) -> ClarifySpecApiOutput:
             detail="Unknown error.",
         )
 
-    @app.post("/api/copilot/refine_spec", tags=["Copilot"], openapi_extra=ALLOW_AGENT)
+    @app.post(
+        "/api/copilot/refine_spec",
+        tags=["Copilot"],
+        openapi_extra=agent_policy_require_approval("Run Copilot spec refinement?"),
+    )
     async def refine_spec(input: RefineSpecApiInput) -> RefineSpecApiOutput:
         api_key = get_copilot_api_key()
         client = get_authenticated_client(api_key)
@@ -168,7 +176,9 @@ async def refine_spec(input: RefineSpecApiInput) -> RefineSpecApiOutput:
         )
 
     @app.post(
-        "/api/copilot/generate_batch", tags=["Copilot"], openapi_extra=ALLOW_AGENT
+        "/api/copilot/generate_batch",
+        tags=["Copilot"],
+        openapi_extra=agent_policy_require_approval("Run Copilot batch generation?"),
     )
     async def generate_batch(input: GenerateBatchApiInput) -> GenerateBatchApiOutput:
         api_key = get_copilot_api_key()
@@ -195,7 +205,11 @@ async def generate_batch(input: GenerateBatchApiInput) -> GenerateBatchApiOutput
             detail="Unknown error.",
         )
 
-    @app.post("/api/copilot/question_spec", tags=["Copilot"], openapi_extra=ALLOW_AGENT)
+    @app.post(
+        "/api/copilot/question_spec",
+        tags=["Copilot"],
+        openapi_extra=agent_policy_require_approval("Run Copilot spec questioner?"),
+    )
     async def question_spec(
         input: SpecQuestionerApiInput,
     ) -> QuestionSet:
@@ -226,7 +240,9 @@ async def question_spec(
     @app.post(
         "/api/copilot/refine_spec_with_question_answers",
         tags=["Copilot"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run Copilot spec refinement with question answers?"
+        ),
     )
     async def submit_question_answers(
         request: SubmitAnswersRequest,
@@ -256,7 +272,7 @@ async def submit_question_answers(
     @app.post(
         "/api/projects/{project_id}/tasks/{task_id}/spec_with_copilot",
         tags=["Copilot"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Create spec with Copilot?"),
     )
     async def create_spec_with_copilot(
         project_id: Annotated[

app/desktop/studio_server/data_gen_api.py

@@ -23,7 +23,10 @@
 )
 from kiln_server.project_api import project_from_id
 from kiln_server.task_api import task_from_id
-from kiln_server.utils.agent_checks.policy import ALLOW_AGENT
+from kiln_server.utils.agent_checks.policy import (
+    ALLOW_AGENT,
+    agent_policy_require_approval,
+)
 from openai.types.chat import (
     ChatCompletionSystemMessageParam,
     ChatCompletionUserMessageParam,
@@ -127,7 +130,7 @@ def connect_data_gen_api(app: FastAPI):
         "/api/projects/{project_id}/tasks/{task_id}/generate_categories",
         summary="Generate Categories",
         tags=["Synthetic Data"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Generate categories using LLM?"),
     )
     async def generate_categories(
         project_id: Annotated[
@@ -172,7 +175,7 @@ async def generate_categories(
         "/api/projects/{project_id}/tasks/{task_id}/generate_inputs",
         summary="Generate Inputs",
         tags=["Synthetic Data"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Generate inputs using LLM?"),
     )
     async def generate_samples(
         project_id: Annotated[
@@ -241,7 +244,7 @@ async def save_sample(
         "/api/projects/{project_id}/tasks/{task_id}/generate_sample",
         summary="Generate Sample",
         tags=["Synthetic Data"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Generate a sample using LLM?"),
     )
     async def generate_sample(
         project_id: Annotated[
@@ -312,7 +315,7 @@ async def generate_sample(
         "/api/projects/{project_id}/tasks/{task_id}/generate_qna",
         summary="Generate Q&A Pairs",
         tags=["Synthetic Data"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Generate Q&A pairs using LLM?"),
     )
     async def generate_qna_pairs(
         project_id: Annotated[

app/desktop/studio_server/eval_api.py

@@ -764,7 +764,7 @@ async def create_eval_config(
         "/api/projects/{project_id}/tasks/{task_id}/evals/{eval_id}/eval_config/{eval_config_id}/run_comparison",
         summary="Run Run Config Comparison",
         tags=["Evals"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Run eval comparison?"),
     )
     async def run_eval_config(
         project_id: Annotated[
@@ -866,7 +866,9 @@ async def set_default_eval_config(
         "/api/projects/{project_id}/tasks/{task_id}/evals/{eval_id}/run_calibration",
         summary="Run Calibration",
         tags=["Evals"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run eval calibration? This runs LLM calls across all eval configs and uses AI credits."
+        ),
     )
     async def run_eval_config_eval(
         project_id: Annotated[

app/desktop/studio_server/finetune_api.py

@@ -53,6 +53,7 @@
 from kiln_server.task_api import task_from_id
 from kiln_server.utils.agent_checks.policy import (
     ALLOW_AGENT,
+    DENY_AGENT,
     agent_policy_require_approval,
 )
 from pydantic import BaseModel, Field, model_validator
@@ -642,7 +643,7 @@ async def create_finetune(
         "/api/download_dataset_jsonl",
         summary="Download Dataset JSONL",
         tags=["Fine-tuning"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=DENY_AGENT,
     )
     async def download_dataset_jsonl(
         project_id: Annotated[

app/desktop/studio_server/repair_api.py

@@ -14,7 +14,10 @@
 from kiln_ai.datamodel.task_output import DataSource, DataSourceType
 from kiln_ai.utils.config import Config
 from kiln_server.run_api import model_provider_from_string, task_and_run_from_id
-from kiln_server.utils.agent_checks.policy import ALLOW_AGENT
+from kiln_server.utils.agent_checks.policy import (
+    ALLOW_AGENT,
+    agent_policy_require_approval,
+)
 from pydantic import BaseModel, ConfigDict, Field, ValidationError
 
 
@@ -39,7 +42,7 @@ def connect_repair_api(app: FastAPI):
         "/api/projects/{project_id}/tasks/{task_id}/runs/{run_id}/generate_repair",
         summary="Generate Repair",
         tags=["Runs"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Generate a repair using LLM?"),
     )
     async def run_repair(
         project_id: Annotated[

libs/server/kiln_server/document_api.py

@@ -1373,7 +1373,9 @@ async def get_extractor_config(
     @app.get(
         "/api/projects/{project_id}/extractor_configs/{extractor_config_id}/run_extractor_config",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run document extractor? This processes all documents and may take significant time."
+        ),
     )
     async def run_extractor_config(
         project_id: Annotated[
@@ -1515,7 +1517,7 @@ async def get_extraction(
     @app.get(
         "/api/projects/{project_id}/documents/{document_id}/download",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=DENY_AGENT,
     )
     async def download_document_file(
         project_id: Annotated[
@@ -1547,7 +1549,7 @@ async def download_document_file(
     @app.get(
         "/api/projects/{project_id}/documents/{document_id}/download_extraction/{extraction_id}",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=DENY_AGENT,
     )
     async def download_extraction(
         project_id: Annotated[
@@ -1716,7 +1718,9 @@ async def get_extraction_progress(
     @app.post(
         "/api/projects/{project_id}/documents/{document_id}/extract",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run document extraction? This processes the document and may take time."
+        ),
     )
     async def extract_file(
         project_id: Annotated[
@@ -2349,7 +2353,9 @@ async def get_rag_config(
     @app.get(
         "/api/projects/{project_id}/rag_configs/{rag_config_id}/run",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run RAG config indexing? This re-indexes documents and may take time."
+        ),
     )
     async def run_rag_config(
         project_id: Annotated[
@@ -2484,7 +2490,9 @@ async def check_library_state(
     @app.post(
         "/api/projects/{project_id}/extractor_configs/{extractor_config_id}/documents/{document_id}/ephemeral_split",
         tags=["Documents"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval(
+            "Run ephemeral document split? This triggers backend processing."
+        ),
     )
     async def ephemeral_split_document(
         project_id: Annotated[

libs/server/kiln_server/run_api.py

@@ -377,7 +377,7 @@ async def delete_runs(
         "/api/projects/{project_id}/tasks/{task_id}/run",
         summary="Execute Run",
         tags=["Runs"],
-        openapi_extra=ALLOW_AGENT,
+        openapi_extra=agent_policy_require_approval("Run task with LLM?"),
     )
     async def run_task(
         project_id: Annotated[

libs/server/kiln_server/utils/agent_checks/annotations/get_api_download_dataset_jsonl.json

@@ -2,7 +2,7 @@
   "method": "get",
   "path": "/api/download_dataset_jsonl",
   "agent_policy": {
-    "permission": "allow",
+    "permission": "deny",
     "requires_approval": false
   }
 }

libs/server/kiln_server/utils/agent_checks/annotations/get_api_projects_project_id_documents_document_id_download.json

@@ -2,7 +2,7 @@
   "method": "get",
   "path": "/api/projects/{project_id}/documents/{document_id}/download",
   "agent_policy": {
-    "permission": "allow",
+    "permission": "deny",
     "requires_approval": false
   }
 }

libs/server/kiln_server/utils/agent_checks/annotations/get_api_projects_project_id_documents_document_id_download_extraction_extraction_id.json

@@ -2,7 +2,7 @@
   "method": "get",
   "path": "/api/projects/{project_id}/documents/{document_id}/download_extraction/{extraction_id}",
   "agent_policy": {
-    "permission": "allow",
+    "permission": "deny",
     "requires_approval": false
   }
 }