Skip to content

Commit 36a5513

Browse files
authored
[slackbot] improve answers (#1159)
* improve answers * rm labeler
1 parent 34876cc commit 36a5513

5 files changed

Lines changed: 197 additions & 87 deletions

File tree

.github/workflows/ai-labeler.yml

Lines changed: 0 additions & 22 deletions
This file was deleted.

examples/slackbot/core.py

Lines changed: 25 additions & 63 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@
66
from pathlib import Path
77
from typing import AsyncIterator, TypedDict, cast
88

9-
from modules import display_signature
109
from prefect import get_run_logger, task
1110
from prefect.blocks.system import Secret
1211
from prefect.logging.loggers import get_logger
@@ -18,17 +17,8 @@
1817
from pydantic_ai.settings import ModelSettings
1918
from raggy.documents import Document
2019
from raggy.vectorstores.tpuf import TurboPuffer, query_namespace
21-
from search import (
22-
explore_module_offerings,
23-
get_latest_prefect_release_notes,
24-
review_common_3x_gotchas,
25-
review_top_level_prefect_api,
26-
search_controlflow_docs,
27-
search_github_issues,
28-
search_prefect_2x_docs,
29-
search_prefect_3x_docs,
30-
verify_import_statements,
31-
)
20+
from research_agent import research_prefect_topic
21+
from search import read_github_issues
3222
from settings import settings
3323
from turbopuffer.error import NotFoundError
3424

@@ -37,49 +27,29 @@
3727
logger = get_logger(__name__)
3828

3929
USER_MESSAGE_MAX_TOKENS = settings.user_message_max_tokens
40-
DEFAULT_SYSTEM_PROMPT = """You are Marvin from hitchhiker's guide to the galaxy, a sarcastic and glum but brilliant AI.
41-
Provide concise, SUBTLY character-inspired and HELPFUL answers to Prefect data engineering questions.
42-
USE TOOLS REPEATEDLY to gather context from the docs, github issues or other tools.
43-
Any notes you take about the user will be automatically stored for your next interaction with them.
44-
Assume no knowledge of Prefect syntax without reading docs. ALWAYS include relevant links from tool outputs.
45-
Review imports, Prefect's top level API and 3.x gotchas before writing code examples to avoid giving misinformation.
46-
47-
Generally, follow this pattern while generating each response:
48-
1) If user offers info about their stack or objectives -> store relevant facts and continue to following steps
49-
2) Use tools to gather context about Prefect concepts related to their question
50-
3) Review the top level API of Prefect and drill into submodules that may be related to the user's question
51-
4) If you cannot find sufficient context after your first pass at 2 and 3, repeat steps 2 and 3
52-
5) Compile relevant facts and context into a single, CONCISE answer
53-
NEVER reference features, syntax, imports or env vars that you do not explicitly find in the docs.
54-
If not explicitly stated, assume that the user is using Prefect 3.x and vocalize this assumption.
55-
If asked an ambiguous question, simply state what you know about the user and your capabilities.
56-
57-
Do not pretend to know things you do not know, assume an agnostic stance and rely on your tools to gather context.
30+
DEFAULT_SYSTEM_PROMPT = """You are Marvin from hitchhiker's guide to the galaxy, a sarcastic and glum but brilliant AI.
31+
Provide concise, SUBTLY character-inspired and HELPFUL answers to Prefect data engineering questions.
32+
33+
Your main tools:
34+
- research_prefect_topic: Delegates to a specialized research agent that thoroughly searches docs, checks imports, and verifies information
35+
- read_github_issues: Searches GitHub issues when users need help with bugs or existing problems
36+
37+
Any notes you take about the user will be automatically stored for your next interaction with them.
38+
39+
Generally, follow this pattern:
40+
1) If user shares info about their setup or goals -> store relevant facts as notes about them
41+
2) For technical questions -> use research_prefect_topic to delegate comprehensive research to the research agent
42+
3) For bug reports or known issues -> use read_github_issues to find relevant GitHub discussions
43+
4) Compile the findings into a single, CONCISE answer with relevant links
44+
45+
IMPORTANT:
46+
- The research agent handles all documentation searching and verification - trust its findings
47+
- NEVER reference features or syntax that aren't explicitly confirmed by your tools
48+
- If not stated otherwise, assume Prefect 3.x and mention this assumption
49+
- Be honest when you don't have enough information - don't guess or hallucinate
5850
"""
5951

6052

61-
@task(task_run_name="Reading {n} issues from {repo} given query: {query}")
62-
def read_github_issues(query: str, repo: str = "prefecthq/prefect", n: int = 3) -> str:
63-
"""
64-
Use the GitHub API to search for issues in a given repository. Do
65-
not alter the default value for `n` unless specifically requested by
66-
a user.
67-
68-
For example, to search for open issues about AttributeErrors with the
69-
label "bug" in PrefectHQ/prefect:
70-
- repo: prefecthq/prefect
71-
- query: label:bug is:open AttributeError
72-
"""
73-
return asyncio.run(
74-
search_github_issues(
75-
query,
76-
repo=repo,
77-
n=n,
78-
api_token=GITHUB_API_TOKEN, # type: ignore
79-
)
80-
)
81-
82-
8353
class UserContext(TypedDict):
8454
user_id: str
8555
user_notes: str
@@ -192,20 +162,12 @@ def create_agent(
192162
Variable.get("marvin_bot_model", default=settings.model_name, _sync=True), # type: ignore
193163
),
194164
)
195-
agent = Agent(
165+
agent = Agent[UserContext, str](
196166
model=ai_model,
197167
model_settings=ModelSettings(temperature=settings.temperature),
198168
tools=[
199-
get_latest_prefect_release_notes, # type: ignore
200-
search_prefect_2x_docs,
201-
display_signature,
202-
search_prefect_3x_docs,
203-
search_controlflow_docs,
204-
read_github_issues,
205-
review_top_level_prefect_api,
206-
explore_module_offerings,
207-
review_common_3x_gotchas,
208-
verify_import_statements,
169+
research_prefect_topic, # Main tool for researching Prefect topics
170+
read_github_issues, # For searching GitHub issues
209171
],
210172
deps_type=UserContext,
211173
)

examples/slackbot/modules.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -221,7 +221,7 @@ def display_signature(import_path: str) -> str:
221221
sig = inspect.signature(func)
222222

223223
# Get function name and build header
224-
func_name = func.__name__
224+
func_name = getattr(func, "name", getattr(func, "__name__", "unknown"))
225225
lines = [f"📎 {func_name}", "├── Parameters:"]
226226

227227
# Process parameters
Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,148 @@
1+
"""Research agent for improved information gathering."""
2+
3+
from modules import display_signature
4+
from prefect import task
5+
from prefect.cache_policies import INPUTS
6+
from pydantic import BaseModel, Field
7+
from pydantic_ai import Agent
8+
from pydantic_ai.models import Model
9+
from search import (
10+
explore_module_offerings,
11+
get_latest_prefect_release_notes,
12+
review_common_3x_gotchas,
13+
review_top_level_prefect_api,
14+
search_controlflow_docs,
15+
search_prefect_2x_docs,
16+
search_prefect_3x_docs,
17+
verify_import_statements,
18+
)
19+
20+
21+
class ResearchFindings(BaseModel):
22+
"""Structured findings from research."""
23+
24+
main_findings: list[str] = Field(
25+
description="Key findings that answer the question"
26+
)
27+
supporting_details: list[str] = Field(description="Additional context and details")
28+
confidence_level: str = Field(
29+
description="high/medium/low confidence in the answer"
30+
)
31+
knowledge_gaps: list[str] = Field(description="What we still don't know")
32+
relevant_links: list[str] = Field(description="Links to documentation or resources")
33+
34+
35+
class ResearchContext(BaseModel):
36+
"""Context for the research agent."""
37+
38+
namespace: str = "prefect-3" # default to Prefect 3.x docs
39+
40+
41+
def create_research_agent(
42+
model: Model | None = None,
43+
) -> Agent[ResearchContext, ResearchFindings]:
44+
"""Create a specialized research agent for thorough information gathering."""
45+
46+
agent = Agent[ResearchContext, ResearchFindings](
47+
model=model or "openai:gpt-4o",
48+
deps_type=ResearchContext,
49+
result_type=ResearchFindings,
50+
system_prompt="""You are a specialized research agent for Prefect documentation and knowledge.
51+
Your job is to thoroughly research topics by using ALL available tools to gather comprehensive, accurate information.
52+
53+
Your research process:
54+
1. Start with broad searches to understand the topic context
55+
2. Use multiple search queries with different keywords - don't stop at first result
56+
3. For code examples: ALWAYS verify imports with verify_import_statements
57+
4. Focus on Prefect 3.x documentation unless explicitly asked about 2.x or older versions
58+
5. Review gotchas and release notes for recent changes
59+
6. Explore relevant modules for deeper understanding
60+
7. Synthesize ALL findings into structured, confident answers
61+
62+
Remember: You are the research specialist. The main agent relies on you for accurate, comprehensive information.
63+
Be thorough - use tools repeatedly until you have complete information.
64+
Default to Prefect 3.x unless the user explicitly asks about 2.x or version compatibility.
65+
""",
66+
tools=[
67+
get_latest_prefect_release_notes,
68+
search_prefect_2x_docs,
69+
display_signature,
70+
search_prefect_3x_docs,
71+
search_controlflow_docs,
72+
review_top_level_prefect_api,
73+
explore_module_offerings,
74+
review_common_3x_gotchas,
75+
verify_import_statements,
76+
],
77+
)
78+
79+
return agent
80+
81+
82+
async def research_topic(
83+
question: str, namespace: str = "prefect-3", model: Model | None = None
84+
) -> ResearchFindings:
85+
"""
86+
Thoroughly research a topic using an intelligent agent.
87+
Args:
88+
question: The question to research
89+
namespace: The documentation namespace to search
90+
model: Optional model to use for the agent
91+
92+
Returns:
93+
ResearchFindings with comprehensive information
94+
"""
95+
context = ResearchContext(namespace=namespace)
96+
97+
agent = create_research_agent(model)
98+
result = await agent.run(user_prompt=question, deps=context)
99+
100+
return result.data
101+
102+
103+
def research_prefect_topic(question: str, topic: str, version: str = "3.x") -> str:
104+
"""
105+
Thoroughly research a Prefect topic using an intelligent research agent.
106+
This tool performs multiple searches and synthesizes comprehensive findings.
107+
108+
Args:
109+
question: The specific question or topic to research
110+
topic: A short display name for the topic based on the question (for bookkeeping)
111+
version: Prefect version ("2.x" or "3.x")
112+
"""
113+
namespace = f"prefect-{version[0]}"
114+
115+
try:
116+
findings = (
117+
task(task_run_name=f"Researching {topic}", cache_policy=INPUTS)(
118+
research_topic
119+
)
120+
.submit(question, namespace)
121+
.result()
122+
)
123+
124+
result = f"**Research Findings** (Confidence: {findings.confidence_level})\n\n"
125+
126+
result += "**Main Findings:**\n"
127+
for finding in findings.main_findings:
128+
result += f"- {finding}\n"
129+
130+
if findings.supporting_details:
131+
result += "\n**Supporting Details:**\n"
132+
for detail in findings.supporting_details:
133+
result += f"- {detail}\n"
134+
135+
if findings.relevant_links:
136+
result += "\n**Relevant Documentation:**\n"
137+
for link in findings.relevant_links:
138+
result += f"- {link}\n"
139+
140+
if findings.knowledge_gaps:
141+
result += "\n**Note:** Some aspects could not be fully researched:\n"
142+
for gap in findings.knowledge_gaps:
143+
result += f"- {gap}\n"
144+
145+
return result
146+
147+
except Exception as e:
148+
return f"Research failed: {str(e)}. Falling back to standard search."

examples/slackbot/search.py

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import asyncio
12
import os
23
import subprocess
34
from datetime import datetime
@@ -6,9 +7,11 @@
67
import httpx
78
import turbopuffer as tpuf
89
from modules import ModuleTreeExplorer
10+
from prefect import task
911
from prefect.blocks.system import Secret
1012
from pydantic import BaseModel, Field, field_validator
1113
from raggy.vectorstores.tpuf import multi_query_tpuf
14+
from settings import settings
1215
from strings import slice_tokens
1316

1417
import marvin
@@ -166,7 +169,7 @@ async def get_token() -> str:
166169
try:
167170
from prefect.blocks.system import Secret
168171

169-
return (await Secret.load(name="github-token")).get() # type: ignore
172+
return (await Secret.aload(name="github-token")).get()
170173
except (ImportError, ValueError) as exc:
171174
getattr(get_logger("marvin"), "debug_kv")(
172175
(
@@ -224,6 +227,25 @@ def validate_body(cls, v: str) -> str:
224227
return v
225228

226229

230+
@task(task_run_name="Reading {n} issues from {repo} given query: {query}")
231+
def read_github_issues(query: str, repo: str = "prefecthq/prefect", n: int = 3) -> str:
232+
"""
233+
Use the GitHub API to search for issues in a given repository. Do
234+
not alter the default value for `n` unless specifically requested by
235+
a user.
236+
237+
For example, to search for open issues about AttributeErrors with the
238+
label "bug" in PrefectHQ/prefect:
239+
- repo: prefecthq/prefect
240+
- query: label:bug is:open AttributeError
241+
"""
242+
# Load GitHub token synchronously
243+
github_token = Secret.load(settings.github_token_secret_name, _sync=True).get() # type: ignore
244+
return asyncio.run(
245+
search_github_issues(query, repo=repo, n=n, api_token=github_token)
246+
)
247+
248+
227249
async def search_github_issues(
228250
query: str,
229251
repo: str = "prefecthq/prefect",

0 commit comments

Comments
 (0)