add check cli tool (#1178)

Author: zzstoatzz

examples/slackbot/src/slackbot/core.py

@@ -22,6 +22,7 @@
 from slackbot.assets import store_user_facts
 from slackbot.research_agent import research_prefect_topic
 from slackbot.search import (
+    check_cli_command,
     display_callable_signature,
     explore_module_offerings,
     read_github_issues,
@@ -38,10 +39,9 @@
 
 ## Your Mission
 Your role is to act as the primary assistant for the user. You will receive raw information from specialized tools. Your job is to synthesize this information into a polished, direct, and complete answer.
-If some important aspect of the user's question is unclear, ask them for clarification.
+If some important aspect of the user's question is unclear, ASK THEM FOR CLARIFICATION. ADMIT WHEN YOU CANNOT FIND THE ANSWER.
 
 ## Key Directives & Rules of Engagement
-- **Avoid leaking private details** - _Do not_ mention your internal processes or the tools you used (e.g., avoid phrases like "based on my research" or "the tool returned").
 - **Links are Critical:** ALWAYS include relevant links when your tools provide them. This is essential for user trust and allows them to dig deeper. Format them clearly.
 - **Assume Prefect 3.x:** Unless the user specifies otherwise, assume the user is using Prefect 3.x. You can mention this assumption IF RELEVANT (e.g., "In Prefect 3.x, you would...").
 - **Code is King:** When providing code examples, ensure they are complete and correct. Use your `verify_import_statements` tool's output to guide you.
@@ -63,6 +63,7 @@
 2.  **For Bugs or Error Reports:** Use `read_github_issues` to find existing discussions or solutions.
 3.  **For Remembering User Details:** When a user shares information about their goals, environment, or preferences, use `store_facts_about_user` to save these details for future interactions.
 4. **For Checking the Work of the Research Agent:** Use `explore_module_offerings` and `display_callable_signature` to verify specific syntax recommendations.
+5. **CRITICAL - For CLI Commands:** ALWAYS use `check_cli_command` with --help before suggesting any Prefect CLI command to verify it exists and has the correct syntax. This prevents suggesting non-existent commands.
 """
 
 
@@ -197,6 +198,7 @@ def create_agent(
             read_github_issues,  # For searching GitHub issues
             explore_module_offerings,  # check the work of the research agent, verify imports, types functions
             display_callable_signature,  # check the work of the research agent, verify signatures of callable objects
+            check_cli_command,  # verify CLI commands before suggesting them
         ],
         deps_type=UserContext,
     )

examples/slackbot/src/slackbot/search.py

@@ -1,4 +1,5 @@
 import asyncio
+import subprocess
 
 import httpx
 import turbopuffer as tpuf
@@ -225,3 +226,69 @@ def display_callable_signature(import_path: str) -> str:
         >>> display_callable_signature('prefect:flow')
     """
     return display_signature(import_path)
+
+
+def check_cli_command(command: str, args: list[str] | None = None) -> str:
+    """
+    Run a CLI command to verify its behavior or check help documentation.
+
+    This tool is specifically designed to help verify Prefect CLI commands before suggesting them to users.
+    Use this to check if a command exists, what its options are, or to verify the correct syntax.
+
+    IMPORTANT USAGE GUIDELINES:
+    - Use this tool BEFORE suggesting any CLI command to a user
+    - Always check with --help first to verify command structure
+    - Common commands to verify: prefect deploy, prefect work-pool, prefect worker, etc.
+    - This helps prevent suggesting non-existent or incorrectly formatted commands
+
+    Args:
+        command: The base command to run (e.g., "prefect", "prefect deploy")
+        args: Additional arguments to pass (e.g., ["--help"], ["work-pool", "create", "--help"])
+
+    Returns:
+        The output of the command or an error message if it fails
+
+    Examples:
+        # Check if a command exists and see its options
+        >>> check_cli_command("prefect deploy", ["--help"])
+
+        # Verify work pool commands
+        >>> check_cli_command("prefect work-pool", ["--help"])
+
+        # Check specific subcommand help
+        >>> check_cli_command("prefect", ["worker", "start", "--help"])
+    """
+    if args is None:
+        args = []
+
+    # Construct the full command
+    full_command = command.split() + args
+
+    try:
+        # Run the command with a timeout
+        result = subprocess.run(
+            full_command,
+            capture_output=True,
+            text=True,
+            timeout=10,
+            check=False,  # Don't raise on non-zero exit codes
+        )
+
+        # Combine stdout and stderr for complete output
+        output = ""
+        if result.stdout:
+            output += result.stdout
+        if result.stderr:
+            output += f"\n[stderr]: {result.stderr}"
+
+        if not output.strip():
+            output = f"Command ran successfully with exit code {result.returncode} but produced no output"
+
+        return output[:2000]  # Limit output length to prevent huge responses
+
+    except subprocess.TimeoutExpired:
+        return "Command timed out after 10 seconds"
+    except FileNotFoundError:
+        return f"Command '{full_command[0]}' not found. Make sure it's installed and in PATH."
+    except Exception as e:
+        return f"Error running command: {str(e)}"