Add runtime UI element inspection

Author: Joshua

docs/TOOLS.md

@@ -65,7 +65,7 @@ Calls take the form:
 | `camera_manage` | `create`, `configure`, `set_limits_2d`, `set_damping_2d`, `follow_2d`, `get`, `list`, `apply_preset` |
 | `signal_manage` | `list`, `connect`, `disconnect` |
 | `input_map_manage` | `list`, `add_action`, `remove_action`, `bind_event` |
-| `game_manage` | `get_scene_tree`, `get_node_info`, `input_key`, `input_mouse`, `input_gamepad`, `input_state` |
+| `game_manage` | `get_scene_tree`, `get_node_info`, `get_ui_elements`, `input_key`, `input_mouse`, `input_gamepad`, `input_state` |
 | `autoload_manage` | `list`, `add`, `remove` |
 | `filesystem_manage` | `read_text`, `write_text`, `reimport`, `search` |
 | `theme_manage` | `create`, `set_color`, `set_constant`, `set_font_size`, `set_stylebox_flat`, `apply` |

plugin/addons/godot_ai/runtime/game_helper.gd

@@ -197,6 +197,8 @@ func _handle_game_command(data: Array) -> void:
 			result = _game_get_scene_tree(json.data)
 		"get_node_info":
 			result = _game_get_node_info(json.data)
+		"get_ui_elements":
+			result = _game_get_ui_elements(json.data)
 		"input_key":
 			result = _game_input_key(json.data)
 		"input_mouse":
@@ -267,6 +269,94 @@ func _game_get_node_info(params: Dictionary) -> Dictionary:
 	return info
 
 
+func _game_get_ui_elements(params: Dictionary) -> Dictionary:
+	var max_depth := maxi(0, int(params.get("max_depth", 10)))
+	var include_hidden := bool(params.get("include_hidden", false))
+	var include_disabled := bool(params.get("include_disabled", true))
+	var root_path := str(params.get("root_path", ""))
+	var root := _resolve_runtime_node(root_path)
+	if root == null:
+		return {"root": "", "elements": [], "total_count": 0, "not_found": root_path}
+
+	var elements: Array[Dictionary] = []
+	_collect_ui_elements(root, 0, max_depth, include_hidden, include_disabled, elements)
+	return {
+		"root": _runtime_path(root),
+		"elements": elements,
+		"total_count": elements.size(),
+	}
+
+
+func _collect_ui_elements(
+	node: Node,
+	current_depth: int,
+	max_depth: int,
+	include_hidden: bool,
+	include_disabled: bool,
+	out: Array[Dictionary]
+) -> void:
+	if node is Control:
+		var control := node as Control
+		var visible := _control_visible_in_tree(control)
+		var disabled := _control_disabled(control)
+		if (include_hidden or visible) and (include_disabled or not disabled):
+			out.append(_ui_element_info(control, visible, disabled))
+
+	if current_depth >= max_depth:
+		return
+	for child in node.get_children():
+		if child is Node:
+			_collect_ui_elements(
+				child,
+				current_depth + 1,
+				max_depth,
+				include_hidden,
+				include_disabled,
+				out
+			)
+
+
+func _ui_element_info(control: Control, visible: bool, disabled: bool) -> Dictionary:
+	var info := {
+		"path": _runtime_path(control),
+		"name": control.name,
+		"type": control.get_class(),
+		"visible": visible,
+		"disabled": disabled,
+		"rect": _variant_to_json(control.get_rect()),
+		"global_rect": _variant_to_json(control.get_global_rect()),
+	}
+	if _object_has_property(control, "text"):
+		info["text"] = str(control.get("text"))
+	return info
+
+
+func _control_disabled(control: Control) -> bool:
+	if _object_has_property(control, "disabled"):
+		return bool(control.get("disabled"))
+	return false
+
+
+func _control_visible_in_tree(control: Control) -> bool:
+	if not control.visible:
+		return false
+	var parent := control.get_parent()
+	while parent != null:
+		if parent is CanvasItem and not (parent as CanvasItem).visible:
+			return false
+		parent = parent.get_parent()
+	if Engine.is_editor_hint():
+		return true
+	return control.is_visible_in_tree()
+
+
+func _object_has_property(obj: Object, property_name: String) -> bool:
+	for prop in obj.get_property_list():
+		if str(prop.get("name", "")) == property_name:
+			return true
+	return false
+
+
 func _runtime_node_properties(node: Node) -> Dictionary:
 	var props := {}
 	for p in node.get_property_list():
@@ -279,7 +369,7 @@ func _runtime_node_properties(node: Node) -> Dictionary:
 
 
 func _resolve_runtime_node(path: String) -> Node:
-	var scene_root := get_tree().current_scene
+	var scene_root := _current_scene_root()
 	if scene_root == null:
 		return null
 	if path.is_empty() or path == "/":
@@ -298,14 +388,24 @@ func _resolve_runtime_node(path: String) -> Node:
 
 
 func _runtime_path(node: Node) -> String:
-	var scene_root := get_tree().current_scene
+	var scene_root := _current_scene_root()
 	if scene_root == null:
 		return str(node.get_path())
 	if node == scene_root:
 		return "/" + str(scene_root.name)
 	return "/" + str(scene_root.name) + "/" + str(scene_root.get_path_to(node))
 
 
+func _current_scene_root() -> Node:
+	var tree := get_tree()
+	if tree == null:
+		return null
+	var scene_root := tree.current_scene
+	if scene_root == null and Engine.is_editor_hint():
+		scene_root = EditorInterface.get_edited_scene_root()
+	return scene_root
+
+
 func _game_input_key(params: Dictionary) -> Dictionary:
 	var key_name := str(params.get("key", ""))
 	var keycode := OS.find_keycode_from_string(key_name)

src/godot_ai/handlers/game.py

@@ -40,6 +40,23 @@ async def game_get_node_info(
     )
 
 
+async def game_get_ui_elements(
+    runtime: DirectRuntime,
+    root_path: str = "",
+    include_hidden: bool = False,
+    include_disabled: bool = True,
+    max_depth: int = 10,
+) -> dict:
+    params: dict[str, Any] = {
+        "include_hidden": include_hidden,
+        "include_disabled": include_disabled,
+        "max_depth": max_depth,
+    }
+    if root_path:
+        params["root_path"] = root_path
+    return await _game_command(runtime, "get_ui_elements", params)
+
+
 async def game_input_key(
     runtime: DirectRuntime,
     key: str,

src/godot_ai/server.py

@@ -195,8 +195,8 @@ def _emit_startup() -> None:
             "                   follow_2d, get, list, apply_preset\n"
             "  signal_manage    list, connect, disconnect\n"
             "  input_map_manage list, add_action, remove_action, bind_event\n"
-            "  game_manage      get_scene_tree, get_node_info, input_key,\n"
-            "                   input_mouse, input_gamepad, input_state\n"
+            "  game_manage      get_scene_tree, get_node_info, get_ui_elements,\n"
+            "                   input_key, input_mouse, input_gamepad, input_state\n"
             "  autoload_manage  list, add, remove\n"
             "  filesystem_manage read_text, write_text, reimport, search\n"
             "  theme_manage     create, set_color, set_constant, set_font_size,\n"

src/godot_ai/tools/game.py

@@ -20,6 +20,10 @@
         path or a scene-relative path rooted at the current scene.
   - get_node_info(path, include_properties=True)
         Inspect one running node's metadata and optional property snapshot.
+  - get_ui_elements(root_path="", include_hidden=False,
+                    include_disabled=True, max_depth=10)
+        Inspect visible runtime Control nodes for UI testing. Includes path,
+        type, text where present, disabled state, and rect metadata.
   - input_key(key, pressed=True, echo=False)
         Send a key press/release to the running game.
   - input_mouse(event, position=None, button="left", pressed=True)
@@ -38,6 +42,7 @@ def register_game_tools(mcp: FastMCP) -> None:
         ops={
             "get_scene_tree": game_handlers.game_get_scene_tree,
             "get_node_info": game_handlers.game_get_node_info,
+            "get_ui_elements": game_handlers.game_get_ui_elements,
             "input_key": game_handlers.game_input_key,
             "input_mouse": game_handlers.game_input_mouse,
             "input_gamepad": game_handlers.game_input_gamepad,
@@ -46,6 +51,7 @@ def register_game_tools(mcp: FastMCP) -> None:
         read_resource_forms={
             "get_scene_tree": None,
             "get_node_info": None,
+            "get_ui_elements": None,
             "input_key": None,
             "input_mouse": None,
             "input_gamepad": None,

test_project/tests/test_game_helper.gd

@@ -0,0 +1,119 @@
+@tool
+extends McpTestSuite
+
+const GameHelper := preload("res://addons/godot_ai/runtime/game_helper.gd")
+
+const ROOT_NAME := "McpUiElementsRoot"
+
+var _helper: Node
+var _root: Node
+
+
+func suite_name() -> String:
+	return "game_helper"
+
+
+func suite_setup(_ctx: Dictionary) -> void:
+	var scene_root := EditorInterface.get_edited_scene_root()
+	if scene_root == null:
+		fail_setup("current_scene required")
+		return
+	_helper = GameHelper.new()
+	scene_root.add_child(_helper)
+	_root = CanvasLayer.new()
+	_root.name = ROOT_NAME
+	scene_root.add_child(_root)
+
+
+func suite_teardown() -> void:
+	if _root != null:
+		_root.queue_free()
+		_root = null
+	if _helper != null:
+		_helper.queue_free()
+		_helper = null
+
+
+func setup() -> void:
+	if _root == null:
+		return
+	for child in _root.get_children():
+		_root.remove_child(child)
+		child.free()
+
+
+func test_get_ui_elements_returns_controls_with_text_and_rects() -> void:
+	assert_true(_helper.has_method("_game_get_ui_elements"),
+		"game helper should expose get_ui_elements")
+	var container := Node.new()
+	container.name = "Container"
+	_root.add_child(container)
+
+	var title := Label.new()
+	title.name = "Title"
+	title.text = "Score: 10"
+	title.position = Vector2(10, 20)
+	title.size = Vector2(120, 30)
+	container.add_child(title)
+
+	var button := Button.new()
+	button.name = "StartButton"
+	button.text = "Start"
+	button.disabled = true
+	button.position = Vector2(20, 60)
+	button.size = Vector2(90, 40)
+	container.add_child(button)
+
+	var result = _helper.call("_game_get_ui_elements", {
+		"root_path": "/Main/%s" % ROOT_NAME,
+		"include_hidden": true,
+		"max_depth": 4,
+	})
+
+	assert_true(result is Dictionary, "get_ui_elements should return a Dictionary")
+	assert_eq(result.root, "/Main/%s" % ROOT_NAME)
+	assert_eq(result.total_count, 2)
+	assert_eq(result.elements[0].name, "Title")
+	assert_eq(result.elements[0].type, "Label")
+	assert_eq(result.elements[0].text, "Score: 10")
+	assert_has_key(result.elements[0], "visible")
+	assert_eq(result.elements[0].disabled, false)
+	assert_eq(result.elements[0].rect.position.x, 10.0)
+	assert_eq(result.elements[0].rect.size.y, 30.0)
+	assert_eq(result.elements[1].name, "StartButton")
+	assert_eq(result.elements[1].disabled, true)
+	assert_eq(result.elements[1].text, "Start")
+
+
+func test_get_ui_elements_can_filter_disabled_and_include_hidden() -> void:
+	assert_true(_helper.has_method("_game_get_ui_elements"),
+		"game helper should expose get_ui_elements")
+	var visible_enabled := LineEdit.new()
+	visible_enabled.name = "NameInput"
+	visible_enabled.text = "Ada"
+	_root.add_child(visible_enabled)
+
+	var disabled_button := Button.new()
+	disabled_button.name = "DisabledButton"
+	disabled_button.disabled = true
+	_root.add_child(disabled_button)
+
+	var hidden_label := Label.new()
+	hidden_label.name = "HiddenButIncluded"
+	hidden_label.text = "Hidden"
+	hidden_label.visible = false
+	_root.add_child(hidden_label)
+
+	var result = _helper.call("_game_get_ui_elements", {
+		"root_path": "/Main/%s" % ROOT_NAME,
+		"include_hidden": true,
+		"include_disabled": false,
+		"max_depth": 1,
+	})
+
+	assert_true(result is Dictionary, "get_ui_elements should return a Dictionary")
+	assert_eq(result.total_count, 2)
+	var names := [result.elements[0].name, result.elements[1].name]
+	assert_true(names.has("NameInput"), "enabled control should be included")
+	assert_true(names.has("HiddenButIncluded"), "hidden control should be included when requested")
+	assert_false(names.has("DisabledButton"), "disabled control should be filtered when requested")

test_project/tests/test_game_helper.gd.uid

@@ -0,0 +1 @@
+uid://ttc0dicgnaea

tests/unit/test_runtime_handlers.py

@@ -1408,6 +1408,31 @@ async def test_game_get_node_info_sends_game_command():
     }
 
 
+async def test_game_get_ui_elements_sends_game_command():
+    client = StubClient()
+    runtime = DirectRuntime(registry=SessionRegistry(), client=client)
+
+    await game_handlers.game_get_ui_elements(
+        runtime,
+        root_path="/Main/HUD",
+        include_hidden=True,
+        include_disabled=False,
+        max_depth=3,
+    )
+
+    assert client.calls[-1]["command"] == "game_command"
+    assert client.calls[-1]["params"] == {
+        "op": "get_ui_elements",
+        "params": {
+            "root_path": "/Main/HUD",
+            "include_hidden": True,
+            "include_disabled": False,
+            "max_depth": 3,
+        },
+    }
+    assert client.calls[-1]["timeout"] == game_handlers.GAME_COMMAND_TIMEOUT_SEC
+
+
 async def test_game_input_key_sends_game_command():
     client = StubClient()
     runtime = DirectRuntime(registry=SessionRegistry(), client=client)