Add runtime UI element inspection

Author: Joshua

docs/TOOLS.md

@@ -65,7 +65,7 @@ Calls take the form:
 | `camera_manage` | `create`, `configure`, `set_limits_2d`, `set_damping_2d`, `follow_2d`, `get`, `list`, `apply_preset` |
 | `signal_manage` | `list`, `connect`, `disconnect` |
 | `input_map_manage` | `list`, `add_action`, `remove_action`, `bind_event` |
-| `game_manage` | `get_scene_tree`, `get_node_info`, `input_key`, `input_mouse`, `input_gamepad`, `input_state` |
+| `game_manage` | `get_scene_tree`, `get_node_info`, `get_ui_elements`, `input_key`, `input_mouse`, `input_gamepad`, `input_state` |
 | `autoload_manage` | `list`, `add`, `remove` |
 | `filesystem_manage` | `read_text`, `write_text`, `reimport`, `search` |
 | `theme_manage` | `create`, `set_color`, `set_constant`, `set_font_size`, `set_stylebox_flat`, `apply` |

plugin/addons/godot_ai/runtime/game_helper.gd

@@ -197,6 +197,8 @@ func _handle_game_command(data: Array) -> void:
 			result = _game_get_scene_tree(json.data)
 		"get_node_info":
 			result = _game_get_node_info(json.data)
+		"get_ui_elements":
+			result = _game_get_ui_elements(json.data)
 		"input_key":
 			result = _game_input_key(json.data)
 		"input_mouse":
@@ -267,6 +269,110 @@ func _game_get_node_info(params: Dictionary) -> Dictionary:
 	return info
 
 
+func _game_get_ui_elements(params: Dictionary) -> Dictionary:
+	var max_depth := maxi(0, int(params.get("max_depth", 10)))
+	var include_hidden := bool(params.get("include_hidden", false))
+	var include_disabled := bool(params.get("include_disabled", true))
+	var root_path := str(params.get("root_path", ""))
+	var root := _resolve_runtime_node(root_path)
+	if root == null:
+		return {"root": "", "elements": [], "total_count": 0, "not_found": root_path}
+
+	var elements: Array[Dictionary] = []
+	_collect_ui_elements(root, 0, max_depth, include_hidden, include_disabled, elements)
+	return {
+		"root": _runtime_path(root),
+		"elements": elements,
+		"total_count": elements.size(),
+	}
+
+
+func _collect_ui_elements(
+	node: Node,
+	current_depth: int,
+	max_depth: int,
+	include_hidden: bool,
+	include_disabled: bool,
+	out: Array[Dictionary]
+) -> void:
+	if node is Control:
+		var control := node as Control
+		var visible := _control_visible_in_tree(control)
+		var disabled := _control_disabled(control)
+		if (include_hidden or visible) and (include_disabled or not disabled):
+			out.append(_ui_element_info(control, visible, disabled))
+
+	if current_depth >= max_depth:
+		return
+	for child in node.get_children():
+		if child is Node:
+			_collect_ui_elements(
+				child,
+				current_depth + 1,
+				max_depth,
+				include_hidden,
+				include_disabled,
+				out
+			)
+
+
+func _ui_element_info(control: Control, visible: bool, disabled: bool) -> Dictionary:
+	var info := {
+		"path": _runtime_path(control),
+		"name": control.name,
+		"type": control.get_class(),
+		"visible": visible,
+		"disabled": disabled,
+		"rect": _variant_to_json(control.get_rect()),
+		"global_rect": _variant_to_json(control.get_global_rect()),
+	}
+	if _object_has_property(control, "text"):
+		info["text"] = str(control.get("text"))
+	return info
+
+
+func _control_disabled(control: Control) -> bool:
+	if _object_has_property(control, "disabled"):
+		return bool(control.get("disabled"))
+	return false
+
+
+func _control_visible_in_tree(control: Control) -> bool:
+	if not control.visible:
+		return false
+	var parent := control.get_parent()
+	while parent != null:
+		if parent is CanvasItem and not (parent as CanvasItem).visible:
+			return false
+		parent = parent.get_parent()
+	if Engine.is_editor_hint():
+		return true
+	return control.is_visible_in_tree()
+
+
+static var _property_name_cache: Dictionary = {}
+
+
+func _object_has_property(obj: Object, property_name: String) -> bool:
+	var key := _property_cache_key(obj)
+	if not _property_name_cache.has(key):
+		var names := {}
+		for prop in obj.get_property_list():
+			names[str(prop.get("name", ""))] = true
+		_property_name_cache[key] = names
+	return (_property_name_cache[key] as Dictionary).has(property_name)
+
+
+func _property_cache_key(obj: Object) -> String:
+	var script = obj.get_script()
+	if script == null:
+		return obj.get_class()
+	var script_id := str(script.get_instance_id())
+	if not script.resource_path.is_empty():
+		script_id = script.resource_path
+	return "%s:%s" % [obj.get_class(), script_id]
+
+
 func _runtime_node_properties(node: Node) -> Dictionary:
 	var props := {}
 	for p in node.get_property_list():
@@ -279,7 +385,7 @@ func _runtime_node_properties(node: Node) -> Dictionary:
 
 
 func _resolve_runtime_node(path: String) -> Node:
-	var scene_root := get_tree().current_scene
+	var scene_root := _current_scene_root()
 	if scene_root == null:
 		return null
 	if path.is_empty() or path == "/":
@@ -298,14 +404,24 @@ func _resolve_runtime_node(path: String) -> Node:
 
 
 func _runtime_path(node: Node) -> String:
-	var scene_root := get_tree().current_scene
+	var scene_root := _current_scene_root()
 	if scene_root == null:
 		return str(node.get_path())
 	if node == scene_root:
 		return "/" + str(scene_root.name)
 	return "/" + str(scene_root.name) + "/" + str(scene_root.get_path_to(node))
 
 
+func _current_scene_root() -> Node:
+	var tree := get_tree()
+	if tree == null:
+		return null
+	var scene_root := tree.current_scene
+	if scene_root == null and Engine.is_editor_hint():
+		scene_root = EditorInterface.get_edited_scene_root()
+	return scene_root
+
+
 func _game_input_key(params: Dictionary) -> Dictionary:
 	var key_name := str(params.get("key", ""))
 	var keycode := OS.find_keycode_from_string(key_name)

src/godot_ai/handlers/game.py

@@ -40,6 +40,23 @@ async def game_get_node_info(
     )
 
 
+async def game_get_ui_elements(
+    runtime: DirectRuntime,
+    root_path: str = "",
+    include_hidden: bool = False,
+    include_disabled: bool = True,
+    max_depth: int = 10,
+) -> dict:
+    params: dict[str, Any] = {
+        "include_hidden": include_hidden,
+        "include_disabled": include_disabled,
+        "max_depth": max_depth,
+    }
+    if root_path:
+        params["root_path"] = root_path
+    return await _game_command(runtime, "get_ui_elements", params)
+
+
 async def game_input_key(
     runtime: DirectRuntime,
     key: str,

src/godot_ai/server.py

@@ -195,8 +195,8 @@ def _emit_startup() -> None:
             "                   follow_2d, get, list, apply_preset\n"
             "  signal_manage    list, connect, disconnect\n"
             "  input_map_manage list, add_action, remove_action, bind_event\n"
-            "  game_manage      get_scene_tree, get_node_info, input_key,\n"
-            "                   input_mouse, input_gamepad, input_state\n"
+            "  game_manage      get_scene_tree, get_node_info, get_ui_elements,\n"
+            "                   input_key, input_mouse, input_gamepad, input_state\n"
             "  autoload_manage  list, add, remove\n"
             "  filesystem_manage read_text, write_text, reimport, search\n"
             "  theme_manage     create, set_color, set_constant, set_font_size,\n"

src/godot_ai/tools/game.py

@@ -20,6 +20,10 @@
         path or a scene-relative path rooted at the current scene.
   - get_node_info(path, include_properties=True)
         Inspect one running node's metadata and optional property snapshot.
+  - get_ui_elements(root_path="", include_hidden=False,
+                    include_disabled=True, max_depth=10)
+        Inspect visible runtime Control nodes for UI testing. Includes path,
+        type, text where present, disabled state, and rect metadata.
   - input_key(key, pressed=True, echo=False)
         Send a key press/release to the running game.
   - input_mouse(event, position=None, button="left", pressed=True)
@@ -38,6 +42,7 @@ def register_game_tools(mcp: FastMCP) -> None:
         ops={
             "get_scene_tree": game_handlers.game_get_scene_tree,
             "get_node_info": game_handlers.game_get_node_info,
+            "get_ui_elements": game_handlers.game_get_ui_elements,
             "input_key": game_handlers.game_input_key,
             "input_mouse": game_handlers.game_input_mouse,
             "input_gamepad": game_handlers.game_input_gamepad,
@@ -46,6 +51,7 @@ def register_game_tools(mcp: FastMCP) -> None:
         read_resource_forms={
             "get_scene_tree": None,
             "get_node_info": None,
+            "get_ui_elements": None,
             "input_key": None,
             "input_mouse": None,
             "input_gamepad": None,

test_project/tests/test_game_helper.gd

@@ -0,0 +1,171 @@
+@tool
+extends McpTestSuite
+
+const GameHelper := preload("res://addons/godot_ai/runtime/game_helper.gd")
+
+const ROOT_NAME := "McpUiElementsRoot"
+
+
+class PropertyProbe:
+	extends Object
+
+	static var property_list_calls := 0
+
+	func _get_property_list() -> Array[Dictionary]:
+		property_list_calls += 1
+		return [{
+			"name": "probe_value",
+			"type": TYPE_STRING,
+			"usage": PROPERTY_USAGE_DEFAULT,
+		}]
+
+
+class AlphaProbe:
+	extends Object
+
+	func _get_property_list() -> Array[Dictionary]:
+		return [{
+			"name": "alpha_value",
+			"type": TYPE_STRING,
+			"usage": PROPERTY_USAGE_DEFAULT,
+		}]
+
+
+class BetaProbe:
+	extends Object
+
+	func _get_property_list() -> Array[Dictionary]:
+		return [{
+			"name": "beta_value",
+			"type": TYPE_STRING,
+			"usage": PROPERTY_USAGE_DEFAULT,
+		}]
+
+
+var _helper: Node
+var _root: Node
+
+
+func suite_name() -> String:
+	return "game_helper"
+
+
+func suite_setup(_ctx: Dictionary) -> void:
+	var scene_root := EditorInterface.get_edited_scene_root()
+	if scene_root == null:
+		fail_setup("current_scene required")
+		return
+	_helper = GameHelper.new()
+	scene_root.add_child(_helper)
+	_root = CanvasLayer.new()
+	_root.name = ROOT_NAME
+	scene_root.add_child(_root)
+
+
+func suite_teardown() -> void:
+	if _root != null:
+		_root.queue_free()
+		_root = null
+	if _helper != null:
+		_helper.queue_free()
+		_helper = null
+
+
+func setup() -> void:
+	if _root == null:
+		return
+	for child in _root.get_children():
+		_root.remove_child(child)
+		child.free()
+
+
+func test_object_has_property_caches_property_lists_by_script() -> void:
+	_helper._property_name_cache.clear()
+	PropertyProbe.property_list_calls = 0
+	var probe := PropertyProbe.new()
+
+	assert_true(_helper.call("_object_has_property", probe, "probe_value"))
+	assert_true(_helper.call("_object_has_property", probe, "probe_value"))
+	assert_eq(PropertyProbe.property_list_calls, 1)
+
+	assert_true(_helper.call("_object_has_property", AlphaProbe.new(), "alpha_value"))
+	assert_false(_helper.call("_object_has_property", AlphaProbe.new(), "beta_value"))
+	assert_true(_helper.call("_object_has_property", BetaProbe.new(), "beta_value"))
+	assert_false(_helper.call("_object_has_property", BetaProbe.new(), "alpha_value"))
+
+
+func test_get_ui_elements_returns_controls_with_text_and_rects() -> void:
+	assert_true(_helper.has_method("_game_get_ui_elements"),
+		"game helper should expose get_ui_elements")
+	var container := Node.new()
+	container.name = "Container"
+	_root.add_child(container)
+
+	var title := Label.new()
+	title.name = "Title"
+	title.text = "Score: 10"
+	title.position = Vector2(10, 20)
+	title.size = Vector2(120, 30)
+	container.add_child(title)
+
+	var button := Button.new()
+	button.name = "StartButton"
+	button.text = "Start"
+	button.disabled = true
+	button.position = Vector2(20, 60)
+	button.size = Vector2(90, 40)
+	container.add_child(button)
+
+	var result = _helper.call("_game_get_ui_elements", {
+		"root_path": "/Main/%s" % ROOT_NAME,
+		"include_hidden": true,
+		"max_depth": 4,
+	})
+
+	assert_true(result is Dictionary, "get_ui_elements should return a Dictionary")
+	assert_eq(result.root, "/Main/%s" % ROOT_NAME)
+	assert_eq(result.total_count, 2)
+	assert_eq(result.elements[0].name, "Title")
+	assert_eq(result.elements[0].type, "Label")
+	assert_eq(result.elements[0].text, "Score: 10")
+	assert_has_key(result.elements[0], "visible")
+	assert_eq(result.elements[0].disabled, false)
+	assert_eq(result.elements[0].rect.position.x, 10.0)
+	assert_eq(result.elements[0].rect.size.y, 30.0)
+	assert_eq(result.elements[1].name, "StartButton")
+	assert_eq(result.elements[1].disabled, true)
+	assert_eq(result.elements[1].text, "Start")
+
+
+func test_get_ui_elements_can_filter_disabled_and_include_hidden() -> void:
+	assert_true(_helper.has_method("_game_get_ui_elements"),
+		"game helper should expose get_ui_elements")
+	var visible_enabled := LineEdit.new()
+	visible_enabled.name = "NameInput"
+	visible_enabled.text = "Ada"
+	_root.add_child(visible_enabled)
+
+	var disabled_button := Button.new()
+	disabled_button.name = "DisabledButton"
+	disabled_button.disabled = true
+	_root.add_child(disabled_button)
+
+	var hidden_label := Label.new()
+	hidden_label.name = "HiddenButIncluded"
+	hidden_label.text = "Hidden"
+	hidden_label.visible = false
+	_root.add_child(hidden_label)
+
+	var result = _helper.call("_game_get_ui_elements", {
+		"root_path": "/Main/%s" % ROOT_NAME,
+		"include_hidden": true,
+		"include_disabled": false,
+		"max_depth": 1,
+	})
+
+	assert_true(result is Dictionary, "get_ui_elements should return a Dictionary")
+	assert_eq(result.total_count, 2)
+	var names := [result.elements[0].name, result.elements[1].name]
+	assert_true(names.has("NameInput"), "enabled control should be included")
+	assert_true(names.has("HiddenButIncluded"), "hidden control should be included when requested")
+	assert_false(names.has("DisabledButton"), "disabled control should be filtered when requested")

test_project/tests/test_game_helper.gd.uid

@@ -0,0 +1 @@
+uid://ttc0dicgnaea