add examples for LLM with desktop

improve desktop snapshot summary
fix self-healing
fixes in find by prompt logic

Author: angelovstanton

src/Bellatrix.Desktop/components/Core/Component.cs

@@ -247,7 +247,7 @@ protected WindowsElement GetAndWaitWebDriverElement()
                 if (_llmSettings.EnableSelfHealing)
                 {
                     var snapshot = _viewSnapshotProvider.GetCurrentViewSnapshot();
-                    LocatorSelfHealingService.SaveWorkingLocator(By.ToString(), snapshot, WrappedDriver.CurrentWindowHandle);
+                    LocatorSelfHealingService.SaveWorkingLocator(By.ToString(), snapshot, WrappedDriver.Title);
                 }
 
                 _untils.Clear();
@@ -263,7 +263,7 @@ protected WindowsElement GetAndWaitWebDriverElement()
                 Logger.LogWarning($"⚠️ Element not found with locator: {By}. Trying AI-based healing...");
 
                 var snapshot = _viewSnapshotProvider.GetCurrentViewSnapshot();
-                var healedXpath = LocatorSelfHealingService.TryHeal(By.ToString(), snapshot, WrappedDriver.CurrentWindowHandle);
+                var healedXpath = LocatorSelfHealingService.TryHeal(By.ToString(), snapshot, WrappedDriver.Title);
 
                 if (!string.IsNullOrEmpty(healedXpath))
                 {

src/Bellatrix.Desktop/llm/FindByPrompt.cs

@@ -171,7 +171,7 @@ private string ResolveViaPromptFallback(string location, WindowsDriver<WindowsEl
             var prompt = SemanticKernelService.Kernel?.InvokeAsync(nameof(LocatorSkill), nameof(LocatorSkill.BuildLocatorPrompt),
                 new()
                 {
-                    ["htmlSummary"] = summaryJson,
+                    ["viewSummaryJson"] = summaryJson,
                     ["instruction"] = Value,
                     ["failedSelectors"] = failedSelectors
                 }).Result.GetValue<string>();

src/Bellatrix.Desktop/llm/extensions/DesktopElementSummary.cs

@@ -25,5 +25,20 @@ public class DesktopElementSummary
     public string ControlType { get; set; }
     public string Value { get; set; }
     public string HelpText { get; set; }
-    public bool Enabled { get; set; }
+    public bool? IsSelected { get; set; }
+    public string ToggleState { get; set; }
+    public string ExpandCollapseState { get; set; }
+    public string Selection { get; set; }
+    public bool? IsEnabled { get; set; }
+    public bool? IsOffscreen { get; set; }
+    public bool? IsPassword { get; set; }
+    public bool? IsAvailable { get; set; }
+    public bool? HasKeyboardFocus { get; set; }
+    public bool? IsKeyboardFocusable { get; set; }
+    public string Orientation { get; set; }
+    public string Minimum { get; set; }
+    public string Maximum { get; set; }
+    public string LargeChange { get; set; }
+    public string SmallChange { get; set; }
+    // Add more as needed from the XML source
 }

src/Bellatrix.Desktop/llm/skills/LocatorSkill.cs

@@ -51,52 +51,44 @@ public string BuildLocatorPrompt(string viewSummaryJson, string instruction, Lis
 
 ---
 
-**WinAppDriver XPath Guidelines (strict compatibility):**
+**STRICT WinAppDriver XPath Rules:**
 
-✅ Use XPath expressions with **direct attribute matches only**:
-- `@AutomationId`
-- `@Name`
-- `@ClassName`
-- `@ControlType`
-- `@HelpText`
-- `@Value.Value` (optional, if available)
-
-✅ Examples of valid XPath:
+✅ Use only **PascalCase** element tags (e.g., `ComboBox`, `Edit`, `Button`, `Text`, `Pane`, etc.)  
+✅ Use only **PascalCase** attribute names (`AutomationId`, `Name`, `ClassName`, `ControlType`, `HelpText`, `Value.Value`).
+✅ Use only direct, case-sensitive attribute matches (no contains, no normalize-space, etc).
+✅ Example valid XPath:
+- `//ComboBox[@AutomationId='select']`
 - `//Edit[@Name='Username']`
 - `//Button[@AutomationId='SubmitBtn']`
 - `//Text[@HelpText='Tooltip message']`
 
-✅ Format rules:
-- Use lowercase tag names (e.g., `edit`, `button`, `text`)
-- Attribute values must be wrapped in single quotes: `@Name='Login'`
-- Return the shortest valid XPath with a single attribute condition
+🚫 NEVER use:
+- Lowercase tag or attribute names (e.g., `//combobox[@automationid='select']` is INVALID)
+- Any case conversion or function (translate, lower-case, upper-case, etc)
+- contains(), normalize-space(), substring(), axes, positions, or multiple conditions.
 
-🚫 Do NOT use:
-- `contains(...)`
-- `normalize-space(...)`
-- `substring(...)`
-- XPath axes like `ancestor::`, `following::`, `preceding-sibling::`
-- Position-based XPath (e.g., `(//Edit)[2]`)
-- Multiple conditions (e.g., `[@Name='X' and @AutomationId='Y']`)
+**You MUST match tag and attribute names in PascalCase exactly as in the WinAppDriver XML.**
 
 ---
 
 **Return Format:**
-Only return a single valid XPath string like:
+Return only a single valid, case-sensitive XPath string using PascalCase for both tag and attribute, e.g.:
+- //ComboBox[@AutomationId='select']
 - //Edit[@Name='Username']
-- //Button[@AutomationId='LoginBtn']
-- //Pane[@ClassName='MainPanel']
 
-Do not include:
+🚫 Do NOT include:
 - Explanations
 - Multiple lines
 - Comments
 - Markdown formatting
+- Code blocks (no triple backticks or ``` around the XPath)
+- Extra whitespace or newlines before or after the XPath
 
-Only return the XPath string.
+✅ Return ONLY the XPath string as a single line, nothing else.
 """;
     }
 
+
     [KernelFunction]
     public string HealBrokenLocator(string failedLocator, string oldSnapshot, string newSnapshot)
     {
@@ -121,23 +113,42 @@ public string HealBrokenLocator(string failedLocator, string oldSnapshot, string
 
 ---
 
-✅ XPath must match one of the following formats:
+**STRICT WinAppDriver XPath Rules:**
+
+✅ Use only **PascalCase** element tags (e.g., `ComboBox`, `Edit`, `Button`, `Text`, `Pane`, etc.)
+✅ Use only **PascalCase** attribute names (`AutomationId`, `Name`, `ClassName`, `ControlType`, `HelpText`, `Value.Value`)
+✅ Use only direct, case-sensitive attribute matches—NO functions, partial, or case-insensitive matching
+
+✅ Example valid XPath:
+- //ComboBox[@AutomationId='select']
 - //Edit[@Name='Username']
-- //Button[@AutomationId='Submit']
-- //Text[@HelpText='Tooltip']
+- //Button[@AutomationId='SubmitBtn']
+- //Text[@HelpText='Tooltip message']
+
+🚫 NEVER use:
+- Lowercase tag or attribute names (e.g., `//combobox[@automationid='select']` is INVALID)
+- Any case conversion or function (`translate`, `lower-case`, `upper-case`)
+- contains(), normalize-space(), substring(), axes, positions, or multiple conditions
 
-🚫 Do NOT use:
-- contains()
-- normalize-space()
-- substring()
-- any complex or relative XPath
+**You MUST match tag and attribute names in PascalCase exactly as in the WinAppDriver XML.**
 
 ---
 
 **Return Format:**
-Return only a single valid XPath expression, as a one-line string.
+Only return a single valid, case-sensitive XPath string using PascalCase for both tag and attribute, e.g.:
+- //ComboBox[@AutomationId='select']
+- //Edit[@Name='Username']
 
-Return NOTHING else.
+🚫 Do NOT include:
+- Explanations
+- Multiple lines
+- Comments
+- Markdown formatting
+- Code blocks (no triple backticks or ``` around the XPath)
+- Extra whitespace or newlines before or after the XPath
+
+✅ Return ONLY the XPath string as a single line, nothing else.
 """;
     }
+
 }

src/Bellatrix.Desktop/services/AppService.cs

@@ -56,19 +56,43 @@ public string GetCurrentViewSnapshot()
         }
 
         List<DesktopElementSummary> elements = nodes
-            .Select(node => new DesktopElementSummary
-            {
-                Tag = node.Name,
-                AutomationId = node.GetAttributeValue("AutomationId", null),
-                Name = node.GetAttributeValue("Name", null),
-                ClassName = node.GetAttributeValue("ClassName", null),
-                ControlType = node.GetAttributeValue("ControlType", null),
-                Value = node.GetAttributeValue("Value.Value", null),
-                HelpText = node.GetAttributeValue("HelpText", null)
-            })
-            .Where(x => !string.IsNullOrWhiteSpace(x.Name) || !string.IsNullOrWhiteSpace(x.AutomationId))
-            .ToList();
+      .Select(node => new DesktopElementSummary
+      {
+          Tag = node.Name,
+          AutomationId = node.GetAttributeValue("AutomationId", null),
+          Name = node.GetAttributeValue("Name", null),
+          ClassName = node.GetAttributeValue("ClassName", null),
+          ControlType = node.GetAttributeValue("ControlType", null),
+          Value = node.GetAttributeValue("Value.Value", null) ?? node.GetAttributeValue("Value", null),
+          HelpText = node.GetAttributeValue("HelpText", null),
+          IsSelected = ParseNullableBool(node.GetAttributeValue("IsSelected", null)),
+          ToggleState = node.GetAttributeValue("ToggleState", null),
+          ExpandCollapseState = node.GetAttributeValue("ExpandCollapseState", null),
+          Selection = node.GetAttributeValue("Selection", null),
+          IsEnabled = ParseNullableBool(node.GetAttributeValue("IsEnabled", null)),
+          IsOffscreen = ParseNullableBool(node.GetAttributeValue("IsOffscreen", null)),
+          IsPassword = ParseNullableBool(node.GetAttributeValue("IsPassword", null)),
+          IsAvailable = ParseNullableBool(node.GetAttributeValue("IsAvailable", null)),
+          HasKeyboardFocus = ParseNullableBool(node.GetAttributeValue("HasKeyboardFocus", null)),
+          IsKeyboardFocusable = ParseNullableBool(node.GetAttributeValue("IsKeyboardFocusable", null)),
+          Orientation = node.GetAttributeValue("Orientation", null),
+          Minimum = node.GetAttributeValue("Minimum", null),
+          Maximum = node.GetAttributeValue("Maximum", null),
+          LargeChange = node.GetAttributeValue("LargeChange", null),
+          SmallChange = node.GetAttributeValue("SmallChange", null)
+      })
+      .Where(x => !string.IsNullOrWhiteSpace(x.Name) || !string.IsNullOrWhiteSpace(x.AutomationId))
+      .ToList();
 
         return JsonConvert.SerializeObject(elements, Formatting.None);
     }
+
+    private static bool? ParseNullableBool(string value)
+    {
+        if (value == null)
+            return null;
+        if (bool.TryParse(value, out var result))
+            return result;
+        return null;
+    }
 }

templates/Bellatrix.Desktop.GettingStarted/32. Prompts Support/MainPage/MainDesktopPage.Actions.cs

@@ -0,0 +1,25 @@
+using Bellatrix.Desktop.PageObjects;
+
+namespace Bellatrix.Desktop.GettingStarted.LLM;
+
+// 1. All BELLATRIX page objects are implemented as partial classes which means that you have separate files for different parts of it- actions, elements, assertions
+// but at the end, they are all built into a single type. This makes the maintainability and readability of these classes much better. Also, you can easier locate what you need.
+//
+// You can always create BELLATRIX page objects yourself inheriting DesktopPage class
+// We advise you to follow the convention with partial classes, but you are always free to put all pieces in a single file.
+public partial class MainDesktopPage : DesktopPage
+{
+    // 2. These elements are always used together when an item is transferred. There are many test cases where you need to transfer different items and so on.
+    // This way you reuse the code instead of copy-paste it. If there is a change in the way how the item is transferred, change the workflow only here.
+    // Even single line of code is changed in your tests.
+    public void TransferItem(string itemToBeTransfered, string userName, string password)
+    {
+        PermanentTransfer.Check();
+        Items.SelectByText(itemToBeTransfered);
+        ReturnItemAfter.ToNotExists().WaitToBe();
+        UserName.SetText(userName);
+        Password.SetPassword(password);
+        KeepMeLogged.Click();
+        Transfer.Click();
+    }
+}

templates/Bellatrix.Desktop.GettingStarted/32. Prompts Support/MainPage/MainDesktopPage.Assertions.cs

@@ -0,0 +1,24 @@
+namespace Bellatrix.Desktop.GettingStarted.LLM;
+
+public partial class MainDesktopPage
+{
+    public void AssertPermanentTransferIsChecked()
+    {
+        App.Assert.IsTrue(PermanentTransfer.IsChecked);
+    }
+
+    public void AssertRightItemSelected(string itemName)
+    {
+        App.Assert.AreEqual(itemName, Items.InnerText);
+    }
+
+    public void AssertRightUserNameSet(string userName)
+    {
+        App.Assert.AreEqual(userName, UserName.InnerText);
+    }
+
+    public void AssertKeepMeLoggedChecked()
+    {
+        App.Assert.IsTrue(KeepMeLogged.IsChecked);
+    }
+}

templates/Bellatrix.Desktop.GettingStarted/32. Prompts Support/MainPage/MainDesktopPage.Map.cs

@@ -0,0 +1,17 @@
+namespace Bellatrix.Desktop.GettingStarted.LLM;
+
+public partial class MainDesktopPage
+{
+    // 1. All elements are placed inside the file PageName.Map so that the declarations of your elements to be in a single place.
+    // It is convenient since if there is a change in some of the locators or elements types you can apply the fix only here.
+    // All elements are implements as properties. Here we use the short syntax for declaring properties, but you can always use the old one.
+    // App.Components property is actually a shorter version of ComponentCreateService
+    public Button Transfer => App.Components.CreateByName<Button>("E Button");
+    public CheckBox PermanentTransfer => App.Components.CreateByName<CheckBox>("BellaCheckBox");
+    public ComboBox Items => App.Components.CreateByAutomationId<ComboBox>("select");
+    public Button ReturnItemAfter => App.Components.CreateByName<Button>("DisappearAfterButton1");
+    public Label Results => App.Components.CreateByAutomationId<Label>("ResultLabelId");
+    public Password Password => App.Components.CreateByAutomationId<Password>("passwordBox");
+    public TextField UserName => App.Components.CreateByAutomationId<TextField>("textBox");
+    public RadioButton KeepMeLogged => App.Components.CreateByName<RadioButton>("RadioButton");
+}

templates/Bellatrix.Desktop.GettingStarted/32. Prompts Support/PageObjectsTests.cs

@@ -0,0 +1,77 @@
+using Bellatrix.Desktop.LLM.Extensions;
+using Bellatrix.Desktop.NUnit;
+using NUnit.Framework;
+using static Bellatrix.AiValidator;
+using static Bellatrix.AiAssert;
+
+namespace Bellatrix.Desktop.GettingStarted.LLM;
+
+[TestFixture]
+public class PageObjectsTests : DesktopTest
+{
+    [Test]
+    public void ActionsWithoutPageObjects_Wpf_LLM()
+    {
+        var permanentTransfer = App.Components.CreateByName<CheckBox>("BellaCheckBox");
+
+        permanentTransfer.Check();
+
+        App.Assert.IsTrue(permanentTransfer.IsChecked);
+
+        //var items = App.Components.CreateByAutomationId<ComboBox>("select");
+        var items = App.Components.CreateByPrompt<ComboBox>("find the select under meissa check box");
+
+
+        items.SelectByText("Item1");
+        ValidateByPrompt("verify select under meissa check box displayed");
+
+        var returnItemAfter = App.Components.CreateByName<Component>("DisappearAfterButton1");
+
+        returnItemAfter.ToNotExists().WaitToBe();
+
+        var password = App.Components.CreateByAutomationId<Password>("passwordBox");
+        //var password = App.Components.CreateByAutomationId<Password>("passwordBoxUpdated");
+
+        password.SetPassword("topsecret");
+
+        var userName = App.Components.CreateByAutomationId<TextField>("textBox");
+
+        userName.SetText("bellatrix");
+
+        App.Assert.AreEqual("bellatrix", userName.InnerText);
+
+        var keepMeLogged = App.Components.CreateByName<RadioButton>("RadioButton");
+
+        keepMeLogged.Click();
+
+        App.Assert.IsTrue(keepMeLogged.IsChecked);
+
+        var byName = App.Components.CreateByName<Button>("E Button");
+
+        byName.Click();
+
+        //var label = App.Components.CreateByAutomationId<Label>("ResultLabelId");
+        var label = App.Components.CreateByPrompt<Label>("find the label above the calendar");
+
+        ValidateByPrompt("validate that the label says that the button was clicked");
+        //App.Assert.IsTrue(label.IsPresent);
+    }
+
+    [Test]
+    [Category(Categories.CI)]
+    public void ActionsWithPageObjects_Wpf()
+    {
+        // 5. You can use the App Create method to get an instance of it.
+        var mainPage = App.Create<MainDesktopPage>();
+
+        // 6. After you have the instance, you can directly start using the action methods of the page.
+        // As you can see the test became much shorter and more readable.
+        // The additional code pays off in future when changes are made to the page, or you need to reuse some of the methods.
+        mainPage.TransferItem("Item2", "bellatrix", "topsecret");
+
+        mainPage.AssertKeepMeLoggedChecked();
+        mainPage.AssertPermanentTransferIsChecked();
+        mainPage.AssertRightItemSelected("Item2");
+        mainPage.AssertRightUserNameSet("bellatrix");
+    }
+}