Skip to content

Commit b30b8d3

Browse files
teo-nikolovteo-nikolov
authored
Merge pull request #52 from AutomateThePlanet/ai-augmention-updates
Fix AI locator strategy in Playwright module
2 parents b0f9380 + afe03fe commit b30b8d3

File tree

16 files changed

+269
-116
lines changed

16 files changed

+269
-116
lines changed

‎src/Bellatrix.Desktop/llm/FindByPrompt.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -163,11 +163,11 @@ private string TryResolveFromPageObjectMemory(string instruction, WindowsDriver<
163163
private string ResolveViaPromptFallback(string location, WindowsDriver<WindowsElement> driver, int maxAttempts = 3)
164164
{
165165
var viewSnapshotProvider = ServicesCollection.Current.Resolve<IViewSnapshotProvider>();
166-
var summaryJson = viewSnapshotProvider.GetCurrentViewSnapshot();
167166
var failedSelectors = new List<string>();
168167

169168
for (var attempt = 1; attempt <= maxAttempts; attempt++)
170169
{
170+
var summaryJson = viewSnapshotProvider.GetCurrentViewSnapshot();
171171
var prompt = SemanticKernelService.Kernel?.InvokeAsync(nameof(LocatorSkill), nameof(LocatorSkill.BuildLocatorPrompt),
172172
new()
173173
{
@@ -229,4 +229,4 @@ private static bool IsElementPresent(WindowsDriver<WindowsElement> driver, strin
229229
}
230230

231231
public override string ToString() => $"Prompt = {Value}";
232-
}
232+
}

‎src/Bellatrix.Mobile/llm/android/FindByPrompt.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -156,11 +156,11 @@ private string TryResolveFromPageObjectMemory(string instruction, object context
156156
private string ResolveViaPromptFallback(string location, object context, int maxAttempts = 3)
157157
{
158158
var snapshotProvider = ServicesCollection.Current.Resolve<IViewSnapshotProvider>();
159-
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
160159
var failedSelectors = new List<string>();
161160

162161
for (var attempt = 1; attempt <= maxAttempts; attempt++)
163162
{
163+
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
164164
var prompt = SemanticKernelService.Kernel.InvokeAsync(nameof(AndroidLocatorSkill), nameof(AndroidLocatorSkill.BuildLocatorPrompt), new()
165165
{
166166
["viewSummaryJson"] = summaryJson,
@@ -224,4 +224,4 @@ private static bool IsElementPresent(object context, string uiautomator)
224224
}
225225

226226
public override string ToString() => $"Prompt = {Value}";
227-
}
227+
}

‎src/Bellatrix.Mobile/llm/ios/FindByPrompt.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -159,11 +159,11 @@ private string TryResolveFromPageObjectMemory(string instruction, object context
159159
private string ResolveViaPromptFallback(string location, object context, int maxAttempts = 3)
160160
{
161161
var snapshotProvider = ServicesCollection.Current.Resolve<IViewSnapshotProvider>();
162-
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
163162
var failedSelectors = new List<string>();
164163

165164
for (var attempt = 1; attempt <= maxAttempts; attempt++)
166165
{
166+
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
167167
var prompt = SemanticKernelService.Kernel?.InvokeAsync(nameof(IOSLocatorSkill), nameof(IOSLocatorSkill.BuildLocatorPrompt),
168168
new()
169169
{
@@ -227,4 +227,4 @@ private static bool IsElementPresent(object context, string nspredicate)
227227
}
228228

229229
public override string ToString() => $"Prompt = {Value}";
230-
}
230+
}

‎src/Bellatrix.Playwright/components/common/CheckBox.cs‎

Lines changed: 53 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,32 @@ public class CheckBox : Component, IComponentDisabled, IComponentChecked, ICompo
3535
/// <param name="options"></param>
3636
public virtual void Check(LocatorCheckOptions options = default)
3737
{
38-
DefaultCheck(Checking, Checked, options);
38+
var tempOptions = options ?? new LocatorCheckOptions();
39+
tempOptions.Timeout = 1;
40+
41+
Checking?.Invoke(this, new ComponentActionEventArgs(this));
42+
this.ValidateIsPresent();
43+
44+
try
45+
{
46+
DefaultCheck(null, null, tempOptions);
47+
}
48+
catch
49+
{
50+
// Fallback to JsClick, checkbox may be custom element with hidden input
51+
52+
if (!IsChecked)
53+
{
54+
var clickOptions = new LocatorClickOptions
55+
{
56+
Force = true,
57+
Timeout = options?.Timeout,
58+
};
59+
DefaultClick(null, null, clickOptions);
60+
}
61+
}
62+
63+
Checked?.Invoke(this, new ComponentActionEventArgs(this));
3964
}
4065

4166
/// <summary>
@@ -44,7 +69,33 @@ public virtual void Check(LocatorCheckOptions options = default)
4469
/// <param name="options"></param>
4570
public virtual void Uncheck(LocatorUncheckOptions options = default)
4671
{
47-
DefaultUncheck(Unchecking, Unchecked, options);
72+
var tempOptions = options ?? new LocatorUncheckOptions();
73+
tempOptions.Timeout = 1;
74+
75+
Unchecking?.Invoke(this, new ComponentActionEventArgs(this));
76+
this.ValidateIsPresent();
77+
78+
try
79+
{
80+
DefaultUncheck(null, null, tempOptions);
81+
}
82+
catch
83+
{
84+
// Fallback to JsClick, checkbox may be custom element with hidden input
85+
86+
if (IsChecked)
87+
{
88+
var clickOptions = new LocatorClickOptions
89+
{
90+
Force = true,
91+
Timeout = options?.Timeout,
92+
};
93+
DefaultClick(null, null, clickOptions);
94+
}
95+
96+
}
97+
98+
Unchecked?.Invoke(this, new ComponentActionEventArgs(this));
4899
}
49100

50101
public virtual void Hover()

‎src/Bellatrix.Playwright/components/common/RadioButton.cs‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,28 @@ public virtual void Hover()
4747
/// <param name="options"></param>
4848
public virtual void Click(LocatorCheckOptions options = default)
4949
{
50-
DefaultCheck(Clicking, Clicked, options);
50+
var tempOptions = options ?? new LocatorCheckOptions();
51+
tempOptions.Timeout = 1;
52+
53+
Clicking?.Invoke(this, new ComponentActionEventArgs(this));
54+
this.ValidateIsPresent();
55+
56+
try
57+
{
58+
DefaultCheck(null, null, tempOptions);
59+
}
60+
catch
61+
{
62+
// Fallback to JsClick, radio button may be custom element with hidden input
63+
var clickOptions = new LocatorClickOptions
64+
{
65+
Force = true,
66+
Timeout = options?.Timeout,
67+
};
68+
69+
DefaultClick(null, null, clickOptions);
70+
}
71+
72+
Clicked?.Invoke(this, new ComponentActionEventArgs(this));
5173
}
5274
}

‎src/Bellatrix.Playwright/components/core/Component.DefaultActions.cs‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,7 @@ internal void DefaultClick(EventHandler<ComponentActionEventArgs> clicking, Even
5050
if (options.Force != null && (bool)options.Force) PerformJsClick();
5151
else WrappedElement.Click(options);
5252
}
53-
53+
5454
else WrappedElement.Click();
5555

5656
clicked?.Invoke(this, new ComponentActionEventArgs(this));
@@ -73,7 +73,6 @@ internal void DefaultUncheck(EventHandler<ComponentActionEventArgs> unchecking,
7373

7474
WrappedElement.Uncheck(options);
7575

76-
7776
@unchecked?.Invoke(this, new ComponentActionEventArgs(this));
7877
}
7978

‎src/Bellatrix.Playwright/components/core/Component.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ public bool IsPresent
321321
{
322322
try
323323
{
324-
return WrappedElement.ElementHandle(new LocatorElementHandleOptions { Timeout = ConfigurationService.GetSection<WebSettings>().TimeoutSettings.InMilliseconds().ElementToExistTimeout }) != null;
324+
return WrappedElement.IsPresent;
325325
}
326326
catch
327327
{

‎src/Bellatrix.Playwright/components/eventhandlers/RadioButtonEventHandlers.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,4 +33,4 @@ public override void UnsubscribeToAll()
3333
RadioButton.Hovering -= HoveringEventHandler;
3434
RadioButton.Hovered -= HoveredEventHandler;
3535
}
36-
}
36+
}

‎src/Bellatrix.Playwright/llm/FindByPrompt.cs‎

Lines changed: 67 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -17,18 +17,15 @@
1717
using Bellatrix.LLM;
1818
using Bellatrix.LLM.Plugins;
1919
using Bellatrix.Playwright.LLM.Plugins;
20-
using Bellatrix.Playwright.Locators;
21-
using Microsoft.Identity.Client;
2220
using Microsoft.SemanticKernel;
23-
using Pipelines.Sockets.Unofficial.Arenas;
2421
using System.Text.RegularExpressions;
2522
using System.Threading;
2623

2724
namespace Bellatrix.Playwright.LLM;
2825

2926
public class FindByPrompt : FindStrategy
3027
{
31-
private bool _tryResolveFromPages = true;
28+
private readonly bool _tryResolveFromPages;
3229
/// <summary>
3330
/// Initializes a new instance of the <see cref="FindByPrompt"/> class with the specified prompt value.
3431
/// </summary>
@@ -58,51 +55,80 @@ private FindStrategy TryResolveLocator(string location, IViewSnapshotProvider sn
5855
{
5956
if (_tryResolveFromPages)
6057
{
61-
// Try from memory
62-
var match = SemanticKernelService.Memory
63-
.SearchAsync(Value, index: "PageObjects", limit: 1)
64-
.Result.Results.FirstOrDefault();
65-
66-
if (match != null)
58+
var ragLocator = TryResolveFromPageObjectMemory(Value);
59+
if (ragLocator != null && ragLocator.Resolve(WrappedBrowser.CurrentPage).IsPresent)
6760
{
68-
var pageSummary = match.Partitions.FirstOrDefault()?.Text ?? "";
69-
var mappedPrompt = SemanticKernelService.Kernel
70-
.InvokeAsync(nameof(LocatorMapperSkill), nameof(LocatorMapperSkill.MatchPromptToKnownLocator), new()
71-
{
72-
["pageSummary"] = pageSummary,
73-
["instruction"] = Value
74-
}).Result.GetValue<string>();
75-
76-
var result = SemanticKernelService.Kernel.InvokePromptAsync(mappedPrompt).Result;
77-
var rawLocator = result?.GetValue<string>()?.Trim();
78-
var ragLocator = new FindXpathStrategy(rawLocator);
79-
if (ragLocator != null)
80-
{
81-
Logger.LogInformation($"✅ Using RAG-located element '{ragLocator}' For '${Value}'");
82-
return ragLocator;
83-
}
61+
Logger.LogInformation($"✅ Using RAG-located element '{ragLocator}' For '${Value}'");
62+
return ragLocator;
8463
}
8564
}
8665

87-
// Try cache
88-
var cached = LocatorCacheService.TryGetCached(location, Value);
89-
if (!string.IsNullOrEmpty(cached))
66+
// Step 2: Try local persistent cache
67+
Logger.LogInformation("⚠️ RAG-located element not present. Trying cached selectors...");
68+
var cached = LocatorCacheService.TryGetCached(WrappedBrowser.CurrentPage.Url, Value);
69+
70+
var strategy = new FindXpathStrategy(cached);
71+
if (!string.IsNullOrEmpty(cached) && strategy.Resolve(WrappedBrowser.CurrentPage).IsPresent)
9072
{
91-
return new FindXpathStrategy(cached);
73+
Logger.LogInformation("✅ Using cached selector.");
74+
return strategy;
9275
}
9376

94-
// Remove broken and fall back
95-
LocatorCacheService.Remove(location, Value);
77+
// Step 3: Fall back to AI + prompt regeneration
78+
Logger.LogInformation("⚠️ Cached selector failed or not found. Re-querying AI...");
79+
LocatorCacheService.Remove(WrappedBrowser.CurrentPage.Url, Value);
80+
9681
return ResolveViaPromptFallback(location, snapshotProvider);
9782
}
9883

84+
private static FindXpathStrategy TryResolveFromPageObjectMemory(string instruction)
85+
{
86+
var match = SemanticKernelService.Memory
87+
.SearchAsync(instruction, index: "PageObjects", limit: 1)
88+
.Result.Results.FirstOrDefault();
89+
90+
if (match == null) return null;
91+
92+
var pageSummary = match.Partitions.FirstOrDefault()?.Text ?? string.Empty;
93+
var mappedPrompt = SemanticKernelService.Kernel
94+
.InvokeAsync(nameof(LocatorMapperSkill), nameof(LocatorMapperSkill.MatchPromptToKnownLocator),
95+
new()
96+
{
97+
["pageSummary"] = pageSummary,
98+
["instruction"] = instruction
99+
}).Result.GetValue<string>();
100+
101+
var locatorResult = SemanticKernelService.Kernel
102+
.InvokePromptAsync(mappedPrompt).Result.GetValue<string>();
103+
104+
return ParsePromptLocatorToStrategy(locatorResult);
105+
}
106+
107+
private static FindXpathStrategy ParsePromptLocatorToStrategy(string promptResult)
108+
{
109+
if (promptResult == "Unknown")
110+
{
111+
return null;
112+
}
113+
114+
var parts = Regex.Match(promptResult, @"^\s*xpath\s*=\s*(//.+)$", RegexOptions.IgnoreCase);
115+
if (!parts.Success)
116+
{
117+
throw new ArgumentException($"❌ Invalid format. Expected: xpath=//... but received '{promptResult}'");
118+
}
119+
120+
var xpath = parts.Groups[1].Value.Trim();
121+
122+
return new FindXpathStrategy(xpath);
123+
}
124+
99125
private FindStrategy ResolveViaPromptFallback(string location, IViewSnapshotProvider snapshotProvider, int maxAttempts = 3)
100126
{
101-
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
102127
var failedSelectors = new List<string>();
103128

104129
for (var attempt = 1; attempt <= maxAttempts; attempt++)
105130
{
131+
var summaryJson = snapshotProvider.GetCurrentViewSnapshot();
106132
var prompt = SemanticKernelService.Kernel
107133
.InvokeAsync(nameof(LocatorSkill), nameof(LocatorSkill.BuildLocatorPrompt),
108134
new()
@@ -113,25 +139,22 @@ private FindStrategy ResolveViaPromptFallback(string location, IViewSnapshotProv
113139
}).Result.GetValue<string>();
114140

115141
var result = SemanticKernelService.Kernel.InvokePromptAsync(prompt).Result;
116-
var raw = result?.GetValue<string>()?.Trim();
142+
var rawSelector = result?.GetValue<string>()?.Trim();
117143

118-
if (!string.IsNullOrWhiteSpace(raw))
144+
var strategy = new FindXpathStrategy(rawSelector);
145+
if (!string.IsNullOrWhiteSpace(rawSelector) && strategy.Resolve(WrappedBrowser.CurrentPage).IsPresent)
119146
{
120-
var locator = new FindXpathStrategy(raw);
121-
if (locator != null)
122-
{
123-
LocatorCacheService.Update(location, Value, locator.Value);
124-
return locator;
125-
}
126-
127-
failedSelectors.Add(raw);
147+
LocatorCacheService.Update(location, Value, strategy.Value);
148+
return strategy;
128149
}
129150

151+
failedSelectors.Add(rawSelector);
152+
Logger.LogInformation($"[Attempt {attempt}] Selector failed: {rawSelector}");
130153
Thread.Sleep(300);
131154
}
132155

133156
throw new ArgumentException($"❌ No valid locator found for: {Value}");
134157
}
135158

136159
public override string ToString() => $"Prompt = {Value}";
137-
}
160+
}

‎src/Bellatrix.Playwright/plugins/screenshots/ScreenshotEngine.cs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,14 @@ internal static class ScreenshotEngine
2222
public static string TakeScreenshot(ServicesCollection serviceContainer, bool fullPage)
2323
{
2424
var browser = serviceContainer.Resolve<WrappedBrowser>();
25-
return Convert.ToBase64String(browser.CurrentPage.Screenshot(new PageScreenshotOptions { FullPage = fullPage, Type = ScreenshotType.Png }));
25+
if (browser is not null)
26+
{
27+
return Convert.ToBase64String(browser.CurrentPage.Screenshot(new PageScreenshotOptions { FullPage = fullPage, Type = ScreenshotType.Png }));
28+
}
29+
else
30+
{
31+
return string.Empty;
32+
}
2633
}
2734

2835
public static string GetEmbeddedResource(string resourceName, Assembly assembly)
@@ -44,4 +51,4 @@ private static string FormatResourceName(Assembly assembly, string resourceName)
4451
.Replace("\\", ".")
4552
.Replace("/", ".");
4653
}
47-
}
54+
}

0 commit comments

Comments
 (0)
⚡