Merge pull request #1175 from hchen2020/master

Add IWebElementLocator

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/ElementLocatingArgs.cs

@@ -5,6 +5,9 @@ namespace BotSharp.Abstraction.Browsing.Models;
 [DebuggerStepThrough]
 public class ElementLocatingArgs
 {
+    [JsonPropertyName("element_locator_desc")]
+    public string ElementLocatorDescription { get; set; } = string.Empty;
+
     [JsonPropertyName("match_rule")]
     public string MatchRule { get; set; } = string.Empty;
 
@@ -26,6 +29,8 @@ public class ElementLocatingArgs
     [JsonPropertyName("selector")]
     public string? Selector { get; set; }
 
+    public ElementPosition? Position { get; set; }
+
     public bool Parent { get; set; }
 
     public bool FailIfMultiple { get; set; }

src/Infrastructure/BotSharp.Abstraction/Browsing/Settings/IWebElementLocator.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.Browsing.Models;
+
+namespace BotSharp.Abstraction.Browsing.Settings;
+
+public interface IWebElementLocator
+{
+    Task<ElementPosition> DetectElementCoordinates(IWebBrowser browser, string contextId, string elementDescription);
+}

src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj

@@ -16,8 +16,6 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-action_on_element.fn.liquid" />
-    <None Remove="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-go_to_page.fn.liquid" />
     <None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\instructions\instruction.liquid" />
     <None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\templates\extract_data.liquid" />
     <None Remove="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\templates\html_parser.liquid" />
@@ -28,6 +26,9 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-action_on_element.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-take_screenshot.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-close_browser.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
@@ -37,12 +38,6 @@
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-locate_element.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-action_on_element.fn.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\templates\util-web-go_to_page.fn.liquid">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
     <Content Include="data\agents\f3ae2a0f-e6ba-4ee1-a0b9-75d7431ff32b\agent.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.ActionOnElement.cs

@@ -8,9 +8,12 @@ public async Task<BrowserActionResult> ActionOnElement(MessageInfo message, Elem
         var result = await LocateElement(message, location);
         if (result.IsSuccess)
         {
+            action.Position = location.Position;
             await DoAction(message, action, result);
-            result.UrlAfterAction = _instance.GetPage(message.ContextId)?.Url;
         }
+
+        result.UrlAfterAction = _instance.GetPage(message.ContextId)?.Url;
+
         return result;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs

@@ -1,5 +1,6 @@
 using System.IO;
 using System.Net.Http;
+using System.Xml.Linq;
 
 namespace BotSharp.Plugin.WebDriver.Drivers.PlaywrightDriver;
 
@@ -8,33 +9,66 @@ public partial class PlaywrightWebDriver
     public async Task DoAction(MessageInfo message, ElementActionArgs action, BrowserActionResult result)
     {
         var page = _instance.GetPage(message.ContextId);
-        if (string.IsNullOrEmpty(result.Selector))
+        if (string.IsNullOrEmpty(result.Selector) && action.Position == null)
         {
             Serilog.Log.Error($"Selector is not set.");
             return;
         }
 
-        ILocator locator = page.Locator(result.Selector);
-        var count = await locator.CountAsync();
-
-        if (count == 0)
-        {
-            Serilog.Log.Error($"Element not found: {result.Selector}");
-            return;
-        }
-        else if (count > 1)
+        ILocator? locator;
+        
+        if (result.Selector != null)
         {
-            if (!action.FirstIfMultipleFound)
+            locator = page.Locator(result.Selector);
+
+            var count = await locator.CountAsync();
+
+            if (count == 0)
             {
-                Serilog.Log.Error($"Multiple eElements were found: {result.Selector}");
+                Serilog.Log.Error($"Element not found: {result.Selector}");
                 return;
             }
-            else
+            else if (count > 1)
             {
-                locator = page.Locator(result.Selector).First;// 匹配到多个时取第一个，否则当await locator.ClickAsync();匹配到多个就会抛异常。
+                if (!action.FirstIfMultipleFound)
+                {
+                    Serilog.Log.Error($"Multiple eElements were found: {result.Selector}");
+                    return;
+                }
+                else
+                {
+                    locator = page.Locator(result.Selector).First;// 匹配到多个时取第一个，否则当await locator.ClickAsync();匹配到多个就会抛异常。
+                }
             }
+
+            await ExecuteAction(message, page, locator, action);
         }
+        else if (action.Position != null && action.Position.X != 0 && action.Position.Y != 0)
+        {
+            if (action.Position != null && action.Position.X != 0 && action.Position.Y != 0)
+            {
+                var elementHandle = await page.EvaluateHandleAsync(
+                    @"(coords) => document.elementFromPoint(coords.x, coords.y)",
+                    new { x = (int)action.Position.X, y = (int)action.Position.Y }
+                );
 
+                await ExecuteAction(message, page, elementHandle.AsElement(), action);
+            }
+        }
+        else
+        {
+            Serilog.Log.Error($"Selector or position is not set.");
+            return;
+        }
+
+        if (action.WaitTime > 0)
+        {
+            await Task.Delay(1000 * action.WaitTime);
+        }
+    }
+
+    private async Task ExecuteAction(MessageInfo message, IPage page, ILocator locator, ElementActionArgs action)
+    {
         if (action.Action == BroswerActionEnum.Click)
         {
             if (action.Position == null)
@@ -201,12 +235,174 @@ await locator.ClickAsync(new LocatorClickOptions
                 }
             }
         }
+    }
 
-        if (action.WaitTime > 0)
+    private async Task ExecuteAction(MessageInfo message, IPage page, IElementHandle elementHandle, ElementActionArgs action)
+    {
+        var body = page.Locator("body");
+
+        if (action.Action == BroswerActionEnum.Click)
         {
+            await body.ClickAsync(new LocatorClickOptions
+            {
+                Position = new Position
+                {
+                    X = action.Position.X,
+                    Y = action.Position.Y
+                }
+            });
+        }
+        else if (action.Action == BroswerActionEnum.DropDown)
+        {
+            var tagName = await body.EvaluateAsync<string>("el => el.tagName.toLowerCase()");
+            if (tagName == "select")
+            {
+                await HandleSelectDropDownAsync(page, body, action);
+            }
+            else
+            {
+                await body.ClickAsync();
+                if (!string.IsNullOrWhiteSpace(action.PressKey))
+                {
+                    await page.Keyboard.PressAsync(action.PressKey);
+                    await page.Keyboard.PressAsync("Enter");
+                }
+                else
+                {
+                    var optionLocator = page.Locator($"//div[text()='{action.Content}']");
+                    var optionCount = await optionLocator.CountAsync();
+                    if (optionCount == 0)
+                    {
+                        Serilog.Log.Error($"Dropdown option not found: {action.Content}");
+                        return;
+                    }
+                    await optionLocator.First.ClickAsync();
+                }
+            }
+        }
+        else if (action.Action == BroswerActionEnum.InputText)
+        {
+            await elementHandle.FillAsync(action.Content);
+
+            if (action.PressKey != null)
+            {
+                if (action.DelayBeforePressingKey > 0)
+                {
+                    await Task.Delay(action.DelayBeforePressingKey);
+                }
+                await body.PressAsync(action.PressKey);
+            }
+        }
+        else if (action.Action == BroswerActionEnum.FileUpload)
+        {
+            var _states = _services.GetRequiredService<IConversationStateService>();
+            var files = new List<string>();
+            if (action.FileUrl != null && action.FileUrl.Length > 0)
+            {
+                files.AddRange(action.FileUrl);
+            }
+            var hooks = _services.GetServices<IWebDriverHook>();
+            foreach (var hook in hooks)
+            {
+                files.AddRange(await hook.GetUploadFiles(message));
+            }
+            if (files.Count == 0)
+            {
+                Serilog.Log.Warning($"No files found to upload: {action.Content}");
+                return;
+            }
+            var fileChooser = await page.RunAndWaitForFileChooserAsync(async () =>
+            {
+                await body.ClickAsync();
+            });
+            var guid = Guid.NewGuid().ToString();
+            var directory = Path.Combine(Path.GetTempPath(), guid);
+            DeleteDirectory(directory);
+            Directory.CreateDirectory(directory);
+            var localPaths = new List<string>();
+            var http = _services.GetRequiredService<IHttpClientFactory>();
+            using var httpClient = http.CreateClient();
+            foreach (var fileUrl in files)
+            {
+                try
+                {
+                    using var fileData = await httpClient.GetAsync(fileUrl);
+                    var fileName = new Uri(fileUrl).AbsolutePath;
+                    var localPath = Path.Combine(directory, Path.GetFileName(fileName));
+                    await using var fs = new FileStream(localPath, FileMode.Create, FileAccess.Write, FileShare.None);
+                    await fileData.Content.CopyToAsync(fs);
+                    localPaths.Add(localPath);
+                }
+                catch (Exception ex)
+                {
+                    Serilog.Log.Error($"FileUpload failed for {fileUrl}. Message: {ex.Message}");
+                }
+            }
+            await fileChooser.SetFilesAsync(localPaths);
             await Task.Delay(1000 * action.WaitTime);
         }
+        else if (action.Action == BroswerActionEnum.Typing)
+        {
+            await body.PressSequentiallyAsync(action.Content);
+            if (action.PressKey != null)
+            {
+                if (action.DelayBeforePressingKey > 0)
+                {
+                    await Task.Delay(action.DelayBeforePressingKey);
+                }
+                await body.PressAsync(action.PressKey);
+            }
+        }
+        else if (action.Action == BroswerActionEnum.Hover)
+        {
+            await body.HoverAsync();
+        }
+        else if (action.Action == BroswerActionEnum.DragAndDrop)
+        {
+            // Locate the element to drag
+            var box = await body.BoundingBoxAsync();
+
+            if (box != null)
+            {
+                // Calculate start position
+                float startX = box.X + box.Width / 2; // Start at the center of the element
+                float startY = box.Y + box.Height / 2;
+
+                // Drag offsets
+                float offsetX = action.Position.X;
+                // Move horizontally
+                if (action.Position.Y == 0)
+                {
+                    // Perform drag-and-move
+                    // Move mouse to the start position
+                    var mouse = page.Mouse;
+                    await mouse.MoveAsync(startX, startY);
+                    await mouse.DownAsync();
+
+                    // Move mouse smoothly in increments
+                    var tracks = GetVelocityTrack(offsetX);
+                    foreach (var track in tracks)
+                    {
+                        startX += track;
+                        await page.Mouse.MoveAsync(startX, 0, new MouseMoveOptions
+                        {
+                            Steps = 3
+                        });
+                    }
+
+                    // Release mouse button
+                    await Task.Delay(1000);
+                    await mouse.UpAsync();
+                }
+                else
+                {
+                    throw new NotImplementedException();
+                }
+            }
+        }
     }
+
+
     private void DeleteDirectory(string directory)
     {
         if (Directory.Exists(directory))

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.LocateElement.cs

@@ -21,6 +21,23 @@ public async Task<BrowserActionResult> LocateElement(MessageInfo message, Elemen
                 IsSuccess = false
             };
         }
+
+        // Use IWebElementLocator to detect element position by element description
+        var locators = _services.GetServices<IWebElementLocator>();
+        foreach (var el in locators)
+        {
+            location.Position = await el.DetectElementCoordinates(this, message.ContextId, location.ElementLocatorDescription);
+
+            if (location.Position != null && location.Position.X > 0 && location.Position.Y > 0)
+            {
+                result.Message = $"Position based locating is found at {location.Position}";
+                return new BrowserActionResult
+                {
+                    IsSuccess = true
+                };
+            }
+        }
+
         ILocator locator = page.Locator("body");
         int count = 0;
         var keyword = string.Empty;

src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebUtilityHook.cs

@@ -7,6 +7,7 @@ public class WebUtilityHook : IAgentUtilityHook
     private const string GO_TO_PAGE_FN = $"{PREFIX}go_to_page";
     private const string LOCATE_ELEMENT_FN = $"{PREFIX}locate_element";
     private const string ACTION_ON_ELEMENT_FN = $"{PREFIX}action_on_element";
+    private const string TAKE_SCREENSHOT_FN = $"{PREFIX}take_screenshot";
 
     public void AddUtilities(List<AgentUtility> utilities)
     {
@@ -20,12 +21,10 @@ public void AddUtilities(List<AgentUtility> utilities)
                     new UtilityItem
                     {
                         FunctionName = GO_TO_PAGE_FN,
-                        TemplateName = $"{GO_TO_PAGE_FN}.fn"
                     },
                     new UtilityItem
                     {
                         FunctionName = ACTION_ON_ELEMENT_FN,
-                        TemplateName = $"{ACTION_ON_ELEMENT_FN}.fn"
                     },
                     new UtilityItem
                     {
@@ -34,6 +33,10 @@ public void AddUtilities(List<AgentUtility> utilities)
                     new UtilityItem
                     {
                         FunctionName = CLOSE_BROWSER_FN
+                    },
+                    new UtilityItem
+                    {
+                        FunctionName = TAKE_SCREENSHOT_FN
                     }
                 ]
             }