support input password

src/Plugins/BotSharp.Plugin.WebDriver/BotSharp.Plugin.WebDriver.csproj

@@ -38,6 +38,9 @@
   </ItemGroup>
 
   <ItemGroup>
+    <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-locate_element.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
     <Content Include="data\agents\6745151e-6d46-4a02-8de4-1c4f21c7da95\functions\util-web-action_on_element.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.Screenshot.cs

@@ -6,10 +6,10 @@ public async Task<BrowserActionResult> ScreenshotAsync(MessageInfo message, stri
     {
         var result = new BrowserActionResult();
 
-        await _instance.Wait(message.ContextId);
+        await _instance.Wait(message.ContextId, waitNetworkIdle: false);
         var page = _instance.GetPage(message.ContextId);
 
-        await Task.Delay(500);
+        await Task.Delay(300);
         var bytes = await page.ScreenshotAsync(new PageScreenshotOptions
         {
             Path = path,

src/Plugins/BotSharp.Plugin.WebDriver/Hooks/WebUtilityHook.cs

@@ -3,7 +3,9 @@ namespace BotSharp.Plugin.WebDriver.Hooks;
 public class WebUtilityHook : IAgentUtilityHook
 {
     private const string PREFIX = "util-web-";
+    private const string CLOSE_BROWSER_FN = $"{PREFIX}close_browser";
     private const string GO_TO_PAGE_FN = $"{PREFIX}go_to_page";
+    private const string LOCATE_ELEMENT_FN = $"{PREFIX}locate_element";
     private const string ACTION_ON_ELEMENT_FN = $"{PREFIX}action_on_element";
 
     public void AddUtilities(List<AgentUtility> utilities)
@@ -15,7 +17,9 @@ public void AddUtilities(List<AgentUtility> utilities)
                 Name = "web.tools",
                 Functions = 
                 [
+                    new(CLOSE_BROWSER_FN),
                     new(GO_TO_PAGE_FN),
+                    new(LOCATE_ELEMENT_FN),
                     new(ACTION_ON_ELEMENT_FN)
                 ],
                 Templates = 

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs

@@ -21,22 +21,35 @@ public async Task<bool> Execute(RoleDialogModel message)
         var actionArgs = JsonSerializer.Deserialize<ElementActionArgs>(message.FunctionArgs);
         if (actionArgs.Action == BroswerActionEnum.InputText)
         {
-            actionArgs.PressKey = "Enter";
+            // Replace variable in input text
+            if (actionArgs.Content.StartsWith("@"))
+            {
+                var config = _services.GetRequiredService<IConfiguration>();
+                var key = actionArgs.Content.Replace("@", string.Empty);
+                actionArgs.Content = key.Replace(key, config[key]);
+            }
         }
+
         actionArgs.WaitTime = 2;
 
         var conv = _services.GetRequiredService<IConversationService>();
 
         var browser = _services.GetRequiredService<IWebBrowser>();
-        var result = await browser.ActionOnElement(new MessageInfo
+        var msg = new MessageInfo
         {
             AgentId = message.CurrentAgentId,
             MessageId = message.MessageId,
             ContextId = message.CurrentAgentId,
-        }, locatorArgs, actionArgs);
+        };
+        var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
 
         message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}";
 
+        var webDriverService = _services.GetRequiredService<WebDriverService>();
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
+
+        message.Data = await browser.ScreenshotAsync(msg, path);
+
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebCloseBrowserFn.cs

@@ -20,10 +20,22 @@ public async Task<bool> Execute(RoleDialogModel message)
         var conv = _services.GetRequiredService<IConversationService>();
 
         var browser = _services.GetRequiredService<IWebBrowser>();
+        var msg = new MessageInfo
+        {
+            AgentId = message.CurrentAgentId,
+            MessageId = message.MessageId,
+            ContextId = message.CurrentAgentId,
+        };
+
         await browser.CloseBrowser(message.CurrentAgentId);
 
         message.Content = $"Browser closed.";
 
+        var webDriverService = _services.GetRequiredService<WebDriverService>();
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
+
+        message.Data = await browser.ScreenshotAsync(msg, path);
+
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebGoToPageFn.cs

@@ -35,10 +35,20 @@ public async Task<bool> Execute(RoleDialogModel message)
             ContextId = message.CurrentAgentId,
         };
         await browser.CloseCurrentPage(msg);
-        await browser.GoToPage(msg, args);
+        var result = await browser.GoToPage(msg, args);
+        if (!result.IsSuccess)
+        {
+            message.Content = "Open web page failed.";
+            return false;
+        }
 
         message.Content = $"Open web page successfully.";
 
+        var webDriverService = _services.GetRequiredService<WebDriverService>();
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
+
+        message.Data = await browser.ScreenshotAsync(msg, path);
+
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebLocateElementFn.cs

@@ -0,0 +1,42 @@
+namespace BotSharp.Plugin.WebDriver.UtilFunctions;
+
+public class UtilWebLocateElementFn : IFunctionCallback
+{
+    public string Name => "util-web-locate_element";
+    public string Indication => "Locating element.";
+    private readonly IServiceProvider _services;
+    private readonly ILogger _logger;
+
+    public UtilWebLocateElementFn(
+        IServiceProvider services,
+        ILogger<UtilWebLocateElementFn> logger)
+    {
+        _services = services;
+        _logger = logger;
+    }
+
+    public async Task<bool> Execute(RoleDialogModel message)
+    {
+        var locatorArgs = JsonSerializer.Deserialize<ElementLocatingArgs>(message.FunctionArgs ?? "{}");
+        var conv = _services.GetRequiredService<IConversationService>();
+        locatorArgs.Highlight = true;
+
+        var browser = _services.GetRequiredService<IWebBrowser>();
+        var msg = new MessageInfo
+        {
+            AgentId = message.CurrentAgentId,
+            MessageId = message.MessageId,
+            ContextId = message.CurrentAgentId,
+        };
+        var result = await browser.LocateElement(msg, locatorArgs);
+
+        message.Content = $"Locating element {(result.IsSuccess ? "success" : "failed")}";
+
+        var webDriverService = _services.GetRequiredService<WebDriverService>();
+        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
+
+        message.Data = await browser.ScreenshotAsync(msg, path);
+
+        return true;
+    }
+}

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json

@@ -16,6 +16,11 @@
       "content": {
         "type": "string",
         "description": "content to input in element"
+      },
+      "press_key": {
+        "type": "string",
+        "enum": [ "Enter" ],
+        "description": "press key after input text"
       }
     },
     "required": [ "selector", "action" ]

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-locate_element.json

@@ -0,0 +1,14 @@
+{
+  "name": "util-web-locate_element",
+  "description": "Check if specific element exists in web page",
+  "parameters": {
+    "type": "object",
+    "properties": {
+      "selector": {
+        "type": "string",
+        "description": "element selector in XPath, use syntax of Playwright in .NET"
+      }
+    },
+    "required": [ "selector" ]
+  }
+}