Merge pull request #1016 from visagang/features/vguruparan

Add webdriver hook to support file uploads

Author: Oceania2018

src/Infrastructure/BotSharp.Abstraction/Browsing/IWebDriverHook.cs

@@ -0,0 +1,8 @@
+using BotSharp.Abstraction.Browsing.Models;
+
+namespace BotSharp.Abstraction.Browsing;
+
+public interface IWebDriverHook
+{
+    Task<List<string>> GetUploadFiles(MessageInfo message);
+}

src/Infrastructure/BotSharp.Abstraction/Browsing/Models/MessageInfo.cs

@@ -12,6 +12,7 @@ public class MessageInfo : ICacheKey
     public string? MessageId { get; set; }
     public string? TaskId { get; set; }
     public string StepId { get; set; } = Guid.NewGuid().ToString();
+    public string? FunctionArgs { get; set; }
 
     public string GetCacheKey()
         => $"{nameof(MessageInfo)}";

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.DoAction.cs

@@ -80,8 +80,20 @@ await locator.ClickAsync(new LocatorClickOptions
         }
         else if (action.Action == BroswerActionEnum.FileUpload)
         {
-            if (action.FileUrl.Length == 0)
+            var _states = _services.GetRequiredService<IConversationStateService>();
+            var files = new List<string>();
+            if (action.FileUrl != null && action.FileUrl.Length > 0)
             {
+                files.AddRange(action.FileUrl);
+            }
+            var hooks = _services.GetServices<IWebDriverHook>();
+            foreach (var hook in hooks)
+            {
+                files.AddRange(await hook.GetUploadFiles(message));
+            }
+            if (files.Count == 0)
+            {
+                Serilog.Log.Warning($"No files found to upload: {action.Content}");
                 return;
             }
             var fileChooser = await page.RunAndWaitForFileChooserAsync(async () =>
@@ -97,7 +109,7 @@ await locator.ClickAsync(new LocatorClickOptions
             Directory.CreateDirectory(directory);
             var localPaths = new List<string>();
             using var httpClient = new HttpClient();
-            foreach (var fileUrl in action.FileUrl)
+            foreach (var fileUrl in files)
             {
                 var bytes = await httpClient.GetByteArrayAsync(fileUrl);
                 var fileName = new Uri(fileUrl).AbsolutePath;

src/Plugins/BotSharp.Plugin.WebDriver/Drivers/PlaywrightDriver/PlaywrightWebDriver.cs

@@ -71,7 +71,7 @@ public void SetAgent(Agent agent)
 
     public void SetServiceProvider(IServiceProvider services)
     {
-        _instance.SetServiceProvider(_services);
+        _instance.SetServiceProvider(services);
     }
 
     public async Task PressKey(MessageInfo message, string key)

src/Plugins/BotSharp.Plugin.WebDriver/UtilFunctions/UtilWebActionOnElementFn.cs

@@ -19,44 +19,52 @@ public async Task<bool> Execute(RoleDialogModel message)
     {
         var locatorArgs = JsonSerializer.Deserialize<ElementLocatingArgs>(message.FunctionArgs);
         var actionArgs = JsonSerializer.Deserialize<ElementActionArgs>(message.FunctionArgs);
-        if (actionArgs.Action == BroswerActionEnum.InputText)
+        try
         {
-            // Replace variable in input text
-            if (actionArgs.Content.StartsWith("@"))
+            if (actionArgs.Action == BroswerActionEnum.InputText)
             {
-                var config = _services.GetRequiredService<IConfiguration>();
-                var key = actionArgs.Content.Replace("@", string.Empty);
-                actionArgs.Content = key.Replace(key, config[key]);
+                // Replace variable in input text
+                if (actionArgs.Content.StartsWith("@"))
+                {
+                    var config = _services.GetRequiredService<IConfiguration>();
+                    var key = actionArgs.Content.Replace("@", string.Empty);
+                    actionArgs.Content = key.Replace(key, config[key]);
+                }
             }
-        }
-
-        actionArgs.WaitTime = actionArgs.WaitTime > 0 ? actionArgs.WaitTime : 2;
 
-        var conv = _services.GetRequiredService<IConversationService>();
+            actionArgs.WaitTime = actionArgs.WaitTime > 0 ? actionArgs.WaitTime : 2;
 
-        var services = _services.CreateScope().ServiceProvider;
-        var browser = services.GetRequiredService<IWebBrowser>();
-        var webDriverService = _services.GetRequiredService<WebDriverService>();
-        var msg = new MessageInfo
-        {
-            AgentId = message.CurrentAgentId,
-            MessageId = message.MessageId,
-            ContextId = webDriverService.GetMessageContext(message),
-        };
-        var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
+            var services = _services.CreateScope().ServiceProvider;
+            var browser = services.GetRequiredService<IWebBrowser>();
+            var webDriverService = _services.GetRequiredService<WebDriverService>();
+            var msg = new MessageInfo
+            {
+                AgentId = message.CurrentAgentId,
+                MessageId = message.MessageId,
+                ContextId = webDriverService.GetMessageContext(message),
+                FunctionArgs = message.FunctionArgs
+            };
+            browser.SetServiceProvider(_services);
+            var result = await browser.ActionOnElement(msg, locatorArgs, actionArgs);
 
-        message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}.";
+            message.Content = $"{actionArgs.Action} executed {(result.IsSuccess ? "success" : "failed")}.";
 
-        // Add Current Url info to the message
-        if (actionArgs.ShowCurrentUrl)
-        {
-            message.Content += $" Current page url: '{result.UrlAfterAction}'.";
-        }
+            // Add Current Url info to the message
+            if (actionArgs.ShowCurrentUrl)
+            {
+                message.Content += $" Current page url: '{result.UrlAfterAction}'.";
+            }
 
-        var path = webDriverService.GetScreenshotFilePath(message.MessageId);
+            var path = webDriverService.GetScreenshotFilePath(message.MessageId);
 
-        message.Data = await browser.ScreenshotAsync(msg, path);
+            message.Data = await browser.ScreenshotAsync(msg, path);
 
+        }
+        catch (Exception ex)
+        {
+            message.Data = $"{actionArgs.Action} execution failed.";
+            _logger.LogError($"UtilWebActionOnElementFn exception: {ex.Message}. StackTrace: {ex.StackTrace}");
+        }
         return true;
     }
 }

src/Plugins/BotSharp.Plugin.WebDriver/data/agents/6745151e-6d46-4a02-8de4-1c4f21c7da95/functions/util-web-action_on_element.json

@@ -37,6 +37,10 @@
       "wait_time": {
         "type": "number",
         "description": "wait time after action in seconds"
+      },
+      "metadata": {
+        "type": "string",
+        "description": "meta data information if user provided"
       }
     },
     "required": [ "selector", "action" ]