Logging Kibana URL for transformers with failures

servicex/app/transforms.py

@@ -188,8 +188,11 @@ def create_kibana_link_parameters(log_url, transform_id=None, log_level=None, ti
     """
     Create the _a and _g parameters for the kibana dashboard link
     """
-    a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)},"\
-                  f"{add_query('level', log_level.value.lower())}))"
+    if log_level:
+        a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)},"\
+                      f"{add_query('level', log_level.value.lower())}))"
+    else:
+        a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)}))"
     g_parameter = f"&_g=({select_time(time_frame.value.lower())})"
     kibana_link = re.sub(r"\&\_g\=\(\)", g_parameter + a_parameter, log_url)
     return kibana_link

servicex/query.py

@@ -34,6 +34,8 @@
 import logging
 from typing import List, Optional, Union
 from servicex.expandable_progress import ExpandableProgress
+from rich.logging import RichHandler
+
 
 try:
     import pandas as pd
@@ -62,6 +64,8 @@
 DONE_STATUS = (Status.complete, Status.canceled, Status.fatal)
 ProgressIndicators = Union[Progress, ExpandableProgress]
 logger = logging.getLogger(__name__)
+shell_handler = RichHandler(markup=True)
+logger.addHandler(shell_handler)
 
 
 class ServiceXException(Exception):
@@ -180,6 +184,8 @@ async def submit_and_download(
         :return: Transform results object which contains the list of files downloaded
                  or the list of pre-signed urls
         """
+        from servicex.app.transforms import \
+            create_kibana_link_parameters, TimeFrame, LogLevel
         download_files_task = None
         loop = asyncio.get_running_loop()
 
@@ -212,7 +218,12 @@ def transform_complete(task: Task):
                         f"{self.current_status.files} files failed"
                     )
                     if self.current_status.log_url is not None:
-                        logger.warning(f"More information at: {self.current_status.log_url}")
+                        kibana_link = \
+                            create_kibana_link_parameters(self.current_status.log_url,
+                                                          self.current_status.request_id,
+                                                          LogLevel.error,
+                                                          TimeFrame.month)
+                        logger.warning(f"More information of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]")  # NOQA: E501
                 else:
                     logger.info("Transforms completed successfully")
             else:  # pragma: no cover
@@ -341,6 +352,8 @@ async def transform_status_listener(
         of status. Once we know the number of files in the dataset, update the progress
         bars.
         """
+        from servicex.app.transforms import LogLevel, \
+            create_kibana_link_parameters, TimeFrame
 
         # Actual number of files in the dataset. We only know this once the DID
         # finder has completed its work. In the meantime transformers will already
@@ -387,12 +400,23 @@ async def transform_status_listener(
                     )
                     err_str = f"Request {titlestr}was canceled"
                     if self.current_status.log_url is not None:
-                        err_str += f"\nLogfiles at {self.current_status.log_url}"
+                        kibana_link = \
+                            create_kibana_link_parameters(self.current_status.log_url,
+                                                          self.current_status.request_id,
+                                                          LogLevel.error,
+                                                          TimeFrame.month)
+                        logger.error(f"{err_str}\nMore logfiles of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]")  # NOQA: E501"
                     raise ServiceXException(err_str)
+
                 else:
                     err_str = f"Fatal issue in ServiceX server for request {titlestr}"
                     if self.current_status.log_url is not None:
-                        err_str += f"\nMore logfiles at {self.current_status.log_url}"
+                        kibana_link = \
+                            create_kibana_link_parameters(self.current_status.log_url,
+                                                          self.current_status.request_id,
+                                                          LogLevel.error,
+                                                          TimeFrame.month)
+                        logger.error(f"{err_str}\nMore logfiles of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]")  # NOQA: E501"
                     raise ServiceXException(err_str)
 
             await asyncio.sleep(self.servicex_polling_interval)

tests/test_servicex_app_transforms.py

@@ -58,3 +58,15 @@ def test_create_kibana_link_parameters():
                 "&hide-filter-bar=true"
     assert create_kibana_link_parameters(initial_log_url, transform_id,
                                          log_level, time_frame) == final_url
+
+    transform_id = "93713b34-2f0b-4d53-8412-8afa98626516"
+    log_level = None
+    time_frame = TimeFrame.month
+    final_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"\
+                "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"\
+                "embed=true&_g=(time:(from:now-30d%2Fd,to:now))"\
+                "&_a=(filters:!((query:(match_phrase:"\
+                "(requestId:'93713b34-2f0b-4d53-8412-8afa98626516')))))"\
+                "&show-time-filter=true&hide-filter-bar=true"
+    assert create_kibana_link_parameters(initial_log_url, transform_id,
+                                         log_level, time_frame) == final_url