Process Transform Title and Log URL

servicex/app/transforms.py

@@ -28,6 +28,9 @@
 import asyncio
 from pathlib import Path
 from typing import Optional, List
+import webbrowser
+import re
+from enum import Enum
 
 import rich
 import typer
@@ -71,7 +74,7 @@ def list(
     for t in transforms:
         if not complete or complete and t.status == Status.complete:
             table.add_row(
-                t.request_id, "Not implemented", t.status, str(t.files_completed)
+                t.request_id, t.title, t.status, str(t.files_completed)
             )
 
     rich.print(table)
@@ -138,3 +141,83 @@ async def download_with_progress(filename) -> Path:
 
     for path in result_files:
         print(path.as_posix())
+
+
+class TimeFrame(str, Enum):
+    r"""
+    Time Frame levels: 'day', 'week' & 'month'
+    """
+    day = ("day",)
+    week = ("week",)
+    month = ("month")
+
+
+class LogLevel(str, Enum):
+    r"""
+    Level of the log messages: INFO & ERROR
+    """
+    info = ("INFO",)
+    error = ("ERROR",)
+
+
+def add_query(key, value):
+    """
+    Creates query string from the key and value pairs
+    """
+    query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value)
+    return query_string
+
+
+def select_time(time_frame=TimeFrame.day):
+    """
+    Takes input as 'day','week','month' and returns the time filter
+    """
+    time_string = time_frame
+    if time_frame.lower() == TimeFrame.day:
+        time_string = "time:(from:now%2Fd,to:now%2Fd)"
+    elif time_frame.lower() == TimeFrame.week:
+        time_string = "time:(from:now%2Fw,to:now%2Fw)"
+    elif time_frame.lower() == TimeFrame.month:
+        time_string = "time:(from:now-30d%2Fd,to:now)"
+    else:
+        rich.print("Got a time frame apart from 'day', 'week', 'month'")
+    return time_string
+
+
+def create_kibana_link_parameters(log_url, transform_id=None, log_level=None, time_frame=None):
+    """
+    Create the _a and _g parameters for the kibana dashboard link
+    """
+    a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)},"\
+                  f"{add_query('level', log_level.value.lower())}))"
+    g_parameter = f"&_g=({select_time(time_frame.value.lower())})"
+    kibana_link = re.sub(r"\&\_g\=\(\)", g_parameter + a_parameter, log_url)
+    return kibana_link
+
+
+@transforms_app.command(no_args_is_help=True)
+def logs(
+    url: Optional[str] = url_cli_option,
+    backend: Optional[str] = backend_cli_option,
+    transform_id: str = typer.Option(None, "-t", "--transform-id", help="Transform ID"),
+    log_level: Optional[LogLevel] = typer.Option("ERROR", "-l", "--log-level",
+                                                 help="Level of Logs",
+                                                 case_sensitive=False),
+    time_frame: Optional[TimeFrame] = typer.Option("month", "-f", "--time-frame",
+                                                   help="Time Frame",
+                                                   case_sensitive=False)
+):
+    """
+    Open the URL to the Kibana dashboard of the logs of a tranformer
+    """
+    sx = ServiceXClient(backend=backend, url=url)
+    transforms = sx.get_transform_status(transform_id)
+    if transforms and transforms.request_id == transform_id:
+        kibana_link = create_kibana_link_parameters(transforms.log_url,
+                                                    transform_id=transform_id,
+                                                    log_level=log_level,
+                                                    time_frame=time_frame)
+        print(kibana_link)
+        webbrowser.open(kibana_link)
+    else:
+        rich.print("Invalid Request ID")

servicex/models.py

@@ -107,6 +107,7 @@ class TransformStatus(BaseModel):
     """
     request_id: str
     did: str
+    title: Optional[str]
     selection: str
     tree_name: Optional[str] = Field(alias="tree-name")
     image: str
@@ -125,6 +126,7 @@ class TransformStatus(BaseModel):
     minio_secured: Optional[bool] = Field(alias="minio-secured")
     minio_access_key: Optional[str] = Field(alias="minio-access-key")
     minio_secret_key: Optional[str] = Field(alias="minio-secret-key")
+    log_url: Optional[str] = Field(alias="log-url")
 
     @validator("finish_time", pre=True)
     def parse_finish_time(cls, v):
@@ -157,3 +159,4 @@ class TransformedResults(BaseModel):
     signed_url_list: List[str]
     files: int
     result_format: ResultFormat
+    log_url: Optional[str]

servicex/query.py

@@ -223,7 +223,7 @@ def transform_complete(task: Task):
         # If we get here with a cached record, then we know that the transform
         # has been run, but we just didn't get the files from object store in the way
         # requested by user
-        transform_bar_title = "Transform"
+        transform_bar_title = f"{sx_request.title}: Transform"
         if not cached_record:
             transform_progress = expandable_progress.add_task(
                 transform_bar_title, start=False, total=None
@@ -236,6 +236,7 @@ def transform_complete(task: Task):
         minio_progress_bar_title = (
             "Download" if not signed_urls_only else "Signing URLS"
         )
+        minio_progress_bar_title = minio_progress_bar_title.rjust(len(transform_bar_title))
 
         download_progress = expandable_progress.add_task(
             minio_progress_bar_title, start=False, total=None
@@ -338,7 +339,7 @@ async def retrieve_current_transform_status(self):
         # Is this the first time we've polled status? We now know the request ID.
         # Update the display and set our download directory.
         if not self.current_status:
-            rich.print(f"[bold]ServiceX Transform {s.request_id}[/bold]")
+            rich.print(f"[bold]ServiceX Transform {s.title}: {s.request_id}[/bold]")
             self.download_path = self.cache.cache_path_for_transform(s)
 
         self.current_status = s

servicex/query_cache.py

@@ -64,7 +64,8 @@ def cache_transform(self, transform: TransformRequest,
             file_list=file_list,
             signed_url_list=signed_urls,
             files=completed_status.files,
-            result_format=transform.result_format
+            result_format=transform.result_format,
+            log_url=completed_status.log_url
         )
         self.db.insert(json.loads(record.json()))
         return record

servicex/servicex_adapter.py

@@ -122,5 +122,6 @@ async def get_transform_status(self, request_id: str) -> TransformStatus:
                 raise AuthorizationError(f"Not authorized to access serviceX at {self.url}")
             if r.status_code == 404:
                 raise ValueError(f"Transform ID {request_id} not found")
+
             status = TransformStatus(**r.json())
             return status

tests/test_servicex_app_transforms.py

@@ -0,0 +1,60 @@
+from servicex.app.transforms import LogLevel, TimeFrame
+from servicex.app.transforms import add_query, select_time, create_kibana_link_parameters
+
+
+def test_add_query():
+    key = "abc"
+    value = "123-345-567"
+    query = "(query:(match_phrase:(abc:'123-345-567')))"
+    assert add_query(key, value) == query
+
+    key = "requestId"
+    value = "d2ede739-9779-4075-95b1-0c7fae1de408"
+    query = "(query:(match_phrase:(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408')))"
+    assert add_query(key, value) == query
+
+
+def test_select_time():
+    time_frame = TimeFrame.week
+    time_filter = "time:(from:now%2Fw,to:now%2Fw)"
+    assert time_filter == select_time(time_frame)
+
+    time_frame = "month"
+    time_filter = "time:(from:now-30d%2Fd,to:now)"
+    assert time_filter == select_time(time_frame)
+
+    time_frame = "daY"
+    time_filter = "time:(from:now%2Fd,to:now%2Fd)"
+    assert time_filter == select_time(time_frame)
+
+
+def test_create_kibana_link_parameters():
+    initial_log_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app"\
+                      "/dashboards?auth_provider_hint=anonymous1#/view/"\
+                      "2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?embed=true&_g=()"\
+                      "&show-time-filter=true&hide-filter-bar=true"
+    transform_id = "d2ede739-9779-4075-95b1-0c7fae1de408"
+    log_level = LogLevel.error
+    time_frame = TimeFrame.day
+    final_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"\
+                "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"\
+                "embed=true&_g=(time:(from:now%2Fd,to:now%2Fd))"\
+                "&_a=(filters:!((query:(match_phrase:"\
+                "(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408'))),"\
+                "(query:(match_phrase:(level:'error')))))&show-time-filter=true"\
+                "&hide-filter-bar=true"
+    assert create_kibana_link_parameters(initial_log_url, transform_id,
+                                         log_level, time_frame) == final_url
+
+    transform_id = "93713b34-2f0b-4d53-8412-8afa98626516"
+    log_level = LogLevel.info
+    time_frame = TimeFrame.month
+    final_url = "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"\
+                "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"\
+                "embed=true&_g=(time:(from:now-30d%2Fd,to:now))"\
+                "&_a=(filters:!((query:(match_phrase:"\
+                "(requestId:'93713b34-2f0b-4d53-8412-8afa98626516'))),"\
+                "(query:(match_phrase:(level:'info')))))&show-time-filter=true"\
+                "&hide-filter-bar=true"
+    assert create_kibana_link_parameters(initial_log_url, transform_id,
+                                         log_level, time_frame) == final_url