Update the kibana link code to use the new, cleaner Kibana URL

Author: BenGalewsky

servicex/app/transforms.py

@@ -26,6 +26,8 @@
 # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
 # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 import asyncio
+import urllib.parse
+from urllib.parse import urlunparse
 from pathlib import Path
 from typing import Optional, List
 import webbrowser
@@ -233,58 +235,44 @@ class LogLevel(str, Enum):
     Level of the log messages: INFO & ERROR
     """
 
-    info = ("INFO",)
-    error = ("ERROR",)
+    info = "INFO"
+    error = "ERROR"
 
 
-def add_query(key, value):
+def create_kibana_link_parameters(
+    log_url, transform_id=None, log_level: LogLevel = LogLevel.error
+):
     """
-    Creates query string from the key and value pairs
+    Add a filter to a Kibana dashboard URL to show only results for a given request ID
+    with an ERROR level or higher.
     """
-    query_string = "(query:(match_phrase:({0}:'{1}')))".format(key, value)
-    return query_string
+    decoded_url = urllib.parse.urlparse(log_url)
 
+    view_match = re.search(r"/view/([^?]+)", decoded_url.fragment)
+    instance_match = re.search(r"instance:([^)]+)", decoded_url.fragment)
+    index_match = re.search(r"index:'([^']+)'", decoded_url.fragment)
 
-def select_time(time_frame=TimeFrame.day):
-    """
-    Takes input as 'day','week','month' and returns the time filter
-    """
-    time_string = time_frame
-    if time_frame.lower() == TimeFrame.day:
-        time_string = "time:(from:now%2Fd,to:now%2Fd)"
-    elif time_frame.lower() == TimeFrame.week:
-        time_string = "time:(from:now%2Fw,to:now%2Fw)"
-    elif time_frame.lower() == TimeFrame.month:
-        time_string = "time:(from:now-30d%2Fd,to:now)"
-    else:
-        rich.print("Got a time frame apart from 'day', 'week', 'month'")
-    return time_string
+    view = view_match.group(1) if view_match else None
+    instance = instance_match.group(1) if instance_match else None
+    index = index_match.group(1) if index_match else None
 
+    # If we are unable to parse the fragment, return the original URL
+    if view is None or instance is None or index is None:
+        return log_url
 
-def create_kibana_link_parameters(
-    log_url, transform_id=None, log_level=None, time_frame=None
-):
-    """
-    Create the _a and _g parameters for the kibana dashboard link
-    """
-    if log_level:
-        a_parameter = (
-            f"&_a=(filters:!({add_query('requestId', transform_id)},"
-            f"{add_query('level', log_level.value.lower())}))"
-        )
-    else:
-        a_parameter = f"&_a=(filters:!({add_query('requestId', transform_id)}))"
-    g_parameter = f"&_g=({select_time(time_frame.value.lower())})"
-    kibana_link = re.sub(r"\&\_g\=\(\)", g_parameter + a_parameter, log_url)
-    return kibana_link
+    _a = f"(filters:!((query:(match_phrase:(instance:{instance}))),(query:(match_phrase:(requestId:'{transform_id}'))),(query:(match_phrase:(level:{log_level.value})))),index:'{index}')"  # NOQA  E502
+
+    new_fragment = f"/view/{view}?embed=true&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),time:(from:now-24h/h,to:now))"  # NOQA  E502
+    new_fragment += f"&_a={urllib.parse.quote(_a)}"
+    decoded_url = decoded_url._replace(fragment=new_fragment)
+    return urlunparse(decoded_url)
 
 
 @transforms_app.command(no_args_is_help=True)
 def logs(
     backend: Optional[str] = backend_cli_option,
     transform_id: str = transform_id_arg,
     log_level: Optional[LogLevel] = log_level_opt,
-    time_frame: Optional[TimeFrame] = time_frame_opt,
 ):
     """
     Open the URL to the Kibana dashboard of the logs of a tranformer
@@ -293,10 +281,7 @@ def logs(
     transforms = sx.get_transform_status(transform_id)
     if transforms and transforms.request_id == transform_id:
         kibana_link = create_kibana_link_parameters(
-            transforms.log_url,
-            transform_id=transform_id,
-            log_level=log_level,
-            time_frame=time_frame,
+            transforms.log_url, transform_id=transform_id, log_level=log_level
         )
         print(kibana_link)
         webbrowser.open(kibana_link)

servicex/query_core.py

@@ -183,11 +183,7 @@ async def submit_and_download(
         :return: Transform results object which contains the list of files downloaded
                  or the list of pre-signed urls
         """
-        from servicex.app.transforms import (
-            create_kibana_link_parameters,
-            TimeFrame,
-            LogLevel,
-        )
+        from servicex.app.transforms import create_kibana_link_parameters
 
         download_files_task = None
         loop = asyncio.get_running_loop()
@@ -240,10 +236,7 @@ def transform_complete(task: Task):
                     )
                     if self.current_status.log_url is not None:
                         kibana_link = create_kibana_link_parameters(
-                            self.current_status.log_url,
-                            self.current_status.request_id,
-                            LogLevel.error,
-                            TimeFrame.month,
+                            self.current_status.log_url, self.current_status.request_id
                         )
                         logger.error(
                             f"More information of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]"  # NOQA: E501
@@ -412,11 +405,7 @@ async def transform_status_listener(
         of status. Once we know the number of files in the dataset, update the progress
         bars.
         """
-        from servicex.app.transforms import (
-            LogLevel,
-            create_kibana_link_parameters,
-            TimeFrame,
-        )
+        from servicex.app.transforms import create_kibana_link_parameters
 
         # Actual number of files in the dataset. We only know this once the DID
         # finder has completed its work. In the meantime transformers will already
@@ -476,8 +465,6 @@ async def transform_status_listener(
                             kibana_link = create_kibana_link_parameters(
                                 self.current_status.log_url,
                                 self.current_status.request_id,
-                                LogLevel.error,
-                                TimeFrame.month,
                             )
                             logger.warning(
                                 f"More logfiles of '{self.title}' [bold red on white]"
@@ -504,10 +491,7 @@ async def transform_status_listener(
                     err_str = f"Request {titlestr}was canceled"
                     if self.current_status.log_url is not None:
                         kibana_link = create_kibana_link_parameters(
-                            self.current_status.log_url,
-                            self.current_status.request_id,
-                            LogLevel.error,
-                            TimeFrame.month,
+                            self.current_status.log_url, self.current_status.request_id
                         )
                         logger.error(
                             f"{err_str}\nMore logfiles of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]"  # NOQA: E501
@@ -549,10 +533,7 @@ async def transform_status_listener(
                         )
                     if self.current_status.log_url is not None:
                         kibana_link = create_kibana_link_parameters(
-                            self.current_status.log_url,
-                            self.current_status.request_id,
-                            LogLevel.error,
-                            TimeFrame.month,
+                            self.current_status.log_url, self.current_status.request_id
                         )
                         logger.error(
                             f"More logfiles of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]"  # NOQA: E501
@@ -562,10 +543,7 @@ async def transform_status_listener(
                     err_str = f"Fatal issue in ServiceX server for request {titlestr}"
                     if self.current_status.log_url is not None:
                         kibana_link = create_kibana_link_parameters(
-                            self.current_status.log_url,
-                            self.current_status.request_id,
-                            LogLevel.error,
-                            TimeFrame.month,
+                            self.current_status.log_url, self.current_status.request_id
                         )
                         logger.error(
                             f"{err_str}\nMore logfiles of '{self.title}' [bold red on white][link={kibana_link}]HERE[/link][/bold red on white]"  # NOQA: E501

tests/test_servicex_app_transforms.py

@@ -1,96 +1,73 @@
-from servicex.app.transforms import LogLevel, TimeFrame
-from servicex.app.transforms import (
-    add_query,
-    select_time,
-    create_kibana_link_parameters,
-)
+from servicex.app.transforms import LogLevel, create_kibana_link_parameters
 
 
-def test_add_query():
-    key = "abc"
-    value = "123-345-567"
-    query = "(query:(match_phrase:(abc:'123-345-567')))"
-    assert add_query(key, value) == query
+class TestAddRequestIdFilter:
+    """Tests for add_request_id_filter using a real Kibana dashboard URL."""
 
-    key = "requestId"
-    value = "d2ede739-9779-4075-95b1-0c7fae1de408"
-    query = "(query:(match_phrase:(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408')))"
-    assert add_query(key, value) == query
+    EXAMPLE_URL = (
+        "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards"
+        "?auth_provider_hint=anonymous1#/view/bb682100-5558-11ed-afcf-d91dad577662#"
+        "?embed=true"
+        "&_g=(filters:!(),refreshInterval:(pause:!t,value:1000),"
+        "time:(from:now-24h/h,to:now))"
+        "&_a=(filters:!((query:(match_phrase:(instance:servicex-unit-test))))"
+        ",index:'923eaa00-45b9-11ed-afcf-d91dad577662')"
+    )
 
+    def test_preserves_base_url(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert result.startswith(
+            "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards"
+        )
 
-def test_select_time():
-    time_frame = TimeFrame.week
-    time_filter = "time:(from:now%2Fw,to:now%2Fw)"
-    assert time_filter == select_time(time_frame)
+    def test_preserves_auth_query(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert "?auth_provider_hint=anonymous1" in result
 
-    time_frame = "month"
-    time_filter = "time:(from:now-30d%2Fd,to:now)"
-    assert time_filter == select_time(time_frame)
+    def test_preserves_view_path(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert "#/view/bb682100-5558-11ed-afcf-d91dad577662" in result
 
-    time_frame = "daY"
-    time_filter = "time:(from:now%2Fd,to:now%2Fd)"
-    assert time_filter == select_time(time_frame)
+    def test_includes_embed_true(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert "embed=true" in result
 
+    def test_preserves_instance_in_query(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert "servicex-unit-test" in result
 
-def test_create_kibana_link_parameters():
-    initial_log_url = (
-        "https://atlas-kibana.mwt2.org:5601/s/servicex/app"
-        "/dashboards?auth_provider_hint=anonymous1#/view/"
-        "2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?embed=true&_g=()"
-        "&show-time-filter=true&hide-filter-bar=true"
-    )
-    transform_id = "d2ede739-9779-4075-95b1-0c7fae1de408"
-    log_level = LogLevel.error
-    time_frame = TimeFrame.day
-    final_url = (
-        "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"
-        "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"
-        "embed=true&_g=(time:(from:now%2Fd,to:now%2Fd))"
-        "&_a=(filters:!((query:(match_phrase:"
-        "(requestId:'d2ede739-9779-4075-95b1-0c7fae1de408'))),"
-        "(query:(match_phrase:(level:'error')))))&show-time-filter=true"
-        "&hide-filter-bar=true"
-    )
-    assert (
-        create_kibana_link_parameters(
-            initial_log_url, transform_id, log_level, time_frame
+    def test_includes_request_id_in_query(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
         )
-        == final_url
-    )
+        assert "abc-123" in result
 
-    transform_id = "93713b34-2f0b-4d53-8412-8afa98626516"
-    log_level = LogLevel.info
-    time_frame = TimeFrame.month
-    final_url = (
-        "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"
-        "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"
-        "embed=true&_g=(time:(from:now-30d%2Fd,to:now))"
-        "&_a=(filters:!((query:(match_phrase:"
-        "(requestId:'93713b34-2f0b-4d53-8412-8afa98626516'))),"
-        "(query:(match_phrase:(level:'info')))))&show-time-filter=true"
-        "&hide-filter-bar=true"
-    )
-    assert (
-        create_kibana_link_parameters(
-            initial_log_url, transform_id, log_level, time_frame
+    def test_includes_log_level_in_query(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.error
         )
-        == final_url
-    )
+        assert "level%3AERROR" in result
 
-    transform_id = "93713b34-2f0b-4d53-8412-8afa98626516"
-    log_level = None
-    time_frame = TimeFrame.month
-    final_url = (
-        "https://atlas-kibana.mwt2.org:5601/s/servicex/app/dashboards?"
-        "auth_provider_hint=anonymous1#/view/2d2b3b40-f34e-11ed-a6d8-9f6a16cd6d78?"
-        "embed=true&_g=(time:(from:now-30d%2Fd,to:now))"
-        "&_a=(filters:!((query:(match_phrase:"
-        "(requestId:'93713b34-2f0b-4d53-8412-8afa98626516')))))"
-        "&show-time-filter=true&hide-filter-bar=true"
-    )
-    assert (
-        create_kibana_link_parameters(
-            initial_log_url, transform_id, log_level, time_frame
+    def test_includes_app_state_filter(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
         )
-        == final_url
-    )
+        assert "_a=" in result
+        assert "requestId" in result
+
+    def test_preserves_time_range(self):
+        result = create_kibana_link_parameters(
+            self.EXAMPLE_URL, "abc-123", log_level=LogLevel.info
+        )
+        assert "now-24h/h" in result
+        assert "to:now" in result