Remove codegen caching

Author: ponyisi

servicex/app/codegen.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, IRIS-HEP
+# Copyright (c) 2022-2025, IRIS-HEP
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -36,20 +36,6 @@
 codegen_app = typer.Typer(name="codegen", no_args_is_help=True)
 
 
-@codegen_app.command(no_args_is_help=False)
-def flush(
-    backend: Optional[str] = backend_cli_option,
-    config_path: Optional[str] = config_file_option,
-):
-    """
-    Flush the available code generators from the cache
-    """
-    sx = ServiceXClient(backend=backend, config_path=config_path)
-    cache = sx.query_cache
-    cache.delete_codegen_by_backend(backend)
-    rich.print("Deleted cached code generators.")
-
-
 @codegen_app.command(no_args_is_help=False)
 def list(
     backend: Optional[str] = backend_cli_option,

servicex/query_cache.py

@@ -209,28 +209,3 @@ def delete_record_by_hash(self, hash: str):
         transforms = Query()
         with self.lock:
             self.db.remove(transforms.hash == hash)
-
-    def get_codegen_by_backend(self, backend: str) -> Optional[dict]:
-        codegens = Query()
-        with self.lock:
-            records = self.db.search(codegens.backend == backend)
-
-        if not records:
-            return None
-
-        if len(records) != 1:
-            raise CacheException("Multiple records found in db for same backend")
-        else:
-            return records[0]
-
-    def update_codegen_by_backend(self, backend: str, codegen_list: list):
-        transforms = Query()
-        with self.lock:
-            self.db.upsert(
-                {"backend": backend, "codegens": codegen_list},
-                transforms.backend == backend,
-            )
-
-    def delete_codegen_by_backend(self, backend: str):
-        with self.lock:
-            self.db.remove(where("backend") == backend)

servicex/servicex_adapter.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2022, IRIS-HEP
+# Copyright (c) 2022-2025, IRIS-HEP
 # All rights reserved.
 #
 # Redistribution and use in source and binary forms, with or without
@@ -31,7 +31,6 @@
 from typing import Optional, Dict, List
 from dataclasses import dataclass
 
-import httpx
 from httpx import AsyncClient, Response
 from json import JSONDecodeError
 from httpx_retries import RetryTransport, Retry
@@ -146,7 +145,7 @@ async def get_servicex_info(self) -> ServiceXInfo:
         retry_options = Retry(total=3, backoff_factor=10)
         async with AsyncClient(transport=RetryTransport(retry=retry_options)) as client:
             r = await client.get(url=f"{self.url}/servicex", headers=headers)
-            if r.status_code == 401:
+            if r.status_code in (401, 403):
                 raise AuthorizationError(
                     f"Not authorized to access serviceX at {self.url}"
                 )
@@ -200,15 +199,8 @@ async def get_transforms(self) -> List[TransformStatus]:
             statuses = [TransformStatus(**status) for status in o["requests"]]
             return statuses
 
-    def get_code_generators(self):
-        with httpx.Client() as client:
-            r = client.get(url=f"{self.url}/multiple-codegen-list")
-
-            if r.status_code == 403:
-                raise AuthorizationError(
-                    f"Not authorized to access serviceX at {self.url}"
-                )
-            return r.json()
+    async def get_code_generators(self) -> dict[str, str]:
+        return (await self.get_servicex_info()).code_gen_image
 
     async def get_datasets(
         self, did_finder=None, show_deleted=False

servicex/servicex_client.py

@@ -348,7 +348,7 @@ def __init__(self, backend=None, url=None, config_path=None):
             )
 
         self.query_cache = QueryCache(self.config)
-        self.code_generators = set(self.get_code_generators(backend).keys())
+        self.code_generators = set(self.get_code_generators().keys())
 
     async def get_transforms_async(self) -> List[TransformStatus]:
         r"""
@@ -404,21 +404,12 @@ def cancel_transform(self, transform_id) -> None:
         """
         return _async_execute_and_wait(self.servicex.cancel_transform(transform_id))
 
-    def get_code_generators(self, backend=None):
+    def get_code_generators(self) -> dict[str, str]:
         r"""
         Retrieve the code generators deployed with the serviceX instance
         :return:  The list of code generators as json dictionary
         """
-        cached_backends = None
-        if backend:
-            cached_backends = self.query_cache.get_codegen_by_backend(backend)
-        if cached_backends:
-            logger.info("Returning code generators from cache")
-            return cached_backends["codegens"]
-        else:
-            code_generators = self.servicex.get_code_generators()
-            self.query_cache.update_codegen_by_backend(backend, code_generators)
-            return code_generators
+        return _async_execute_and_wait(self.servicex.get_code_generators())
 
     def generic_query(
         self,

tests/app/test_codegen.py

@@ -41,20 +41,3 @@ def test_codegen_list(script_runner):
 }
 """
         )
-
-
-def test_codegen_flush(script_runner):
-    with patch("servicex.query_cache.QueryCache.delete_codegen_by_backend") as p:
-        result = script_runner.run(
-            [
-                "servicex",
-                "codegen",
-                "flush",
-                "-c",
-                "tests/example_config.yaml",
-                "-b",
-                "localhost",
-            ]
-        )
-        assert result.returncode == 0
-        p.assert_called_once_with("localhost")

tests/test_query_cache.py

@@ -33,7 +33,6 @@
 from servicex.configuration import Configuration
 from servicex.models import ResultFormat
 from servicex.query_cache import QueryCache, CacheException
-from tinydb import Query
 
 file_uris = ["/tmp/foo1.root", "/tmp/foo2.root"]
 
@@ -185,83 +184,7 @@ def test_record_delete(transform_request, completed_status):
         cache.close()
 
 
-def test_get_codegen_by_backend_empty():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        result = cache.get_codegen_by_backend("non-existent")
-        assert result is None
-        cache.close()
-
-
-def test_update_codegen_by_backend_single():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        codegens = Query()
-        cache.update_codegen_by_backend("backend_1", ["codegen_1"])
-        result = cache.db.search(codegens.backend == "backend_1")
-        assert len(result) == 1
-        assert result[0] == {"backend": "backend_1", "codegens": ["codegen_1"]}
-        cache.close()
-
-
-def test_get_codegen_by_backend_single():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        cache.update_codegen_by_backend("backend_1", ["codegen_1"])
-        result = cache.get_codegen_by_backend("backend_1")
-        assert result == {"backend": "backend_1", "codegens": ["codegen_1"]}
-        cache.close()
-
-
-def test_delete_codegen_by_backend():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        cache.update_codegen_by_backend("backend_1", ["codegen_1"])
-        result = cache.get_codegen_by_backend("backend_1")
-        assert result == {"backend": "backend_1", "codegens": ["codegen_1"]}
-
-        cache.delete_codegen_by_backend("backend_1")
-        result = cache.get_codegen_by_backend("backend_1")
-        assert result is None
-        cache.close()
-
-
-def test_delete_codegen_by_backend_nonexistent():
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        cache.delete_codegen_by_backend("backend_1")
-        with pytest.raises(AssertionError):
-            raise AssertionError()
-        cache.close()
-
-
-def test_add_both_codegen_and_transform_to_cache(transform_request, completed_status):
-    with tempfile.TemporaryDirectory() as temp_dir:
-        config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
-        cache = QueryCache(config)
-        cache.cache_transform(
-            cache.transformed_results(
-                transform=transform_request,
-                completed_status=completed_status,
-                data_dir="/foo/bar",
-                file_list=file_uris,
-                signed_urls=[],
-            )
-        )
-
-        cache.update_codegen_by_backend("backend_1", ["codegen_1"])
-        result = cache.get_codegen_by_backend("backend_1")
-        assert result == {"backend": "backend_1", "codegens": ["codegen_1"]}
-        assert len(cache.cached_queries()) == 1
-        cache.close()
-
-
-def test_delete_codegen_by_hash(transform_request, completed_status):
+def test_delete_transform_by_hash(transform_request, completed_status):
     with tempfile.TemporaryDirectory() as temp_dir:
         config = Configuration(cache_path=temp_dir, api_endpoints=[])  # type: ignore
         cache = QueryCache(config)

tests/test_servicex_adapter.py

@@ -162,21 +162,29 @@ async def test_get_transforms_with_refresh(get, post, transform_status_response)
     )
 
 
-@patch("servicex.servicex_adapter.httpx.Client.get")
-def test_get_codegens(get, servicex):
+@patch("servicex.servicex_adapter.AsyncClient.get")
+async def test_get_codegens(get, servicex):
     get.return_value = httpx.Response(
-        200, json={"uproot": "http://uproot-codegen", "xaod": "http://xaod-codegen"}
+        200,
+        json={
+            "app-version": "0.0.0",
+            "code-gen-image": {
+                "uproot": "http://uproot-codegen",
+                "xaod": "http://xaod-codegen",
+            },
+            "capabilities": [],
+        },
     )
-    c = servicex.get_code_generators()
+    c = await servicex.get_code_generators()
     assert len(c) == 2
     assert c["uproot"] == "http://uproot-codegen"
 
 
-@patch("servicex.servicex_adapter.httpx.Client.get")
-def test_get_codegens_error(get, servicex):
+@patch("servicex.servicex_adapter.AsyncClient.get")
+async def test_get_codegens_error(get, servicex):
     get.return_value = httpx.Response(403)
     with pytest.raises(AuthorizationError) as err:
-        servicex.get_code_generators()
+        await servicex.get_code_generators()
         assert "Not authorized to access serviceX at" in str(err.value)
 
 

tests/test_servicex_client.py

@@ -41,6 +41,10 @@
 def servicex_adaptor(mocker):
     adapter_mock = mocker.patch("servicex.servicex_client.ServiceXAdapter")
     mock_adapter = MagicMock(spec=ServiceXAdapter)
+    mock_adapter.get_code_generators.return_value = {
+        "uproot": "http://uproot-codegen",
+        "xaod": "http://xaod-codegen",
+    }
 
     adapter_mock.return_value = mock_adapter
     return mock_adapter
@@ -50,9 +54,6 @@ def servicex_adaptor(mocker):
 def mock_cache(mocker):
     cache_mock = mocker.patch("servicex.servicex_client.QueryCache")
     mock_cache = MagicMock(spec=QueryCache)
-    mock_cache.get_codegen_by_backend.return_value = {
-        "codegens": {"ROOT": "my_root_generator", "UPROOT": "my_uproot_generator"}
-    }
     cache_mock.return_value = mock_cache
     return cache_mock
 

tests/test_servicex_dataset.py

@@ -593,7 +593,7 @@ async def test_submit_generic(mocker, codegen_list):
     mocker.patch("servicex.minio_adapter.MinioAdapter", return_value=mock_minio)
     did = FileListDataset("/foo/bar/baz.root")
     with patch(
-        "servicex.servicex_adapter.ServiceXAdapter.get_code_generators",
+        "servicex.servicex_client.ServiceXClient.get_code_generators",
         return_value=codegen_list,
     ):
         client = ServiceXClient(
@@ -641,7 +641,7 @@ async def test_submit_cancelled(mocker, codegen_list):
     mocker.patch("servicex.minio_adapter.MinioAdapter", return_value=mock_minio)
     did = FileListDataset("/foo/bar/baz.root")
     with patch(
-        "servicex.servicex_adapter.ServiceXAdapter.get_code_generators",
+        "servicex.servicex_client.ServiceXClient.get_code_generators",
         return_value=codegen_list,
     ):
         client = ServiceXClient(