From 7ada79ff87f18e23ee9b468130aa5b4d2f202546 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 01:46:46 +0000 Subject: [PATCH 01/10] Import queries into servicex.query on import --- pyproject.toml | 2 +- servicex/__init__.py | 7 ++--- servicex/databinder_models.py | 2 +- servicex/dataset_group.py | 2 +- servicex/func_adl/func_adl_dataset.py | 2 +- servicex/python_dataset.py | 2 +- servicex/query/__init__.py | 38 +++++++++++++++++++++++++++ servicex/{query.py => query_core.py} | 0 servicex/servicex_client.py | 7 +---- servicex/uproot_raw/uproot_raw.py | 2 +- tests/test_servicex_dataset.py | 2 +- 11 files changed, 50 insertions(+), 16 deletions(-) create mode 100644 servicex/query/__init__.py rename servicex/{query.py => query_core.py} (100%) diff --git a/pyproject.toml b/pyproject.toml index 716f1217..c0b46709 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ develop = [ "servicex[databinder,pandas,test,docs]", ] -[project.entry-points.'servicex.queries'] +[project.entry-points.'servicex.query'] FuncADL_Uproot = "servicex.func_adl.func_adl_dataset:FuncADLQuery_Uproot" FuncADL_ATLASr21 = "servicex.func_adl.func_adl_dataset:FuncADLQuery_ATLASr21" FuncADL_ATLASr22 = "servicex.func_adl.func_adl_dataset:FuncADLQuery_ATLASr22" diff --git a/servicex/__init__.py b/servicex/__init__.py index 565ab123..67c1cdba 100644 --- a/servicex/__init__.py +++ b/servicex/__init__.py @@ -34,11 +34,11 @@ from servicex.uproot_raw.uproot_raw import UprootRawQuery as UprootRaw from servicex.python_dataset import PythonQuery as PythonFunction from servicex.servicex_client import ServiceXClient, deliver -from .query import Query +from .query_core import Query from .models import ResultFormat, ResultDestination from .dataset_group import DatasetGroup from .dataset_identifier import RucioDatasetIdentifier, FileListDataset - +from . import query __all__ = [ "ServiceXClient", @@ -59,5 +59,6 @@ "General", "DefinitionList", "ServiceXSpec", - "deliver" + "deliver", + "query" ] diff --git a/servicex/databinder_models.py b/servicex/databinder_models.py index 0d796d4b..10027da1 100644 --- a/servicex/databinder_models.py +++ b/servicex/databinder_models.py @@ -34,7 +34,7 @@ ) from servicex.dataset_identifier import RucioDatasetIdentifier, FileListDataset -from servicex.query import Query as SXQuery, QueryStringGenerator +from servicex.query_core import Query as SXQuery, QueryStringGenerator from servicex.models import ResultFormat diff --git a/servicex/dataset_group.py b/servicex/dataset_group.py index 505e3425..d7d94f2a 100644 --- a/servicex/dataset_group.py +++ b/servicex/dataset_group.py @@ -30,7 +30,7 @@ from rich.progress import Progress -from servicex.query import Query +from servicex.query_core import Query from servicex.expandable_progress import ExpandableProgress from servicex.func_adl.func_adl_dataset import FuncADLQuery from servicex.models import TransformedResults, ResultFormat diff --git a/servicex/func_adl/func_adl_dataset.py b/servicex/func_adl/func_adl_dataset.py index e54cee24..372dd102 100644 --- a/servicex/func_adl/func_adl_dataset.py +++ b/servicex/func_adl/func_adl_dataset.py @@ -46,7 +46,7 @@ from func_adl import EventDataset, find_EventDataset from func_adl.object_stream import S from servicex.configuration import Configuration -from servicex.query import Query +from servicex.query_core import Query from servicex.func_adl.util import has_tuple from servicex.models import ResultFormat from servicex.query_cache import QueryCache diff --git a/servicex/python_dataset.py b/servicex/python_dataset.py index 3fee1819..5a36fef1 100644 --- a/servicex/python_dataset.py +++ b/servicex/python_dataset.py @@ -30,7 +30,7 @@ from base64 import b64encode from servicex.configuration import Configuration -from servicex.query import Query +from servicex.query_core import Query from servicex.models import ResultFormat from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter diff --git a/servicex/query/__init__.py b/servicex/query/__init__.py new file mode 100644 index 00000000..6ebc8d36 --- /dev/null +++ b/servicex/query/__init__.py @@ -0,0 +1,38 @@ +# Copyright (c) 2024, IRIS-HEP +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import sys +if sys.version_info < (3, 10): + from importlib_metadata import entry_points +else: + from importlib.metadata import entry_points + +plugins = entry_points(group='servicex.query') +for _ in plugins: + globals()[_.name] = _.load() diff --git a/servicex/query.py b/servicex/query_core.py similarity index 100% rename from servicex/query.py rename to servicex/query_core.py diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 34c2d161..c2286477 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -34,12 +34,7 @@ from servicex.models import ResultFormat, TransformStatus, TransformedResults from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter -from servicex.query import ( - GenericQuery, - QueryStringGenerator, - GenericQueryStringGenerator, - Query, -) +from servicex.query_core import GenericQuery, QueryStringGenerator, GenericQueryStringGenerator, Query from servicex.types import DID from servicex.python_dataset import PythonQuery from servicex.dataset_group import DatasetGroup diff --git a/servicex/uproot_raw/uproot_raw.py b/servicex/uproot_raw/uproot_raw.py index 4aaa9e80..bf4e2c0f 100644 --- a/servicex/uproot_raw/uproot_raw.py +++ b/servicex/uproot_raw/uproot_raw.py @@ -30,7 +30,7 @@ import pydantic from typing import List, Union, Mapping, Optional, get_args -from ..query import QueryStringGenerator +from ..query_core import QueryStringGenerator class TreeSubQuery(pydantic.BaseModel): diff --git a/tests/test_servicex_dataset.py b/tests/test_servicex_dataset.py index 07ca40ef..6c4bcbc4 100644 --- a/tests/test_servicex_dataset.py +++ b/tests/test_servicex_dataset.py @@ -39,7 +39,7 @@ from servicex.models import (TransformStatus, Status, ResultFile, ResultFormat, TransformRequest, TransformedResults) from servicex.query_cache import QueryCache -from servicex.query import ServiceXException +from servicex.query_core import ServiceXException from servicex.servicex_client import ServiceXClient from servicex.uproot_raw.uproot_raw import UprootRawQuery From 84a143fdf921d327c2b312cfb97c1baf0344ee30 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 01:50:38 +0000 Subject: [PATCH 02/10] Flake8 --- servicex/servicex_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index c2286477..89fa8f54 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -34,7 +34,8 @@ from servicex.models import ResultFormat, TransformStatus, TransformedResults from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter -from servicex.query_core import GenericQuery, QueryStringGenerator, GenericQueryStringGenerator, Query +from servicex.query_core import (GenericQuery, QueryStringGenerator, GenericQueryStringGenerator, + Query) from servicex.types import DID from servicex.python_dataset import PythonQuery from servicex.dataset_group import DatasetGroup From 8fecf30b3a0143499c2cdd2ffd3964058c65c614 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 02:08:12 +0000 Subject: [PATCH 03/10] Add a test for the imports --- tests/test_databinder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test_databinder.py b/tests/test_databinder.py index 57af4aed..fed76faa 100644 --- a/tests/test_databinder.py +++ b/tests/test_databinder.py @@ -412,3 +412,8 @@ def test_generic_query(codegen_list): # no codegen specified by generic class query = sx.generic_query(dataset_identifier=spec.Sample[0].RucioDID, query=spec.Sample[0].Query) + + +def test_entrypoint_import(): + """ This will check that we have at least the Python transformer defined in servicex.query """ + from servicex.query import Python # noqa \ No newline at end of file From 3972fde1aba93b8a23a147f16f7c6389f5961d0c Mon Sep 17 00:00:00 2001 From: KyungEon Choi Date: Wed, 19 Jun 2024 11:00:43 -0500 Subject: [PATCH 04/10] Add ruamel yaml load. Simplify deliver function --- servicex/servicex_client.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 89fa8f54..221e247b 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -46,6 +46,7 @@ T = TypeVar("T") logger = logging.getLogger(__name__) +yaml = YAML() def _load_ServiceXSpec( config: Union[ServiceXSpec, Mapping[str, Any], str, Path] @@ -72,7 +73,7 @@ def _load_ServiceXSpec( else: from importlib.metadata import entry_points - plugins = entry_points(group="servicex.queries") + plugins = entry_points(group="servicex.query") for _ in plugins: yaml.register_class(_.load()) From 721bae2a8096cb0a8637eb46648c79a2a79b4b22 Mon Sep 17 00:00:00 2001 From: KyungEon Choi Date: Mon, 24 Jun 2024 22:13:24 -0500 Subject: [PATCH 05/10] YAML interface for UprootRaw with examples --- examples/Uproot_UprootRaw_Dict.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/examples/Uproot_UprootRaw_Dict.py b/examples/Uproot_UprootRaw_Dict.py index dcc0cc72..fd933fb0 100644 --- a/examples/Uproot_UprootRaw_Dict.py +++ b/examples/Uproot_UprootRaw_Dict.py @@ -6,9 +6,6 @@ def uproot_uproot_raw_dict(): query = UprootRaw([{"treename": "reco", "filter_name": "el_pt_NOSYS"}]) spec = { - 'General': { - 'ServiceX': "servicex-uc-af" - }, 'Sample': [{ 'Name': "Uproot_PythonFunction_Dict", 'RucioDID': "user.mtost:user.mtost.singletop.p6026.Jun13", From 3b8ce0613961237cb01a64b75db3311914e05090 Mon Sep 17 00:00:00 2001 From: Ben Galewsky Date: Wed, 26 Jun 2024 16:14:15 -0500 Subject: [PATCH 06/10] Really fix funcADL Queries from text --- servicex/servicex_client.py | 1 - 1 file changed, 1 deletion(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 221e247b..3a28ef62 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -46,7 +46,6 @@ T = TypeVar("T") logger = logging.getLogger(__name__) -yaml = YAML() def _load_ServiceXSpec( config: Union[ServiceXSpec, Mapping[str, Any], str, Path] From 91cfa392825537035205ea8e2ce11afbe91e1682 Mon Sep 17 00:00:00 2001 From: KyungEon Choi Date: Thu, 27 Jun 2024 11:27:14 -0500 Subject: [PATCH 07/10] WIP - FuncADLQuery via set_provided_qastle --- servicex/servicex_client.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 3a28ef62..7e38e1af 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -177,15 +177,13 @@ def deliver( datasets = _build_datasets(config, config_path, servicex_name) - group = DatasetGroup(datasets) + # if config.General.Delivery == General.DeliveryEnum.SignedURLs: + # results = group.as_signed_urls() + # return _output_handler(config, results) - if config.General.Delivery == General.DeliveryEnum.SignedURLs: - results = group.as_signed_urls() - return _output_handler(config, results) - - elif config.General.Delivery == General.DeliveryEnum.LocalCache: - results = group.as_files() - return _output_handler(config, results) + # elif config.General.Delivery == General.DeliveryEnum.LocalCache: + # results = group.as_files() + # return _output_handler(config, results) class ServiceXClient: From 4635ae59511d9b132007b19376d3bc8c89ec69f7 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 01:46:46 +0000 Subject: [PATCH 08/10] Import queries into servicex.query on import --- servicex/servicex_client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 7e38e1af..0c5f49a3 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -34,8 +34,12 @@ from servicex.models import ResultFormat, TransformStatus, TransformedResults from servicex.query_cache import QueryCache from servicex.servicex_adapter import ServiceXAdapter -from servicex.query_core import (GenericQuery, QueryStringGenerator, GenericQueryStringGenerator, - Query) +from servicex.query_core import ( + GenericQuery, + QueryStringGenerator, + GenericQueryStringGenerator, + Query, +) from servicex.types import DID from servicex.python_dataset import PythonQuery from servicex.dataset_group import DatasetGroup From 55c882b6d90e9f8b999321a578bef0fc07ca25f1 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 20:05:45 +0000 Subject: [PATCH 09/10] Fix naming --- tests/test_databinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_databinder.py b/tests/test_databinder.py index fed76faa..e78a4672 100644 --- a/tests/test_databinder.py +++ b/tests/test_databinder.py @@ -416,4 +416,4 @@ def test_generic_query(codegen_list): def test_entrypoint_import(): """ This will check that we have at least the Python transformer defined in servicex.query """ - from servicex.query import Python # noqa \ No newline at end of file + from servicex.query import PythonFunction # noqa \ No newline at end of file From d9675854cb606838b89b5c5c1c58834d4478fd72 Mon Sep 17 00:00:00 2001 From: Peter Onyisi Date: Fri, 28 Jun 2024 20:42:28 +0000 Subject: [PATCH 10/10] Patch --- servicex/servicex_client.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/servicex/servicex_client.py b/servicex/servicex_client.py index 0c5f49a3..60e7c721 100644 --- a/servicex/servicex_client.py +++ b/servicex/servicex_client.py @@ -181,13 +181,15 @@ def deliver( datasets = _build_datasets(config, config_path, servicex_name) - # if config.General.Delivery == General.DeliveryEnum.SignedURLs: - # results = group.as_signed_urls() - # return _output_handler(config, results) + group = DatasetGroup(datasets) - # elif config.General.Delivery == General.DeliveryEnum.LocalCache: - # results = group.as_files() - # return _output_handler(config, results) + if config.General.Delivery == General.DeliveryEnum.SignedURLs: + results = group.as_signed_urls() + return _output_handler(config, results) + + elif config.General.Delivery == General.DeliveryEnum.LocalCache: + results = group.as_files() + return _output_handler(config, results) class ServiceXClient: