diff --git a/.github/workflows/deploy-config.json b/.github/workflows/deploy-config.json index f07c0e283..eddadeb2d 100644 --- a/.github/workflows/deploy-config.json +++ b/.github/workflows/deploy-config.json @@ -19,6 +19,11 @@ "image_name": "servicex-did-finder", "test_required": false }, + { + "dir_name": "did_finder_unzip", + "image_name": "servicex-did-finder-unzip", + "test_required": false + }, { "dir_name": "code_generator_funcadl_uproot", "image_name": "servicex_code_gen_func_adl_uproot", diff --git a/code_generator_python/Dockerfile b/code_generator_python/Dockerfile index fb5c60bf9..08e8da206 100644 --- a/code_generator_python/Dockerfile +++ b/code_generator_python/Dockerfile @@ -20,6 +20,7 @@ COPY boot.sh ./ COPY transformer_capabilities.json ./ COPY servicex/ ./servicex COPY scripts/from_text_to_zip.py . +COPY transformer_capabilities.json ./ RUN chmod +x boot.sh USER servicex diff --git a/code_generator_python/servicex/python_code_generator/python_translator.py b/code_generator_python/servicex/python_code_generator/python_translator.py index c336a11a0..31d6fa907 100644 --- a/code_generator_python/servicex/python_code_generator/python_translator.py +++ b/code_generator_python/servicex/python_code_generator/python_translator.py @@ -38,6 +38,7 @@ class PythonTranslator(CodeGenerator): def generate_code(self, query, cache_path: str): src = base64.b64decode(query).decode('ascii') + print("SRC", src) hash = "no-hash" query_file_path = os.path.join(cache_path, hash) @@ -59,6 +60,10 @@ def generate_code(self, query, cache_path: str): shutil.copyfile(capabilities_path, os.path.join(query_file_path, "transformer_capabilities.json")) + with open(os.path.join(query_file_path, 'generated_transformer.py'), 'w') as python_file: + python_file.write(src) + os.system("ls -lht " + query_file_path) os.system(f"cat {query_file_path}/generated_transformer.py") + return GeneratedFileResult(hash, query_file_path) diff --git a/code_generator_python/servicex/python_code_generator/unzip_translator.py b/code_generator_python/servicex/python_code_generator/unzip_translator.py new file mode 100644 index 000000000..44aebeb5a --- /dev/null +++ b/code_generator_python/servicex/python_code_generator/unzip_translator.py @@ -0,0 +1,13 @@ +def run_query(input_filenames=None, tree_name=None): + from stream_unzip import stream_unzip + import httpx + + def zipped_chunks(input_path): + # Iterable that yields the bytes of a zip file + with httpx.stream('GET', input_path) as r: + yield from r.iter_bytes(chunk_size=65536) + + for file_name, file_size, unzipped_chunks in stream_unzip(zipped_chunks(input_filenames[0])): + # unzipped_chunks must be iterated to completion or UnfinishedIterationError will be raised + for chunk in unzipped_chunks: + yield chunk, file_name diff --git a/code_generator_python/servicex/templates/transform_single_file.py b/code_generator_python/servicex/templates/transform_single_file.py index 6987b41da..dbd663767 100644 --- a/code_generator_python/servicex/templates/transform_single_file.py +++ b/code_generator_python/servicex/templates/transform_single_file.py @@ -8,6 +8,7 @@ import uproot instance = os.environ.get('INSTANCE_NAME', 'Unknown') default_tree_name = "servicex" +codegen_type = os.environ.get('CODEGEN_TYPE', 'default') def transform_single_file(file_path: str, output_path: Path, output_format: str): @@ -20,41 +21,58 @@ def transform_single_file(file_path: str, output_path: Path, output_format: str) try: stime = time.time() - awkward_array = generated_transformer.run_query(file_path) - total_events = ak.num(awkward_array, axis=0) + if codegen_type == 'default': + awkward_array = generated_transformer.run_query(file_path) + total_events = ak.num(awkward_array, axis=0) - ttime = time.time() + ttime = time.time() - if output_format == 'root-file': - etime = time.time() - with uproot.recreate(output_path) as writer: - writer[default_tree_name] = {field: awkward_array[field] for field in - awkward_array.fields} if awkward_array.fields \ - else awkward_array - wtime = time.time() + if output_format == 'root-file': + etime = time.time() + with uproot.recreate(output_path) as writer: + writer[default_tree_name] = {field: awkward_array[field] for field in + awkward_array.fields} if awkward_array.fields \ + else awkward_array + wtime = time.time() - else: - explode_records = bool(awkward_array.fields) - try: - arrow = ak.to_arrow_table(awkward_array, explode_records=explode_records) - except TypeError: - arrow = ak.to_arrow_table(ak.repartition(awkward_array, None), - explode_records=explode_records) + else: + explode_records = bool(awkward_array.fields) + try: + arrow = ak.to_arrow_table(awkward_array, explode_records=explode_records) + except TypeError: + arrow = ak.to_arrow_table(ak.repartition(awkward_array, None), + explode_records=explode_records) - etime = time.time() + etime = time.time() - writer = pq.ParquetWriter(output_path, arrow.schema) - writer.write_table(table=arrow) - writer.close() + writer = pq.ParquetWriter(output_path, arrow.schema) + writer.write_table(table=arrow) + writer.close() - wtime = time.time() + wtime = time.time() - output_size = os.stat(output_path).st_size - print(f'Detailed transformer times. query_time:{round(ttime - stime, 3)} ' - f'serialization: {round(etime - ttime, 3)} ' - f'writing: {round(wtime - etime, 3)}') + output_size = os.stat(output_path).st_size + print(f'Detailed transformer times. query_time:{round(ttime - stime, 3)} ' + f'serialization: {round(etime - ttime, 3)} ' + f'writing: {round(wtime - etime, 3)}') - print(f"Transform stats: Total Events: {total_events}, resulting file size {output_size}") + print(f"Transform stats: Total Events: {total_events}, \ + resulting file size {output_size}") + elif codegen_type == 'unzip': + folder_output_path = os.path.dirname(output_path) + for bytes, file_name in generated_transformer.run_query(file_path): + file_output_path = os.path.join(folder_output_path, file_name.decode('utf-8')) + print('File: ', file_output_path) + with open(file_output_path, 'ab') as f: + f.write(bytes) + total_events = 0 + output_size = os.stat(folder_output_path).st_size + elif codegen_type == 'pandas': + folder_output_path = os.path.dirname(output_path) + pd_data = generated_transformer.run_query(file_path) + pd_data.to_parquet(output_path) + total_events = 0 + output_size = os.stat(folder_output_path).st_size except Exception as error: mesg = f"Failed to transform input file {file_path}: {error}" print(mesg) diff --git a/code_generator_python/tests/test_python_translator.py b/code_generator_python/tests/test_python_translator.py index 24eab21ae..78f44234e 100644 --- a/code_generator_python/tests/test_python_translator.py +++ b/code_generator_python/tests/test_python_translator.py @@ -31,12 +31,13 @@ # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions are met: # + import base64 import os import tempfile from servicex.python_code_generator.python_translator import \ - PythonTranslator + PythonTranslator def test_generate_code(): diff --git a/code_generator_python/transformer_capabilities.json b/code_generator_python/transformer_capabilities.json index d94ffde4e..cf8720910 100644 --- a/code_generator_python/transformer_capabilities.json +++ b/code_generator_python/transformer_capabilities.json @@ -4,6 +4,6 @@ "limitations": "Would be good to note what isn't implemented", "file-formats": ["parquet", "root"], "stats-parser": "UprootStats", - "language": "python", + "language": "python3", "command": "/generated/transform_single_file.py" } \ No newline at end of file diff --git a/did_finder_rucio/scripts/did_finder.py b/did_finder_rucio/scripts/did_finder.py index f6616d6d5..83178cfc7 100644 --- a/did_finder_rucio/scripts/did_finder.py +++ b/did_finder_rucio/scripts/did_finder.py @@ -69,6 +69,7 @@ async def callback(did_name, info): request_id=info['request-id'] ) for file in lookup_request.lookup_files(): + logger.info(f"File: {file}") yield file start_did_finder('rucio', diff --git a/did_finder_unzip/Dockerfile b/did_finder_unzip/Dockerfile new file mode 100644 index 000000000..c3fd06981 --- /dev/null +++ b/did_finder_unzip/Dockerfile @@ -0,0 +1,24 @@ +FROM python:3 + +# Create app directory +WORKDIR /usr/src/app + +# for CA certificates +USER root +RUN mkdir -p /etc/grid-security/certificates /etc/grid-security/vomsdir + +ENV POETRY_VERSION=1.2.2 +RUN python3 -m pip install --upgrade pip + +RUN pip install poetry==$POETRY_VERSION +COPY pyproject.toml pyproject.toml +COPY poetry.lock poetry.lock + +RUN poetry config virtualenvs.create false && \ + poetry install --no-root --no-interaction --no-ansi + +COPY . . + +ENV X509_USER_PROXY /tmp/grid-security/x509up +ENV X509_CERT_DIR /etc/grid-security/certificates + diff --git a/did_finder_unzip/poetry.lock b/did_finder_unzip/poetry.lock new file mode 100644 index 000000000..d9df11e16 --- /dev/null +++ b/did_finder_unzip/poetry.lock @@ -0,0 +1,466 @@ +[[package]] +name = "certifi" +version = "2022.12.7" +description = "Python package for providing Mozilla's CA Bundle." +category = "main" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "charset-normalizer" +version = "3.1.0" +description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" +optional = false +python-versions = ">=3.7.0" + +[[package]] +name = "colorama" +version = "0.4.6" +description = "Cross-platform colored terminal text." +category = "dev" +optional = false +python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" + +[[package]] +name = "coverage" +version = "6.5.0" +description = "Code coverage measurement for Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +toml = ["tomli"] + +[[package]] +name = "exceptiongroup" +version = "1.1.1" +description = "Backport of PEP 654 (exception groups)" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.extras] +test = ["pytest (>=6)"] + +[[package]] +name = "flake8" +version = "5.0.4" +description = "the modular source code checker: pep8 pyflakes and co" +category = "dev" +optional = false +python-versions = ">=3.6.1" + +[package.dependencies] +mccabe = ">=0.7.0,<0.8.0" +pycodestyle = ">=2.9.0,<2.10.0" +pyflakes = ">=2.5.0,<2.6.0" + +[[package]] +name = "idna" +version = "3.4" +description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" +optional = false +python-versions = ">=3.5" + +[[package]] +name = "iniconfig" +version = "2.0.0" +description = "brain-dead simple config-ini parsing" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "make-it-sync" +version = "1.0.0" +description = "Create a sync version of an async function" +category = "main" +optional = false +python-versions = ">=3.6" + +[package.extras] +test = ["autopep8", "codecov", "coverage", "flake8", "pytest (>=3.9)", "pytest-asyncio", "pytest-cov", "pytest-mock", "twine", "wheel"] + +[[package]] +name = "mccabe" +version = "0.7.0" +description = "McCabe checker, plugin for flake8" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "minio" +version = "7.1.14" +description = "MinIO Python SDK for Amazon S3 Compatible Cloud Storage" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +certifi = "*" +urllib3 = "*" + +[[package]] +name = "packaging" +version = "23.1" +description = "Core utilities for Python packages" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pika" +version = "1.1.0" +description = "Pika Python AMQP Client Library" +category = "main" +optional = false +python-versions = "*" + +[package.extras] +tornado = ["tornado"] +twisted = ["twisted"] + +[[package]] +name = "pluggy" +version = "1.0.0" +description = "plugin and hook calling mechanisms for python" +category = "dev" +optional = false +python-versions = ">=3.6" + +[package.extras] +dev = ["pre-commit", "tox"] +testing = ["pytest", "pytest-benchmark"] + +[[package]] +name = "pycodestyle" +version = "2.9.1" +description = "Python style guide checker" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pyflakes" +version = "2.5.0" +description = "passive checker of Python programs" +category = "dev" +optional = false +python-versions = ">=3.6" + +[[package]] +name = "pymemcache" +version = "4.0.0" +description = "A comprehensive, fast, pure Python memcached client" +category = "main" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "pytest" +version = "7.3.0" +description = "pytest: simple powerful testing with Python" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +colorama = {version = "*", markers = "sys_platform == \"win32\""} +exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} +iniconfig = "*" +packaging = "*" +pluggy = ">=0.12,<2.0" +tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} + +[package.extras] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] + +[[package]] +name = "pytest-mock" +version = "3.10.0" +description = "Thin-wrapper around the mock package for easier use with pytest" +category = "dev" +optional = false +python-versions = ">=3.7" + +[package.dependencies] +pytest = ">=5.0" + +[package.extras] +dev = ["pre-commit", "pytest-asyncio", "tox"] + +[[package]] +name = "requests" +version = "2.28.2" +description = "Python HTTP for Humans." +category = "main" +optional = false +python-versions = ">=3.7, <4" + +[package.dependencies] +certifi = ">=2017.4.17" +charset-normalizer = ">=2,<4" +idna = ">=2.5,<4" +urllib3 = ">=1.21.1,<1.27" + +[package.extras] +socks = ["PySocks (>=1.5.6,!=1.5.7)"] +use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] + +[[package]] +name = "servicex-did-finder-lib" +version = "1.3.1" +description = "ServiceX DID Library Routines" +category = "main" +optional = false +python-versions = ">=3.6,<4.0" + +[package.dependencies] +make-it-sync = ">=1.0.0,<2.0.0" +pika = "1.1.0" +requests = ">=2.25.0,<3.0.0" + +[[package]] +name = "tomli" +version = "2.0.1" +description = "A lil' TOML parser" +category = "dev" +optional = false +python-versions = ">=3.7" + +[[package]] +name = "urllib3" +version = "1.26.15" +description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" + +[package.extras] +brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] +secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] +socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] + +[metadata] +lock-version = "1.1" +python-versions = "^3.10" +content-hash = "ac1ea47affc119eb5e31fcfcd5e3d945b4be07a37f91ac95e20fed5b6d2c2a59" + +[metadata.files] +certifi = [ + {file = "certifi-2022.12.7-py3-none-any.whl", hash = "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18"}, + {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, +] +charset-normalizer = [ + {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, + {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, + {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, + {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, + {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, + {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, + {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, +] +colorama = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] +coverage = [ + {file = "coverage-6.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ef8674b0ee8cc11e2d574e3e2998aea5df5ab242e012286824ea3c6970580e53"}, + {file = "coverage-6.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:784f53ebc9f3fd0e2a3f6a78b2be1bd1f5575d7863e10c6e12504f240fd06660"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b4a5be1748d538a710f87542f22c2cad22f80545a847ad91ce45e77417293eb4"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:83516205e254a0cb77d2d7bb3632ee019d93d9f4005de31dca0a8c3667d5bc04"}, + {file = "coverage-6.5.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:af4fffaffc4067232253715065e30c5a7ec6faac36f8fc8d6f64263b15f74db0"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:97117225cdd992a9c2a5515db1f66b59db634f59d0679ca1fa3fe8da32749cae"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:a1170fa54185845505fbfa672f1c1ab175446c887cce8212c44149581cf2d466"}, + {file = "coverage-6.5.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:11b990d520ea75e7ee8dcab5bc908072aaada194a794db9f6d7d5cfd19661e5a"}, + {file = "coverage-6.5.0-cp310-cp310-win32.whl", hash = "sha256:5dbec3b9095749390c09ab7c89d314727f18800060d8d24e87f01fb9cfb40b32"}, + {file = "coverage-6.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:59f53f1dc5b656cafb1badd0feb428c1e7bc19b867479ff72f7a9dd9b479f10e"}, + {file = "coverage-6.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4a5375e28c5191ac38cca59b38edd33ef4cc914732c916f2929029b4bfb50795"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c4ed2820d919351f4167e52425e096af41bfabacb1857186c1ea32ff9983ed75"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:33a7da4376d5977fbf0a8ed91c4dffaaa8dbf0ddbf4c8eea500a2486d8bc4d7b"}, + {file = "coverage-6.5.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a8fb6cf131ac4070c9c5a3e21de0f7dc5a0fbe8bc77c9456ced896c12fcdad91"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:a6b7d95969b8845250586f269e81e5dfdd8ff828ddeb8567a4a2eaa7313460c4"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:1ef221513e6f68b69ee9e159506d583d31aa3567e0ae84eaad9d6ec1107dddaa"}, + {file = "coverage-6.5.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cca4435eebea7962a52bdb216dec27215d0df64cf27fc1dd538415f5d2b9da6b"}, + {file = "coverage-6.5.0-cp311-cp311-win32.whl", hash = "sha256:98e8a10b7a314f454d9eff4216a9a94d143a7ee65018dd12442e898ee2310578"}, + {file = "coverage-6.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:bc8ef5e043a2af066fa8cbfc6e708d58017024dc4345a1f9757b329a249f041b"}, + {file = "coverage-6.5.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4433b90fae13f86fafff0b326453dd42fc9a639a0d9e4eec4d366436d1a41b6d"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f4f05d88d9a80ad3cac6244d36dd89a3c00abc16371769f1340101d3cb899fc3"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:94e2565443291bd778421856bc975d351738963071e9b8839ca1fc08b42d4bef"}, + {file = "coverage-6.5.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:027018943386e7b942fa832372ebc120155fd970837489896099f5cfa2890f79"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:255758a1e3b61db372ec2736c8e2a1fdfaf563977eedbdf131de003ca5779b7d"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:851cf4ff24062c6aec510a454b2584f6e998cada52d4cb58c5e233d07172e50c"}, + {file = "coverage-6.5.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:12adf310e4aafddc58afdb04d686795f33f4d7a6fa67a7a9d4ce7d6ae24d949f"}, + {file = "coverage-6.5.0-cp37-cp37m-win32.whl", hash = "sha256:b5604380f3415ba69de87a289a2b56687faa4fe04dbee0754bfcae433489316b"}, + {file = "coverage-6.5.0-cp37-cp37m-win_amd64.whl", hash = "sha256:4a8dbc1f0fbb2ae3de73eb0bdbb914180c7abfbf258e90b311dcd4f585d44bd2"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d900bb429fdfd7f511f868cedd03a6bbb142f3f9118c09b99ef8dc9bf9643c3c"}, + {file = "coverage-6.5.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:2198ea6fc548de52adc826f62cb18554caedfb1d26548c1b7c88d8f7faa8f6ba"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c4459b3de97b75e3bd6b7d4b7f0db13f17f504f3d13e2a7c623786289dd670e"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:20c8ac5386253717e5ccc827caad43ed66fea0efe255727b1053a8154d952398"}, + {file = "coverage-6.5.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b07130585d54fe8dff3d97b93b0e20290de974dc8177c320aeaf23459219c0b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:dbdb91cd8c048c2b09eb17713b0c12a54fbd587d79adcebad543bc0cd9a3410b"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:de3001a203182842a4630e7b8d1a2c7c07ec1b45d3084a83d5d227a3806f530f"}, + {file = "coverage-6.5.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:e07f4a4a9b41583d6eabec04f8b68076ab3cd44c20bd29332c6572dda36f372e"}, + {file = "coverage-6.5.0-cp38-cp38-win32.whl", hash = "sha256:6d4817234349a80dbf03640cec6109cd90cba068330703fa65ddf56b60223a6d"}, + {file = "coverage-6.5.0-cp38-cp38-win_amd64.whl", hash = "sha256:7ccf362abd726b0410bf8911c31fbf97f09f8f1061f8c1cf03dfc4b6372848f6"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:633713d70ad6bfc49b34ead4060531658dc6dfc9b3eb7d8a716d5873377ab745"}, + {file = "coverage-6.5.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:95203854f974e07af96358c0b261f1048d8e1083f2de9b1c565e1be4a3a48cfc"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9023e237f4c02ff739581ef35969c3739445fb059b060ca51771e69101efffe"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:265de0fa6778d07de30bcf4d9dc471c3dc4314a23a3c6603d356a3c9abc2dfcf"}, + {file = "coverage-6.5.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f830ed581b45b82451a40faabb89c84e1a998124ee4212d440e9c6cf70083e5"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:7b6be138d61e458e18d8e6ddcddd36dd96215edfe5f1168de0b1b32635839b62"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:42eafe6778551cf006a7c43153af1211c3aaab658d4d66fa5fcc021613d02518"}, + {file = "coverage-6.5.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:723e8130d4ecc8f56e9a611e73b31219595baa3bb252d539206f7bbbab6ffc1f"}, + {file = "coverage-6.5.0-cp39-cp39-win32.whl", hash = "sha256:d9ecf0829c6a62b9b573c7bb6d4dcd6ba8b6f80be9ba4fc7ed50bf4ac9aecd72"}, + {file = "coverage-6.5.0-cp39-cp39-win_amd64.whl", hash = "sha256:fc2af30ed0d5ae0b1abdb4ebdce598eafd5b35397d4d75deb341a614d333d987"}, + {file = "coverage-6.5.0-pp36.pp37.pp38-none-any.whl", hash = "sha256:1431986dac3923c5945271f169f59c45b8802a114c8f548d611f2015133df77a"}, + {file = "coverage-6.5.0.tar.gz", hash = "sha256:f642e90754ee3e06b0e7e51bce3379590e76b7f76b708e1a71ff043f87025c84"}, +] +exceptiongroup = [ + {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, + {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, +] +flake8 = [ + {file = "flake8-5.0.4-py2.py3-none-any.whl", hash = "sha256:7a1cf6b73744f5806ab95e526f6f0d8c01c66d7bbe349562d22dfca20610b248"}, + {file = "flake8-5.0.4.tar.gz", hash = "sha256:6fbe320aad8d6b95cec8b8e47bc933004678dc63095be98528b7bdd2a9f510db"}, +] +idna = [ + {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, + {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, +] +iniconfig = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] +make-it-sync = [ + {file = "make-it-sync-1.0.0.tar.gz", hash = "sha256:26f5dd7d066295d14eee934326228f9d05cc42267c795e0f35a8966ba54b8b4c"}, + {file = "make_it_sync-1.0.0-py3-none-any.whl", hash = "sha256:e06808ce449f765b0f86a2badc2bbe72a3f6a3d4793472b2b8c221bbdba66511"}, +] +mccabe = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] +minio = [ + {file = "minio-7.1.14-py3-none-any.whl", hash = "sha256:ee5cbd4ba73f71b73555209f3411d5da3e9742bbe3fd038c362042d6d2527256"}, + {file = "minio-7.1.14.tar.gz", hash = "sha256:230faf1d0db1c3ce09aef86b0eb38e994a64769e34dad4b77febf59bc90d8ab0"}, +] +packaging = [ + {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, + {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, +] +pika = [ + {file = "pika-1.1.0-py2.py3-none-any.whl", hash = "sha256:4e1a1a6585a41b2341992ec32aadb7a919d649eb82904fd8e4a4e0871c8cf3af"}, + {file = "pika-1.1.0.tar.gz", hash = "sha256:9fa76ba4b65034b878b2b8de90ff8660a59d925b087c5bb88f8fdbb4b64a1dbf"}, +] +pluggy = [ + {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, + {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, +] +pycodestyle = [ + {file = "pycodestyle-2.9.1-py2.py3-none-any.whl", hash = "sha256:d1735fc58b418fd7c5f658d28d943854f8a849b01a5d0a1e6f3f3fdd0166804b"}, + {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, +] +pyflakes = [ + {file = "pyflakes-2.5.0-py2.py3-none-any.whl", hash = "sha256:4579f67d887f804e67edb544428f264b7b24f435b263c4614f384135cea553d2"}, + {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, +] +pymemcache = [ + {file = "pymemcache-4.0.0-py2.py3-none-any.whl", hash = "sha256:f507bc20e0dc8d562f8df9d872107a278df049fa496805c1431b926f3ddd0eab"}, + {file = "pymemcache-4.0.0.tar.gz", hash = "sha256:27bf9bd1bbc1e20f83633208620d56de50f14185055e49504f4f5e94e94aff94"}, +] +pytest = [ + {file = "pytest-7.3.0-py3-none-any.whl", hash = "sha256:933051fa1bfbd38a21e73c3960cebdad4cf59483ddba7696c48509727e17f201"}, + {file = "pytest-7.3.0.tar.gz", hash = "sha256:58ecc27ebf0ea643ebfdf7fb1249335da761a00c9f955bcd922349bcb68ee57d"}, +] +pytest-mock = [ + {file = "pytest-mock-3.10.0.tar.gz", hash = "sha256:fbbdb085ef7c252a326fd8cdcac0aa3b1333d8811f131bdcc701002e1be7ed4f"}, + {file = "pytest_mock-3.10.0-py3-none-any.whl", hash = "sha256:f4c973eeae0282963eb293eb173ce91b091a79c1334455acfac9ddee8a1c784b"}, +] +requests = [ + {file = "requests-2.28.2-py3-none-any.whl", hash = "sha256:64299f4909223da747622c030b781c0d7811e359c37124b4bd368fb8c6518baa"}, + {file = "requests-2.28.2.tar.gz", hash = "sha256:98b1b2782e3c6c4904938b84c0eb932721069dfdb9134313beff7c83c2df24bf"}, +] +servicex-did-finder-lib = [ + {file = "servicex_did_finder_lib-1.3.1-py3-none-any.whl", hash = "sha256:795f00b718e7e6836f84f5303addb74281851393c362f784492b618346714b42"}, + {file = "servicex_did_finder_lib-1.3.1.tar.gz", hash = "sha256:34fc4e46850952e3ff6dc536e678dfef483f10b1bdc739b6f634e8b9fc42947c"}, +] +tomli = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] +urllib3 = [ + {file = "urllib3-1.26.15-py2.py3-none-any.whl", hash = "sha256:aa751d169e23c7479ce47a0cb0da579e3ede798f994f5816a74e4f4500dcea42"}, + {file = "urllib3-1.26.15.tar.gz", hash = "sha256:8a388717b9476f934a21484e8c8e61875ab60644d29b9b39e11e4b9dc1c6b305"}, +] diff --git a/did_finder_unzip/pyproject.toml b/did_finder_unzip/pyproject.toml new file mode 100644 index 000000000..7f3638473 --- /dev/null +++ b/did_finder_unzip/pyproject.toml @@ -0,0 +1,24 @@ +[tool.poetry] +name = "did-finder-unzip" +version = "0.1.0" +description = "" +authors = ["Prajwal Shriram"] +readme = "README.md" +packages = [{include = "did_finder_unzip"}] + +[tool.poetry.dependencies] +python = "^3.10" +pika = "1.1.0" +pymemcache = "^4.0.0" +minio = "^7.1.14" +servicex-did-finder-lib = "^1.3.1" + +[build-system] +requires = ["poetry-core"] +build-backend = "poetry.core.masonry.api" + +[tool.poetry.group.test.dependencies] +pytest = "^7.2.0" +coverage = "^6.5.0" +pytest-mock = "^3.10.0" +flake8 = "^5.0.4" \ No newline at end of file diff --git a/did_finder_unzip/runme.sh b/did_finder_unzip/runme.sh new file mode 100755 index 000000000..cf87d6577 --- /dev/null +++ b/did_finder_unzip/runme.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +/usr/src/app/proxy-exporter.sh & + +#while true; do +# date +# ls ${X509_USER_PROXY} +# RESULT=$? +# if [ $RESULT -eq 0 ]; then +# break +# fi +# echo "INFO $INSTANCE_NAME did-finder none Waiting for the proxy." +# sleep 5 +#done + +export PYTHONPATH=. + +# Assume $REPORT_LOGICAL_FILES is set to --report-logical-files to activate +echo "----------->$PYTHONPATH" +ls -lht $PYTHONPATH +python3 scripts/did_finder.py --rabbit-uri $RMQ_URI $REPORT_LOGICAL_FILES + diff --git a/did_finder_unzip/scripts/did_finder.py b/did_finder_unzip/scripts/did_finder.py new file mode 100644 index 000000000..2aa30a91c --- /dev/null +++ b/did_finder_unzip/scripts/did_finder.py @@ -0,0 +1,89 @@ +# Copyright (c) 2019, IRIS-HEP +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import argparse +import logging +import os +import json + +from minio import Minio +from servicex_did_finder_lib import add_did_finder_cnd_arguments, start_did_finder + + +def run_rucio_finder(): + '''Run the rucio finder + ''' + logger = logging.getLogger() + + # Parse the command line arguments + parser = argparse.ArgumentParser() + parser.add_argument("--report-logical-files", action="store_true") + add_did_finder_cnd_arguments(parser) + + args = parser.parse_args() + + logger.info("ServiceX DID Finder starting up. ") + print(args) + if args.report_logical_files: + logger.info("---- DID Finder Only Returning Logical Names, not replicas -----") + + # Run the DID Finder + try: + logger.info('Starting rucio DID finder') + minio_url = os.environ.get("MINIO_URL") + minio_secret_key = os.environ.get("MINIO_SECRET_KEY") + minio_access_key = os.environ.get("MINIO_ACCESS_KEY") + use_https = False + + async def callback(did_name, info): + minio_client = Minio(endpoint=minio_url, access_key=minio_access_key, + secret_key=minio_secret_key, secure=use_https) + for file in minio_client.list_objects(did_name): + url = minio_client.get_presigned_url( + "GET", + file.bucket_name, + file.object_name + ) + return_obj = { + 'adler32': 0, + 'file_size': 0, + 'file_events': 0, + 'paths': [url] + } + yield return_obj + + start_did_finder('unzip', + callback, + parsed_args=args) + + finally: + logger.info('Done running rucio DID finder') + + +if __name__ == "__main__": + run_rucio_finder() diff --git a/helm/servicex/local-dev-values.yaml b/helm/servicex/local-dev-values.yaml index c3f5b271b..8aec1d4f2 100644 --- a/helm/servicex/local-dev-values.yaml +++ b/helm/servicex/local-dev-values.yaml @@ -1,23 +1,29 @@ noCerts: false didFinder: + unzip: + enabled: true + tag: pandas-tf + pullPolicy: Always + image: sslhep/servicex-did-finder-unzip rucio: enabled: true - tag: sidecar-monorepo + tag: pandas-tf pullPolicy: Always - + image: sslhep/servicex-did-finder CERNOpenData: enabled: false YT: enabled: false app: - tag: sidecar-monorepo + tag: pandas-tf pullPolicy: IfNotPresent validateTransformerImage: false + checksImage: shriram192/checks transformer: autoscalerEnabled: false - sidecarTag: sidecar-monorepo + sidecarTag: pandas-tf sidecarPullPolicy: IfNotPresent # language: python @@ -29,12 +35,18 @@ transformer: scienceContainerPullPolicy: IfNotPresent codeGen: - #image: sslhep/servicex_code_gen_func_adl_uproot - image: sslhep/servicex_code_gen_atlas_xaod - tag: sidecar-monorepo - pullPolicy: IfNotPresent - defaultScienceContainerImage: sslhep/servicex_func_adl_xaod_transformer - defaultScienceContainerTag: sidecar-monorepo + python: + enabled: true + image: sslhep/servicex_code_gen_python + pullPolicy: Always + tag: pandas-tf + defaultScienceContainerImage: sslhep/servicex_func_adl_uproot_transformer + defaultScienceContainerTag: pandas-tf + type: unzip + uproot: + enabled: false + cms: + enabled: false objectStore: enabled: true diff --git a/helm/servicex/templates/app/configmap.yaml b/helm/servicex/templates/app/configmap.yaml index 29a6e7550..5ebe0f825 100644 --- a/helm/servicex/templates/app/configmap.yaml +++ b/helm/servicex/templates/app/configmap.yaml @@ -81,7 +81,7 @@ data: TRANSFORMER_MANAGER_ENABLED = True - TRANSFORMER_CACHE_PREFIX = {{ .Values.transformer.cachePrefix }} + TRANSFORMER_CACHE_PREFIX = "{{ .Values.transformer.cachePrefix }}" TRANSFORMER_AUTOSCALE_ENABLED = {{- ternary "True" "False" .Values.transformer.autoscaler.enabled }} TRANSFORMER_CPU_LIMIT = {{ .Values.transformer.cpuLimit }} TRANSFORMER_CPU_SCALE_THRESHOLD = {{ .Values.transformer.autoscaler.cpuScaleThreshold }} @@ -149,7 +149,8 @@ data: {{ if .Values.codeGen }} {{- $code_gen_service_urls := list }} - {{- $code_gen_images := list }} + {{- $code_gen_images := list }} + {{- $code_gen_types := list }} {{- $transformer_images := list }} {{- range $codeGenName, $v := .Values.codeGen }} {{- if .enabled }} @@ -159,10 +160,14 @@ data: {{- $codeGenImage := printf "%s:%s" .image .tag }} {{- $dictItem := printf "\"%s\":\"%s\"" $codeGenName $codeGenImage }} {{- $code_gen_images = append $code_gen_images $dictItem }} + {{- $type := .type }} + {{- $dictItem := printf "\"%s\":\"%s\"" $codeGenName $type }} + {{- $code_gen_types = append $code_gen_types $dictItem }} {{ end }} {{- end }} CODE_GEN_SERVICE_URLS = { {{ join "," $code_gen_service_urls }} } CODE_GEN_IMAGES = { {{ join "," $code_gen_images }} } + CODE_GEN_TYPES = { {{ join "," $code_gen_types }} } {{- end }} {{- $didFinders := list }} @@ -174,7 +179,10 @@ data: DID_RUCIO_FINDER_TAG = '{{ .Values.didFinder.rucio.tag }}' {{- $didFinders = append $didFinders "rucio" }} {{- end }} - + {{- if .Values.didFinder.unzip.enabled }} + DID_UNZIP_FINDER_TAG = '{{ .Values.didFinder.unzip.tag }}' + {{- $didFinders = append $didFinders "unzip" }} + {{- end }} {{- $defaultScheme := .Values.app.defaultDIDFinderScheme }} {{- if and (not $defaultScheme) .Values.didFinder.rucio.enabled }} {{- $defaultScheme = "rucio" }} @@ -182,6 +190,9 @@ data: {{- if and (not $defaultScheme) .Values.didFinder.CERNOpenData.enabled }} {{- $defaultScheme = "cernopendata" }} {{- end }} + {{- if and (not $defaultScheme) .Values.didFinder.unzip.enabled }} + {{- $defaultScheme = "unzip" }} + {{- end }} DID_FINDER_DEFAULT_SCHEME = '{{ $defaultScheme }}' VALID_DID_SCHEMES = [ "{{ join "\",\"" $didFinders }}" ] diff --git a/helm/servicex/templates/did_finder_unzip/deployment.yaml b/helm/servicex/templates/did_finder_unzip/deployment.yaml new file mode 100644 index 000000000..f80b2c61a --- /dev/null +++ b/helm/servicex/templates/did_finder_unzip/deployment.yaml @@ -0,0 +1,77 @@ +--- +{{ if .Values.didFinder.unzip.enabled}} + {{- if .Values.noCerts }} + {{- fail "Unzip DID Finder requires x509 Certs to be installed" }} + {{- end }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ .Release.Name }}-did-finder-unzip +spec: + replicas: 1 + selector: + matchLabels: + app: {{ .Release.Name }}-did-finder-unzip + template: + metadata: + labels: + app: {{ .Release.Name }}-did-finder-unzip + spec: + containers: + - name: did-finder + image: {{ .Values.didFinder.unzip.image }}:{{ .Values.didFinder.unzip.tag }} + command: ["/usr/src/app/runme.sh"] + imagePullPolicy: {{ .Values.didFinder.unzip.pullPolicy }} + env: + {{- if .Values.secrets }} + - name: RMQ_PASS + valueFrom: + secretKeyRef: + name: {{ .Values.secrets }} + key: rabbitmq-password + - name: RMQ_URI + value: amqp://user:$(RMQ_PASS)@{{ .Release.Name }}-rabbitmq:5672/?heartbeat=9000 + {{- else }} + - name: RMQ_URI + value: amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/?heartbeat=9000 + {{- end }} + {{- if ((.Values.didFinder.unzip.memcache).enabled) }} + - name: MEMCACHE + value: "True" + - name: MEMCACHE_TTL + value: "{{ .Values.didFinder.unzip.memcache.ttl }}" + {{- end }} + - name: MINIO_URL + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-unzip-config + key: minio_url + - name: MINIO_ACCESS_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-unzip-config + key: minio_access_key + - name: MINIO_SECRET_KEY + valueFrom: + configMapKeyRef: + name: {{ .Release.Name }}-unzip-config + key: minio_secret_key + volumeMounts: + - name: x509-secret + mountPath: /etc/grid-security-ro + readOnly: true + - name: unzip-cfg + mountPath: /opt/unzip/etc/ + {{- if ((.Values.didFinder.unzip.memcache).enabled) }} + - name: memcache + image: {{ .Values.didFinder.rucio.memcache.image }}:{{ .Values.didFinder.rucio.memcache.tag }} + {{- end }} + volumes: + - name: x509-secret + secret: + defaultMode: 292 + secretName: {{ .Release.Name }}-x509-proxy + - name: unzip-cfg + configMap: + name: {{ .Release.Name }}-unzip-config +{{- end }} diff --git a/helm/servicex/templates/did_finder_unzip/unzip-configmap.yaml b/helm/servicex/templates/did_finder_unzip/unzip-configmap.yaml new file mode 100644 index 000000000..ad72cab68 --- /dev/null +++ b/helm/servicex/templates/did_finder_unzip/unzip-configmap.yaml @@ -0,0 +1,31 @@ +{{ if .Values.didFinder.unzip.enabled}} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ .Release.Name }}-unzip-config + labels: + heritage: {{ .Release.Service }} + release: {{ .Release.Name }} + chart: {{ .Chart.Name }}-{{ .Chart.Version }} + app: {{ .Release.Name }} +data: + unzip.cfg: | + [client] + auth_type = x509_proxy + ca_cert = /etc/pki/tls/certs/ca-bundle.crt + account = {{ .Values.gridAccount }} + client_x509_proxy = $X509_USER_PROXY + request_retries = 3 + + [policy] + permission = {{ .Values.x509Secrets.vomsOrg }} + schema = {{ .Values.x509Secrets.vomsOrg }} + lfn2pfn_algorithm_default = hash + {{ if not .Values.objectStore.internal }} + minio_url: "{{ .Values.objectStore.publicURL }}" + {{ else }} + minio_url: "{{ .Release.Name }}-minio:{{ .Values.minio.service.ports.api }}" + {{ end }} + minio_access_key: "{{ .Values.minio.auth.rootUser }}" + minio_secret_key: "{{ .Values.minio.auth.rootPassword }}" +{{ end }} diff --git a/servicex_app/docker-dev.conf b/servicex_app/docker-dev.conf index 47ff99ad5..7f5c780a4 100644 --- a/servicex_app/docker-dev.conf +++ b/servicex_app/docker-dev.conf @@ -49,5 +49,12 @@ CODE_GEN_IMAGES = { 'uproot': 'sslhep/servicex_code_gen_func_adl_uproot:develop' } +CODE_GEN_TYPES = { + 'atlasxaod': 'default', + 'cms': 'default', + 'python':'unzip', + 'uproot': 'default' +} + DID_FINDER_DEFAULT_SCHEME = 'rucio' VALID_DID_SCHEMES = ['rucio'] \ No newline at end of file diff --git a/servicex_app/migrations/versions/v_1_1_5.py b/servicex_app/migrations/versions/v_1_1_5.py new file mode 100644 index 000000000..988c7c1f4 --- /dev/null +++ b/servicex_app/migrations/versions/v_1_1_5.py @@ -0,0 +1,25 @@ +"""SUpport unzip transformer. + +Revision ID: v1_1_5 +Revises: v1_1_4 +Create Date: 2023-04-10 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = 'v1_1_5' +down_revision = 'v1_1_4' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('requests', sa.Column("codegen_type", sa.String(256), nullable=True)) + + +def downgrade(): + op.drop_column('requests', 'codegen_type') + diff --git a/servicex_app/servicex/lookup_result_processor.py b/servicex_app/servicex/lookup_result_processor.py index bf2083d2d..76f742889 100644 --- a/servicex_app/servicex/lookup_result_processor.py +++ b/servicex_app/servicex/lookup_result_processor.py @@ -40,7 +40,6 @@ def add_file_to_dataset(self, submitted_request, dataset_file): # dataset_file.save_to_db() TransformRequest.add_a_file(request_id) - transform_request = { 'request-id': request_id, 'file-id': dataset_file.id, @@ -51,9 +50,10 @@ def add_file_to_dataset(self, submitted_request, dataset_file): "servicex/internal/transformation/" + request_id, "chunk-size": "1000", "result-destination": submitted_request.result_destination, - "result-format": submitted_request.result_format + "result-format": submitted_request.result_format, + "codegen-type": submitted_request.codegen_type } - + print(json.dumps(transform_request)) self.rabbitmq_adaptor.basic_publish(exchange='transformation_requests', routing_key=request_id, body=json.dumps(transform_request)) diff --git a/servicex_app/servicex/models.py b/servicex_app/servicex/models.py index 7acb20d9d..80b36b4c6 100644 --- a/servicex_app/servicex/models.py +++ b/servicex_app/servicex/models.py @@ -175,6 +175,7 @@ class TransformRequest(db.Model): code_gen_image = db.Column(db.String(256), nullable=True) transformer_language = db.Column(db.String(256), nullable=True) transformer_command = db.Column(db.String(256), nullable=True) + codegen_type = db.Column(db.String(256), nullable=True) def save_to_db(self): db.session.add(self) @@ -203,7 +204,8 @@ def to_json(self): 'files-failed': self.files_failed, 'files-remaining': self.files_remaining, 'submit-time': str(self.submit_time.strftime(iso_fmt)), - 'finish-time': str(self.finish_time) + 'finish-time': str(self.finish_time), + 'codegen-type': str(self.codegen_type) } if self.finish_time is not None: result_obj['finish-time'] = str(self.finish_time.strftime(iso_fmt)) diff --git a/servicex_app/servicex/resources/internal/transform_start.py b/servicex_app/servicex/resources/internal/transform_start.py index 59ca841fe..5c0a3f25a 100644 --- a/servicex_app/servicex/resources/internal/transform_start.py +++ b/servicex_app/servicex/resources/internal/transform_start.py @@ -52,7 +52,8 @@ def start_transformers(cls, transformer_manager: TransformerManager, result_destination=request_rec.result_destination, result_format=request_rec.result_format, transformer_language=request_rec.transformer_language, - transformer_command=request_rec.transformer_command + transformer_command=request_rec.transformer_command, + codegen_type=request_rec.codegen_type ) @classmethod diff --git a/servicex_app/servicex/resources/transformation/submit.py b/servicex_app/servicex/resources/transformation/submit.py index fc27d5e6b..6bbbb80d4 100644 --- a/servicex_app/servicex/resources/transformation/submit.py +++ b/servicex_app/servicex/resources/transformation/submit.py @@ -101,6 +101,7 @@ def post(self): did = args.get("did") file_list = args.get("file-list") user_codegen_name = args.get("codegen") + codegen_type = config["CODE_GEN_TYPES"].get(user_codegen_name) code_gen_image_name = config['CODE_GEN_IMAGES'].get(user_codegen_name, None) @@ -143,7 +144,8 @@ def post(self): workflow_name=_workflow_name(args), status='Submitted', app_version=self._get_app_version(), - code_gen_image=code_gen_image_name + code_gen_image=code_gen_image_name, + codegen_type=codegen_type ) # If we are doing the xaod_cpp workflow, then the first thing to do is make @@ -203,6 +205,8 @@ def post(self): ) } + print("DID_REQUEST", json.dumps(did_request)) + print("parsed DID", parsed_did.microservice_queue) self.rabbitmq_adaptor.basic_publish(exchange='', routing_key=parsed_did.microservice_queue, body=json.dumps(did_request)) @@ -217,6 +221,7 @@ def post(self): request_rec, file_record ) + print("Added file to dataset") self.lookup_result_processor.report_fileset_complete( request_rec, diff --git a/servicex_app/servicex/transformer_manager.py b/servicex_app/servicex/transformer_manager.py index 556b8a6a5..37aa3c8fb 100644 --- a/servicex_app/servicex/transformer_manager.py +++ b/servicex_app/servicex/transformer_manager.py @@ -50,7 +50,8 @@ def __init__(self, manager_mode): @staticmethod def create_job_object(request_id, image, rabbitmq_uri, workers, result_destination, result_format, x509_secret, - generated_code_cm, transformer_language, transformer_command): + generated_code_cm, transformer_language, transformer_command, + codegen_type="default"): volume_mounts = [] volumes = [] @@ -99,6 +100,7 @@ def create_job_object(request_id, image, rabbitmq_uri, workers, # Compute Environment Vars env = [client.V1EnvVar(name="BASH_ENV", value="/servicex/.bashrc")] + env = [client.V1EnvVar(name="CODEGEN_TYPE", value=codegen_type)] # provide pods with level and logging server info env += [ @@ -313,13 +315,14 @@ def _create_hpa(api_instance, hpa, namespace): def launch_transformer_jobs(self, image, request_id, workers, rabbitmq_uri, namespace, x509_secret, generated_code_cm, result_destination, result_format, transformer_language, - transformer_command + transformer_command, codegen_type ): api_v1 = client.AppsV1Api() job = self.create_job_object(request_id, image, rabbitmq_uri, workers, result_destination, result_format, x509_secret, generated_code_cm, - transformer_language, transformer_command) + transformer_language, transformer_command, + codegen_type) self._create_job(api_v1, job, namespace) diff --git a/servicex_app/tests/resource_test_base.py b/servicex_app/tests/resource_test_base.py index 1772ccab5..a5d8ad721 100644 --- a/servicex_app/tests/resource_test_base.py +++ b/servicex_app/tests/resource_test_base.py @@ -88,6 +88,12 @@ def _app_config(): 'cms': 'sslhep/servicex_code_gen_cms_aod:develop', 'python': 'sslhep/servicex_code_gen_python:develop', 'uproot': 'sslhep/servicex_code_gen_func_adl_uproot:develop' + }, + 'CODE_GEN_TYPES': { + 'atlasxaod': 'default', + 'cms': 'default', + 'python': 'unzip', + 'uproot': 'default' } } diff --git a/servicex_app/tests/resources/internal/test_transform_start.py b/servicex_app/tests/resources/internal/test_transform_start.py index 93e994e66..dbf502aca 100644 --- a/servicex_app/tests/resources/internal/test_transform_start.py +++ b/servicex_app/tests/resources/internal/test_transform_start.py @@ -69,7 +69,8 @@ def test_transform_start(self, mocker): result_format='arrow', transformer_command="echo", transformer_language="scala", - x509_secret='my-x509-secret') + x509_secret='my-x509-secret', + codegen_type=None) mock_request.save_to_db.assert_called() def test_transform_start_no_kubernetes(self, mocker, mock_rabbit_adaptor): diff --git a/servicex_app/tests/resources/transformation/test_submit.py b/servicex_app/tests/resources/transformation/test_submit.py index 6dc8e190b..77ae96416 100644 --- a/servicex_app/tests/resources/transformation/test_submit.py +++ b/servicex_app/tests/resources/transformation/test_submit.py @@ -294,7 +294,8 @@ def test_submit_transformation_file_list(self, mocker): result_format=submitted_request.result_format, transformer_command=None, transformer_language=None, - x509_secret='my-x509-secret') + x509_secret='my-x509-secret', + codegen_type="default") def test_submit_transformation_request_bad_image( self, mocker, mock_docker_repo_adapter diff --git a/servicex_app/tests/test.config b/servicex_app/tests/test.config index 87c3cbd78..c4c1241a9 100644 --- a/servicex_app/tests/test.config +++ b/servicex_app/tests/test.config @@ -23,3 +23,9 @@ CODE_GEN_IMAGES = { 'uproot': 'sslhep/servicex_code_gen_func_adl_uproot:develop' } +CODE_GEN_TYPES = { + 'atlasxaod': 'default', + 'cms': 'default', + 'python':'unzip', + 'uproot': 'default' +} diff --git a/servicex_app/tests/test_lookup_result_processor.py b/servicex_app/tests/test_lookup_result_processor.py index 65759a58e..d8f32df60 100644 --- a/servicex_app/tests/test_lookup_result_processor.py +++ b/servicex_app/tests/test_lookup_result_processor.py @@ -69,7 +69,8 @@ def test_add_file_to_dataset(self, mocker, mock_rabbit_adaptor): "http://cern.analysis.ch:5000/servicex/internal/transformation/BR549", "chunk-size": "1000", 'result-destination': 'object-store', - "result-format": "arrow" + "result-format": "arrow", + "codegen-type": None })) def test_report_fileset_complete(self, mocker, mock_rabbit_adaptor): diff --git a/servicex_app/tests/test_transformer_manager.py b/servicex_app/tests/test_transformer_manager.py index d4e70e908..e5f5f32e1 100644 --- a/servicex_app/tests/test_transformer_manager.py +++ b/servicex_app/tests/test_transformer_manager.py @@ -111,7 +111,7 @@ def test_launch_transformer_jobs(self, mocker): image='sslhep/servicex-transformer:pytest', request_id='1234', workers=17, rabbitmq_uri='ampq://test.com', namespace='my-ns', result_destination='object-store', result_format='arrow', x509_secret='x509', - generated_code_cm=None) + generated_code_cm=None, codegen_type="default") called_deployment = mock_api.mock_calls[1][2]['body'] assert called_deployment.spec.replicas == cfg['TRANSFORMER_MIN_REPLICAS'] assert len(called_deployment.spec.template.spec.containers) == 2 @@ -181,7 +181,8 @@ def test_launch_transformer_jobs_no_autoscaler(self, mocker): rabbitmq_uri='ampq://test.com', namespace='my-ns', result_destination='object-store', result_format='arrow', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_deployment = mock_api.mock_calls[1][2]['body'] assert called_deployment.spec.replicas == 17 @@ -224,7 +225,8 @@ def test_launch_transformer_with_hostpath(self, mocker): rabbitmq_uri='ampq://test.com', namespace='my-ns', result_destination='object-store', result_format='arrow', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] @@ -270,7 +272,8 @@ def test_launch_transformer_jobs_with_generated_code(self, mocker): result_destination='object-store', result_format='parquet', x509_secret='x509', generated_code_cm="my-config-map", - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] container = called_job.spec.template.spec.containers[0] @@ -317,7 +320,8 @@ def test_launch_transformer_jobs_with_object_store(self, mocker): result_destination='object-store', result_format='parquet', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] container = called_job.spec.template.spec.containers[0] @@ -366,7 +370,8 @@ def test_launch_transformer_jobs_with_secure_object_store(self, mocker): result_destination='object-store', result_format='parquet', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] container = called_job.spec.template.spec.containers[0] @@ -411,7 +416,8 @@ def test_launch_transformer_jobs_with_provided_claim(self, mocker): result_destination='volume', result_format='parquet', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] container = called_job.spec.template.spec.containers[0] @@ -459,7 +465,8 @@ def test_launch_transformer_jobs_with_posix_emptydir(self, mocker): result_destination='volume', result_format='parquet', x509_secret='x509', generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_job = mock_kubernetes.mock_calls[1][2]['body'] container = called_job.spec.template.spec.containers[0] @@ -512,7 +519,8 @@ def test_launch_transformer_jobs_no_certs(self, mocker): rabbitmq_uri='ampq://test.com', namespace='my-ns', result_destination='object-store', result_format='arrow', x509_secret=None, generated_code_cm=None, - transformer_language="scala", transformer_command="echo" + transformer_language="scala", transformer_command="echo", + codegen_type=None ) called_deployment = mock_api.mock_calls[1][2]['body'] assert len(called_deployment.spec.template.spec.containers) == 2 diff --git a/transformer_sidecar/scripts/watch.sh b/transformer_sidecar/scripts/watch.sh index a89e49c04..1991e2b1f 100755 --- a/transformer_sidecar/scripts/watch.sh +++ b/transformer_sidecar/scripts/watch.sh @@ -32,7 +32,7 @@ while true; do if [ "${PIPESTATUS[0]}" == 0 ]; then echo "Success. skipping rest of input_files" - mv "$output_file" "$completed_file" + mv $output_file $completed_file touch "$file".done rm "$file" else diff --git a/transformer_sidecar/src/transformer_sidecar/transformer.py b/transformer_sidecar/src/transformer_sidecar/transformer.py index 7678b98a0..7347c385b 100644 --- a/transformer_sidecar/src/transformer_sidecar/transformer.py +++ b/transformer_sidecar/src/transformer_sidecar/transformer.py @@ -182,7 +182,7 @@ def callback(channel, method, properties, body): start_process_info = get_process_info() total_time = 0 - + codegen_type = transform_request['codegen-type'] transform_success = False try: # Loop through the replicas @@ -203,16 +203,25 @@ def callback(channel, method, properties, body): # The transformer will write results here as they are generated. This # directory isn't monitored. - transform_request['safeOutputFileName'] = os.path.join( - scratch_path, - hashed_file_name - ) + if codegen_type == "unzip": + transform_request['safeOutputFileName'] = os.path.join( + scratch_path, + "*" + ) + else: + transform_request['safeOutputFileName'] = os.path.join( + scratch_path, + hashed_file_name + ) - # Final results are written here and picked up by the watched directory thread - transform_request['completedFileName'] = os.path.join( - request_path, - hashed_file_name - ) + if codegen_type == "unzip": + transform_request['completedFileName'] = os.path.join(request_path) + else: + # Final results are written here and picked up by the watched directory thread + transform_request['completedFileName'] = os.path.join( + request_path, + hashed_file_name + ) # creating json file for use by science transformer jsonfile = str(_file_id) + '.json'