diff --git a/did_finder_cernopendata/poetry.lock b/did_finder_cernopendata/poetry.lock index 29bea2309..537a2d383 100644 --- a/did_finder_cernopendata/poetry.lock +++ b/did_finder_cernopendata/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "amqp" @@ -578,6 +578,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-logstash" +version = "0.4.8" +description = "Python logging handler for Logstash." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "python-logstash-0.4.8.tar.gz", hash = "sha256:d04e1ce11ecc107e4a4f3b807fc57d96811e964a554081b3bbb44732f74ef5f9"}, +] + [[package]] name = "requests" version = "2.32.4" @@ -698,4 +709,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = "~3.10" -content-hash = "2bb50166fe66ef1e654abc4bd7538c5a2dae914447aea2d2afb5f393483f385c" +content-hash = "89c9328850f8392cf502674982aec3856d37921321561144bcd62323ef09ce04" diff --git a/did_finder_cernopendata/pyproject.toml b/did_finder_cernopendata/pyproject.toml index 8e40d15f8..7b9bc6e4d 100644 --- a/did_finder_cernopendata/pyproject.toml +++ b/did_finder_cernopendata/pyproject.toml @@ -9,6 +9,7 @@ packages = [{include = "did_finder_cernopendata", from = "src"}] [tool.poetry.dependencies] python = "~3.10" servicex-did-finder-lib = "^3.1.1" +python-logstash = "^0.4.8" [tool.poetry.group.test] optional = true diff --git a/did_finder_cernopendata/src/did_finder_cernopendata/__init__.py b/did_finder_cernopendata/src/did_finder_cernopendata/__init__.py index e69de29bb..ba00fd882 100644 --- a/did_finder_cernopendata/src/did_finder_cernopendata/__init__.py +++ b/did_finder_cernopendata/src/did_finder_cernopendata/__init__.py @@ -0,0 +1,142 @@ +# Copyright (c) 2025, IRIS-HEP +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import os + +import logstash + + +instance = os.environ.get("INSTANCE_NAME", "Unknown") + + +class LogstashFormatter(logstash.formatter.LogstashFormatterBase): + + def format(self, record): + message = { + "@timestamp": self.format_timestamp(record.created), + "@version": "1", + "message": record.getMessage(), + "path": record.pathname, + "tags": self.tags, + "type": self.message_type, + "instance": instance, + "component": "transformer sidecar", + # Extra Fields + "level": record.levelname, + } + + # Add extra fields + message.update(self.get_extra_fields(record)) + + # If exception, add debug info + if record.exc_info: + message.update(self.get_debug_fields(record)) + + return self.serialize(message) + + +class StreamFormatter(logging.Formatter): + """ + A custom formatter that adds extras. + Normally log messages are "level instance component msg extra: {}" + """ + + def_keys = [ + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + ] + + def format(self, record: logging.LogRecord) -> str: + """ + :param record: LogRecord + :return: formatted log message + """ + + string = super().format(record) + extra = {k: v for k, v in record.__dict__.items() if k not in self.def_keys} + if len(extra) > 0: + string += " extra: " + str(extra) + return string + + +def initialize_logging(log=None, **kwargs): + """ + Get a logger and initialize it so that it outputs the correct format + :param request: Request id to insert into log messages + :param log: optional logger to initialize + :return: logger with correct formatting that outputs to console + """ + + logging.basicConfig(level=logging.INFO, force=True) + + if log is None: + log = logging.getLogger() + + log.setLevel(logging.INFO) + stream_handler = logging.StreamHandler() + stream_formatter = StreamFormatter( + "%(levelname)s " + f"{instance} OpenData DID Finder " + "%(message)s" + ) + stream_handler.setFormatter(stream_formatter) + stream_handler.setLevel(log.level) + log.addHandler(stream_handler) + + logstash_host = os.environ.get("LOGSTASH_HOST") + + if logstash_host: + logstash_port = int(os.environ.get("LOGSTASH_PORT", 5959)) + logstash_handler = logstash.TCPLogstashHandler( + logstash_host, logstash_port, version=1 + ) + logstash_formatter = LogstashFormatter("logstash", None, None) + logstash_handler.setFormatter(logstash_formatter) + logstash_handler.setLevel(log.level) + log.addHandler(logstash_handler) + + log.debug("Initialized logging") + + return log diff --git a/did_finder_cernopendata/src/did_finder_cernopendata/celery.py b/did_finder_cernopendata/src/did_finder_cernopendata/celery.py index e0dc7e107..7a7d0ee07 100644 --- a/did_finder_cernopendata/src/did_finder_cernopendata/celery.py +++ b/did_finder_cernopendata/src/did_finder_cernopendata/celery.py @@ -26,7 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import logging +from . import initialize_logging from subprocess import PIPE, Popen, STDOUT from typing import Any, Dict, Generator @@ -36,7 +36,7 @@ LookupFailureException, ) -__log = logging.getLogger(__name__) +__log = initialize_logging() cache_prefix = os.environ.get("CACHE_PREFIX", "") diff --git a/did_finder_rucio/poetry.lock b/did_finder_rucio/poetry.lock index df8b8aa55..e36db9338 100644 --- a/did_finder_rucio/poetry.lock +++ b/did_finder_rucio/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 2.1.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.1.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -1314,6 +1314,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-logstash" +version = "0.4.8" +description = "Python logging handler for Logstash." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "python-logstash-0.4.8.tar.gz", hash = "sha256:d04e1ce11ecc107e4a4f3b807fc57d96811e964a554081b3bbb44732f74ef5f9"}, +] + [[package]] name = "referencing" version = "0.35.1" @@ -1774,4 +1785,4 @@ propcache = ">=0.2.0" [metadata] lock-version = "2.1" python-versions = "^3.9" -content-hash = "94e3bd318d4d7c099efd0fb2423566d9e018bc1268f26531d652d7e5c2035f1b" +content-hash = "9dc28a79000425e68afcd304f34c7b3a0c798364b7b2389ff4e30e5454dd666d" diff --git a/did_finder_rucio/pyproject.toml b/did_finder_rucio/pyproject.toml index df56a7829..84b76c35f 100644 --- a/did_finder_rucio/pyproject.toml +++ b/did_finder_rucio/pyproject.toml @@ -13,6 +13,7 @@ xmltodict = "^0.13.0" servicex-did-finder-lib = "^3.1.1" geoip2 = "^4.7.0" requests = ">=2.25.0,<3.0.0" +python-logstash = "^0.4.8" [tool.poetry.group.test] optional = true diff --git a/did_finder_rucio/src/rucio_did_finder/__init__.py b/did_finder_rucio/src/rucio_did_finder/__init__.py index fa5482391..1381007c5 100644 --- a/did_finder_rucio/src/rucio_did_finder/__init__.py +++ b/did_finder_rucio/src/rucio_did_finder/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2019, IRIS-HEP +# Copyright (c) 2019-25, IRIS-HEP # All rights reserved. # # Redistribution and use in source and binary forms, with or without @@ -25,3 +25,120 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import os + +import logstash +import functools + + +instance = os.environ.get("INSTANCE_NAME", "Unknown") + + +class LogstashFormatter(logstash.formatter.LogstashFormatterBase): + + def format(self, record): + message = { + "@timestamp": self.format_timestamp(record.created), + "@version": "1", + "message": record.getMessage(), + "path": record.pathname, + "tags": self.tags, + "type": self.message_type, + "instance": instance, + "component": "transformer sidecar", + # Extra Fields + "level": record.levelname, + } + + # Add extra fields + message.update(self.get_extra_fields(record)) + + # If exception, add debug info + if record.exc_info: + message.update(self.get_debug_fields(record)) + + return self.serialize(message) + + +class StreamFormatter(logging.Formatter): + """ + A custom formatter that adds extras. + Normally log messages are "level instance component msg extra: {}" + """ + + def_keys = [ + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + ] + + def format(self, record: logging.LogRecord) -> str: + """ + :param record: LogRecord + :return: formatted log message + """ + + string = super().format(record) + extra = {k: v for k, v in record.__dict__.items() if k not in self.def_keys} + if len(extra) > 0: + string += " extra: " + str(extra) + return string + + +@functools.lru_cache +def initialize_logging(log=None, **kwargs): + """ + Get a logger and initialize it so that it outputs the correct format + :param request: Request id to insert into log messages + :param log: optional logger to initialize + :return: logger with correct formatting that outputs to console + """ + + logging.basicConfig(level=logging.INFO, force=True) + + if log is None: + log = logging.getLogger() + + log.setLevel(logging.INFO) + stream_handler = logging.StreamHandler() + stream_formatter = StreamFormatter( + "%(levelname)s " + f"{instance} Rucio DID finder " + "%(message)s" + ) + stream_handler.setFormatter(stream_formatter) + stream_handler.setLevel(log.level) + log.addHandler(stream_handler) + + logstash_host = os.environ.get("LOGSTASH_HOST") + + if logstash_host: + logstash_port = int(os.environ.get("LOGSTASH_PORT", 5959)) + logstash_handler = logstash.TCPLogstashHandler( + logstash_host, logstash_port, version=1 + ) + logstash_formatter = LogstashFormatter("logstash", None, None) + logstash_handler.setFormatter(logstash_formatter) + logstash_handler.setLevel(log.level) + log.addHandler(logstash_handler) + + log.debug("Initialized logging") + + return log diff --git a/did_finder_rucio/src/rucio_did_finder/celery.py b/did_finder_rucio/src/rucio_did_finder/celery.py index 8ec876d86..9f44c3c15 100644 --- a/did_finder_rucio/src/rucio_did_finder/celery.py +++ b/did_finder_rucio/src/rucio_did_finder/celery.py @@ -25,7 +25,7 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging +from . import initialize_logging import os from rucio.client.didclient import DIDClient @@ -36,7 +36,7 @@ from servicex_did_finder_lib import DIDFinderApp from .replica_distance import ReplicaSorter -__log = logging.getLogger(__name__) +initialize_logging() cache_prefix = os.environ.get("CACHE_PREFIX", "") # Initialize the finder diff --git a/did_finder_rucio/src/rucio_did_finder/lookup_request.py b/did_finder_rucio/src/rucio_did_finder/lookup_request.py index 6b801fbf7..4ae2ac67a 100644 --- a/did_finder_rucio/src/rucio_did_finder/lookup_request.py +++ b/did_finder_rucio/src/rucio_did_finder/lookup_request.py @@ -25,7 +25,7 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging +from . import initialize_logging from datetime import datetime from rucio_did_finder.rucio_adapter import RucioAdapter from .replica_distance import ReplicaSorter @@ -55,8 +55,7 @@ def __init__( self.dataset_id = dataset_id # set logging to a null handler - self.logger = logging.getLogger(__name__) - self.logger.addHandler(logging.NullHandler()) + self.logger = initialize_logging() self.location = location self.replica_sorter = replica_sorter diff --git a/did_finder_rucio/src/rucio_did_finder/replica_distance.py b/did_finder_rucio/src/rucio_did_finder/replica_distance.py index 898361a8e..b37ebc53a 100644 --- a/did_finder_rucio/src/rucio_did_finder/replica_distance.py +++ b/did_finder_rucio/src/rucio_did_finder/replica_distance.py @@ -25,7 +25,7 @@ # CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import logging +from . import initialize_logging import os from typing import List, Mapping, Optional, Tuple @@ -40,7 +40,7 @@ Replica_distance = namedtuple("Replica_distance", "replica distance") -logger = logging.getLogger("ReplicaDistanceService") +logger = initialize_logging() def _haversine_distance(lat1: float, lon1: float, lat2: float, lon2: float): diff --git a/did_finder_rucio/src/rucio_did_finder/rucio_adapter.py b/did_finder_rucio/src/rucio_did_finder/rucio_adapter.py index d93a69318..f37b04d81 100644 --- a/did_finder_rucio/src/rucio_did_finder/rucio_adapter.py +++ b/did_finder_rucio/src/rucio_did_finder/rucio_adapter.py @@ -27,7 +27,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import logging +from . import initialize_logging import requests import xmltodict from rucio.common.exception import DataIdentifierNotFound @@ -46,8 +46,7 @@ def __init__(self, did_client, replica_client, report_logical_files=False): self.report_logical_files = report_logical_files self.all_scopes = [] # set logging to a null handler - self.logger = logging.getLogger(__name__) - self.logger.addHandler(logging.NullHandler()) + self.logger = initialize_logging() def client_location(self): client_location = {} diff --git a/did_finder_xrootd/poetry.lock b/did_finder_xrootd/poetry.lock index 4ed6ca5e3..eaf5ede68 100644 --- a/did_finder_xrootd/poetry.lock +++ b/did_finder_xrootd/poetry.lock @@ -600,6 +600,17 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-logstash" +version = "0.4.8" +description = "Python logging handler for Logstash." +optional = false +python-versions = "*" +groups = ["main"] +files = [ + {file = "python-logstash-0.4.8.tar.gz", hash = "sha256:d04e1ce11ecc107e4a4f3b807fc57d96811e964a554081b3bbb44732f74ef5f9"}, +] + [[package]] name = "requests" version = "2.32.4" @@ -732,4 +743,4 @@ files = [ [metadata] lock-version = "2.1" python-versions = ">=3.10,<4.0" -content-hash = "fe5e654c4b6a437f7fddc52786e3ad33ae31d3427c2bb8da196c8433951a22a7" +content-hash = "9bf63cf3b6ad574f0431adb10d381c120e334682b4dbc1dfa81bb2d0cbb44eb4" diff --git a/did_finder_xrootd/pyproject.toml b/did_finder_xrootd/pyproject.toml index 291330312..6e4b45bc8 100644 --- a/did_finder_xrootd/pyproject.toml +++ b/did_finder_xrootd/pyproject.toml @@ -10,6 +10,7 @@ packages = [{include = "servicex_did_finder_xrootd", from="src"}] python = ">=3.10,<4.0" servicex-did-finder-lib = "^3.1.1" xrootd = ">=5.8.3" +python-logstash = "^0.4.8" [tool.poetry.group.test] optional = true diff --git a/did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py b/did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py index e69de29bb..9595cd949 100644 --- a/did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py +++ b/did_finder_xrootd/src/servicex_did_finder_xrootd/__init__.py @@ -0,0 +1,144 @@ +# Copyright (c) 2019-25, IRIS-HEP +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are met: +# +# * Redistributions of source code must retain the above copyright notice, this +# list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright notice, +# this list of conditions and the following disclaimer in the documentation +# and/or other materials provided with the distribution. +# +# * Neither the name of the copyright holder nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +# SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import logging +import os + +import logstash +import functools + + +instance = os.environ.get("INSTANCE_NAME", "Unknown") + + +class LogstashFormatter(logstash.formatter.LogstashFormatterBase): + + def format(self, record): + message = { + "@timestamp": self.format_timestamp(record.created), + "@version": "1", + "message": record.getMessage(), + "path": record.pathname, + "tags": self.tags, + "type": self.message_type, + "instance": instance, + "component": "transformer sidecar", + # Extra Fields + "level": record.levelname, + } + + # Add extra fields + message.update(self.get_extra_fields(record)) + + # If exception, add debug info + if record.exc_info: + message.update(self.get_debug_fields(record)) + + return self.serialize(message) + + +class StreamFormatter(logging.Formatter): + """ + A custom formatter that adds extras. + Normally log messages are "level instance component msg extra: {}" + """ + + def_keys = [ + "name", + "msg", + "args", + "levelname", + "levelno", + "pathname", + "filename", + "module", + "exc_info", + "exc_text", + "stack_info", + "lineno", + "funcName", + "created", + "msecs", + "relativeCreated", + "thread", + "threadName", + "processName", + "process", + "message", + ] + + def format(self, record: logging.LogRecord) -> str: + """ + :param record: LogRecord + :return: formatted log message + """ + + string = super().format(record) + extra = {k: v for k, v in record.__dict__.items() if k not in self.def_keys} + if len(extra) > 0: + string += " extra: " + str(extra) + return string + + +@functools.lru_cache +def initialize_logging(log=None, **kwargs): + """ + Get a logger and initialize it so that it outputs the correct format + :param request: Request id to insert into log messages + :param log: optional logger to initialize + :return: logger with correct formatting that outputs to console + """ + + logging.basicConfig(level=logging.INFO, force=True) + + if log is None: + log = logging.getLogger() + + log.setLevel(logging.INFO) + stream_handler = logging.StreamHandler() + stream_formatter = StreamFormatter( + "%(levelname)s " + f"{instance} XRootD DID finder " + "%(message)s" + ) + stream_handler.setFormatter(stream_formatter) + stream_handler.setLevel(log.level) + log.addHandler(stream_handler) + + logstash_host = os.environ.get("LOGSTASH_HOST") + + if logstash_host: + logstash_port = int(os.environ.get("LOGSTASH_PORT", 5959)) + logstash_handler = logstash.TCPLogstashHandler( + logstash_host, logstash_port, version=1 + ) + logstash_formatter = LogstashFormatter("logstash", None, None) + logstash_handler.setFormatter(logstash_formatter) + logstash_handler.setLevel(log.level) + log.addHandler(logstash_handler) + + log.debug("Initialized logging") + + return log diff --git a/did_finder_xrootd/src/servicex_did_finder_xrootd/celery.py b/did_finder_xrootd/src/servicex_did_finder_xrootd/celery.py index b093e83a2..0280c850c 100644 --- a/did_finder_xrootd/src/servicex_did_finder_xrootd/celery.py +++ b/did_finder_xrootd/src/servicex_did_finder_xrootd/celery.py @@ -26,7 +26,7 @@ # OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import os -import logging +from . import initialize_logging from typing import Any, Dict, Generator from XRootD import client as xrd @@ -36,7 +36,7 @@ LookupFailureException, ) -__log = logging.getLogger(__name__) +__log = initialize_logging() cache_prefix = os.environ.get("CACHE_PREFIX", "") app = DIDFinderApp("xrootd") diff --git a/helm/servicex/templates/did-finder-cernopendata/deployment.yaml b/helm/servicex/templates/did-finder-cernopendata/deployment.yaml index f82da5d8b..dba1cdf48 100644 --- a/helm/servicex/templates/did-finder-cernopendata/deployment.yaml +++ b/helm/servicex/templates/did-finder-cernopendata/deployment.yaml @@ -39,4 +39,10 @@ spec: - name: BROKER_URL value: amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/%2F {{- end }} + {{- if .Values.logging.logstash.enabled }} + - name: LOGSTASH_HOST + value: "{{ .Values.logging.logstash.host }}" + - name: LOGSTASH_PORT + value: "{{ .Values.logging.logstash.port }}" + {{- end }} {{ end }} diff --git a/helm/servicex/templates/did-finder-rucio/deployment.yaml b/helm/servicex/templates/did-finder-rucio/deployment.yaml index 9118c9f6a..8ef439039 100644 --- a/helm/servicex/templates/did-finder-rucio/deployment.yaml +++ b/helm/servicex/templates/did-finder-rucio/deployment.yaml @@ -66,6 +66,12 @@ spec: {{- end }} {{- end }} {{- end }} + {{- if .Values.logging.logstash.enabled }} + - name: LOGSTASH_HOST + value: "{{ .Values.logging.logstash.host }}" + - name: LOGSTASH_PORT + value: "{{ .Values.logging.logstash.port }}" + {{- end }} - name: INSTANCE_NAME value: {{ .Release.Name }} volumeMounts: diff --git a/helm/servicex/templates/did-finder-xrootd/deployment.yaml b/helm/servicex/templates/did-finder-xrootd/deployment.yaml index 91fe53682..65f5df0e7 100644 --- a/helm/servicex/templates/did-finder-xrootd/deployment.yaml +++ b/helm/servicex/templates/did-finder-xrootd/deployment.yaml @@ -39,6 +39,12 @@ spec: - name: BROKER_URL value: amqp://user:{{ .Values.rabbitmq.auth.password }}@{{ .Release.Name }}-rabbitmq:5672/%2F {{- end }} + {{- if .Values.logging.logstash.enabled }} + - name: LOGSTASH_HOST + value: "{{ .Values.logging.logstash.host }}" + - name: LOGSTASH_PORT + value: "{{ .Values.logging.logstash.port }}" + {{- end }} {{ if not .Values.noCerts}} volumeMounts: - name: x509-secret