From e7e1218588afd2d5b9e32cbf88332f64debe4919 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 23 Sep 2025 13:59:45 +0300 Subject: [PATCH 01/44] updated pull-request.json --- .buildkite/pull-requests.json | 89 +++++++++++++------ .../logstash-filter-acceptance_test.gemspec | 2 +- 2 files changed, 65 insertions(+), 26 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 2414a75b5b6..bc5861ad3ce 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -1,26 +1,65 @@ { - "jobs": [ - { - "enabled": true, - "pipeline_slug": "logstash-pull-request-pipeline", - "allow_org_users": true, - "allowed_repo_permissions": ["admin", "write"], - "allowed_list": ["dependabot[bot]", "mergify[bot]", "github-actions[bot]", "elastic-vault-github-plugin-prod[bot]"], - "set_commit_status": true, - "build_on_commit": true, - "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", - "skip_ci_labels": [ ], - "skip_target_branches": [ ], - "skip_ci_on_only_changed": [ - "^.github/", - "^docs/", - "^.mergify.yml$", - "^.pre-commit-config.yaml", - "\\.md$" - ], - "always_require_ci_on_changed": [ ] - } - ] - } + "jobs": [ + { + "enabled": true, + "pipeline_slug": "logstash-pull-request-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]", + "elastic-vault-github-plugin-prod[bot]" + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:build|test)\\W+(?:this|it))", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [] + }, + { + "enabled": true, + "pipeline_slug": "logstash-exhaustive-tests-pipeline", + "allow_org_users": true, + "allowed_repo_permissions": [ + "admin", + "write" + ], + "allowed_list": [ + "dependabot[bot]", + "mergify[bot]", + "github-actions[bot]", + ], + "set_commit_status": true, + "build_on_commit": true, + "build_on_comment": true, + "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:exhaustive)\\W+(?:this|it))|^/exhaustive$", + "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:exhaustive)\\W+(?:this|it))|^/exhaustive$", + "skip_ci_labels": [], + "skip_target_branches": [], + "skip_ci_on_only_changed": [ + "^.github/", + "^docs/", + "^.mergify.yml$", + "^.pre-commit-config.yaml", + "\\.md$" + ], + "always_require_ci_on_changed": [ + "^qa/acceptance/" + ] + } + ] +} diff --git a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec index 924a341cf66..c2a53932d6e 100644 --- a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec +++ b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec @@ -16,4 +16,4 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - end \ No newline at end of file + end From 4b19fe5c9e0a86416d5cbdebc3232b24ce22dc14 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 23 Sep 2025 15:42:39 +0300 Subject: [PATCH 02/44] restored the qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec --- qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec index c2a53932d6e..924a341cf66 100644 --- a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec +++ b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec @@ -16,4 +16,4 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - end + end \ No newline at end of file From eb8c8e41c66f44a2f30e180deac3ad50d3baca52 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 23 Sep 2025 15:47:20 +0300 Subject: [PATCH 03/44] cleaned always_require_ci_on_changed values after testing --- .buildkite/pull-requests.json | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index bc5861ad3ce..eb962ca862c 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -41,7 +41,7 @@ "allowed_list": [ "dependabot[bot]", "mergify[bot]", - "github-actions[bot]", + "github-actions[bot]" ], "set_commit_status": true, "build_on_commit": true, @@ -57,9 +57,7 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": [ - "^qa/acceptance/" - ] + "always_require_ci_on_changed": [] } ] } From 53323ab51cca01b060dd94b0a01b608476138eee Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 23 Sep 2025 16:32:00 +0300 Subject: [PATCH 04/44] updated the trigger comment --- .buildkite/pull-requests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index eb962ca862c..50026e376f2 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -46,8 +46,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:exhaustive)\\W+(?:this|it))|^/exhaustive$", - "always_trigger_comment_regex": "^(?:(?:buildkite\\W+)?(?:exhaustive)\\W+(?:this|it))|^/exhaustive$", + "trigger_comment_regex": "^(?:(?:run\\W+)?(?:exhaustive)\\W+(?:tests))", + "always_trigger_comment_regex": "^(?:(?:run\\W+)?(?:exhaustive)\\W+(?:tests))", "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ From e7b622b2a7112e695670519de64366466125ff9f Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Tue, 23 Sep 2025 16:46:18 +0300 Subject: [PATCH 05/44] updated the trigger comment --- .buildkite/pull-requests.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 50026e376f2..8f644b64cb6 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -46,8 +46,8 @@ "set_commit_status": true, "build_on_commit": true, "build_on_comment": true, - "trigger_comment_regex": "^(?:(?:run\\W+)?(?:exhaustive)\\W+(?:tests))", - "always_trigger_comment_regex": "^(?:(?:run\\W+)?(?:exhaustive)\\W+(?:tests))", + "trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", + "always_trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ From 6836cbee24f4bae152693e42ae8ca3bda5d235fd Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 24 Sep 2025 11:56:46 +0300 Subject: [PATCH 06/44] testing comments --- .buildkite/pull-requests.json | 2 +- qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 8f644b64cb6..1b97b84c935 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -57,7 +57,7 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": [] + "always_require_ci_on_changed": ["^qa/acceptance"] } ] } diff --git a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec index 924a341cf66..c2a53932d6e 100644 --- a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec +++ b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec @@ -16,4 +16,4 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - end \ No newline at end of file + end From 5f31a13ea018d02c19c520f443e52dd274641309 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 24 Sep 2025 11:58:09 +0300 Subject: [PATCH 07/44] testing comments --- .buildkite/pull-requests.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 1b97b84c935..7f2df185f6e 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -57,7 +57,7 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": ["^qa/acceptance"] + "always_require_ci_on_changed": ["^qa/acceptance/"] } ] } From 66f5eba0662a7d3f9c7ef49305761f1dbe36c5f9 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Wed, 24 Sep 2025 12:06:01 +0300 Subject: [PATCH 08/44] testing comments --- .buildkite/pull-requests.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 7f2df185f6e..f369e28aac9 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -57,7 +57,9 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": ["^qa/acceptance/"] + "always_require_ci_on_changed": [ + "^qa/acceptance/" + ] } ] } From 8060880db5deb2314f075c117ebf73dbfc63a7e3 Mon Sep 17 00:00:00 2001 From: Olga Naydyonock Date: Thu, 25 Sep 2025 13:46:30 +0300 Subject: [PATCH 09/44] updated pull-request.json --- .buildkite/pull-requests.json | 6 ++---- .../fixtures/logstash-filter-acceptance_test.gemspec | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index f369e28aac9..309dc75ad17 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -44,7 +44,7 @@ "github-actions[bot]" ], "set_commit_status": true, - "build_on_commit": true, + "build_on_commit": false, "build_on_comment": true, "trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", "always_trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", @@ -57,9 +57,7 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": [ - "^qa/acceptance/" - ] + "always_require_ci_on_changed": [] } ] } diff --git a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec index c2a53932d6e..924a341cf66 100644 --- a/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec +++ b/qa/acceptance/fixtures/logstash-filter-acceptance_test.gemspec @@ -16,4 +16,4 @@ Gem::Specification.new do |s| s.metadata = { "logstash_plugin" => "true", "logstash_group" => "filter" } # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - end + end \ No newline at end of file From d2bde7470b1b4c6908c74c61f4ea33766077b53b Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Oct 2025 15:21:06 +0200 Subject: [PATCH 10/44] enable changed files --- .buildkite/pull-requests.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index 309dc75ad17..d2a737720a1 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -44,10 +44,10 @@ "github-actions[bot]" ], "set_commit_status": true, - "build_on_commit": false, + "build_on_commit": true, "build_on_comment": true, "trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", - "always_trigger_comment_regex": "^(?:(?:/run\\W+)?(?:exhaustive)\\W+(?:tests))", + "always_trigger_comment_regex": "^(?:(?:/run\\W+)(?:exhaustive)\\W+(?:tests))", "skip_ci_labels": [], "skip_target_branches": [], "skip_ci_on_only_changed": [ @@ -57,7 +57,9 @@ "^.pre-commit-config.yaml", "\\.md$" ], - "always_require_ci_on_changed": [] + "always_require_ci_on_changed": [ + "^qa/acceptance/" + ] } ] } From b5ff3237ff0338931b3e3108c4ef8138c1e3e4ae Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Oct 2025 15:49:45 +0200 Subject: [PATCH 11/44] support changed files and github comment --- .buildkite/pull-requests.json | 2 +- .../smart_exhaustive_tests_pipeline.yml | 32 +++++++++++++ catalog-info.yaml | 45 +++++++++++++++++++ 3 files changed, 78 insertions(+), 1 deletion(-) create mode 100644 .buildkite/smart_exhaustive_tests_pipeline.yml diff --git a/.buildkite/pull-requests.json b/.buildkite/pull-requests.json index d2a737720a1..bd9a7d22005 100644 --- a/.buildkite/pull-requests.json +++ b/.buildkite/pull-requests.json @@ -32,7 +32,7 @@ }, { "enabled": true, - "pipeline_slug": "logstash-exhaustive-tests-pipeline", + "pipeline_slug": "logstash-smart-exhaustive-tests-pipeline", "allow_org_users": true, "allowed_repo_permissions": [ "admin", diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml new file mode 100644 index 00000000000..05aa649621b --- /dev/null +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -0,0 +1,32 @@ +steps: + - label: "Trigger logstash-exhaustive-tests-pipeline for PRs with qa/acceptance/ changes" + if: build.pull_request.id != null + plugins: + - monorepo-diff#v1.0.1: + diff: "git diff --name-only origin/${GITHUB_PR_TARGET_BRANCH}...HEAD" + interpolation: false + watch: + - path: + - ^qa/acceptance/ + - .buildkite/smart_exhaustive_tests_pipeline.yml + config: + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false + + - label: "Trigger logstash-exhaustive-tests-pipeline for GitHub comments" + if: build.env("GITHUB_PR_TRIGGER_COMMENT") != "" + trigger: "logstash-exhaustive-tests-pipeline" + build: + commit: "${BUILDKITE_COMMIT}" + branch: "${BUILDKITE_BRANCH}" + env: + - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} + - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} + - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} diff --git a/catalog-info.yaml b/catalog-info.yaml index 143acb634b7..d312b192a38 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -479,6 +479,51 @@ spec: # SECTION END: Exhaustive tests pipeline # ************************************** +# **************************************** +# SECTION START: Smart exhaustive tests pipeline +# **************************************** + +--- +# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json +apiVersion: backstage.io/v1alpha1 +kind: Resource +metadata: + name: logstash-smart-exhaustive-tests-pipeline + description: 'Logstash Smart Exhaustive tests pipeline' + links: + - title: 'Logstash Smart Exhaustive tests pipeline' + url: https://buildkite.com/elastic/logstash-smart-exhaustive-tests-pipeline +spec: + type: buildkite-pipeline + owner: group:logstash + system: platform-ingest + implementation: + apiVersion: buildkite.elastic.dev/v1 + kind: Pipeline + metadata: + name: "Logstash Smart Exhaustive tests pipeline" + description: '🔍 Run exhaustive tests against Logstash using different operating systems' + spec: + repository: elastic/logstash + pipeline_file: ".buildkite/smart_exhaustive_tests_pipeline.yml" + provider_settings: + trigger_mode: none + cancel_intermediate_builds: false + skip_intermediate_builds: false + teams: + ingest-fp: + access_level: MANAGE_BUILD_AND_READ + logstash: + access_level: MANAGE_BUILD_AND_READ + ingest-eng-prod: + access_level: MANAGE_BUILD_AND_READ + everyone: + access_level: READ_ONLY + +# ************************************** +# SECTION END: Smart exhaustive tests pipeline +# ************************************** + # ******************************************** # Declare supported plugin tests pipeline # ******************************************** From c8268e9f4384926b67dde162ec5f0f594722f485 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Oct 2025 15:56:59 +0200 Subject: [PATCH 12/44] Update .buildkite/smart_exhaustive_tests_pipeline.yml --- .buildkite/smart_exhaustive_tests_pipeline.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.buildkite/smart_exhaustive_tests_pipeline.yml b/.buildkite/smart_exhaustive_tests_pipeline.yml index 05aa649621b..0e541f902d3 100644 --- a/.buildkite/smart_exhaustive_tests_pipeline.yml +++ b/.buildkite/smart_exhaustive_tests_pipeline.yml @@ -30,3 +30,4 @@ steps: - BUILDKITE_PULL_REQUEST=${BUILDKITE_PULL_REQUEST} - BUILDKITE_PULL_REQUEST_BASE_BRANCH=${BUILDKITE_PULL_REQUEST_BASE_BRANCH} - GITHUB_PR_LABELS=${GITHUB_PR_LABELS} + - ELASTIC_SLACK_NOTIFICATIONS_ENABLE=false From 51f0f4349812c1024ad36421eb2fc5e84cb5464c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Oct 2025 15:59:31 +0200 Subject: [PATCH 13/44] same pattern done at logstash-pull-request-pipeline --- catalog-info.yaml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index d312b192a38..d6a640afc8f 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -507,9 +507,15 @@ spec: repository: elastic/logstash pipeline_file: ".buildkite/smart_exhaustive_tests_pipeline.yml" provider_settings: - trigger_mode: none - cancel_intermediate_builds: false - skip_intermediate_builds: false + build_pull_request_forks: false + build_pull_requests: true # requires filter_enabled and filter_condition settings as below when used with buildkite-pr-bot + build_branches: false + build_tags: false + filter_enabled: true + filter_condition: >- + build.creator.name == 'elasticmachine' && build.pull_request.id != null + cancel_intermediate_builds: true + skip_intermediate_builds: true teams: ingest-fp: access_level: MANAGE_BUILD_AND_READ From 0b8bd3f4166b9283003f9e3ad0b1d273c7852d9c Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 21 Oct 2025 16:00:27 +0200 Subject: [PATCH 14/44] Update catalog-info.yaml --- catalog-info.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/catalog-info.yaml b/catalog-info.yaml index d6a640afc8f..b1fada97a77 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -502,7 +502,7 @@ spec: kind: Pipeline metadata: name: "Logstash Smart Exhaustive tests pipeline" - description: '🔍 Run exhaustive tests against Logstash using different operating systems' + description: '🔍 Run smart exhaustive tests against Logstash using different operating systems' spec: repository: elastic/logstash pipeline_file: ".buildkite/smart_exhaustive_tests_pipeline.yml" From 748774d8a2dee031198ad2b8e2a88eb5fcccdd8f Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 23 Sep 2025 16:31:10 +0200 Subject: [PATCH 15/44] Measure average batch byte size and event count (#18000) Implements average batch event count and byte size metrics. The collection of such metric could be disabled, enabled for each batch or done on a sample of the total batches. Exposing metric related to average batch byte size and event count let the user discover the average structure of their batches, understanding if the batches are fulfilled and eventually understand how to set `pipeline.batch.size` and `pipeline.batch.delay` so that goal is reached. - Instantiate metric `pipelines..batch.count` to count number of matches to compute the average events and bytes per batch - Instantiate metric `pipelines..batch.total_bytes` to sumup all the batches event's byte estimation. Exposed metric `pipelines..batch.byte_size.average.lifetime` containing the average byte size of each batch. - create new setting `pipeline.batch.metrics.sampling_mode` which could have 3 values: `disabled`, `minimal` and `full`. In this case id `disable` no `batch` metric is exposed in the `_node/stats` API. `minimal` count batches and estimates the size only for 1% of the total while `full` is for every batch. This setting leverages existing Logstash setting infrastructure so that one defined at pipeline level (defined in `pipelines.yml`) takes precedence over the global one (defined in `logstash.yml`). --- config/logstash.yml | 7 + .../lib/logstash/api/commands/stats.rb | 18 +++ logstash-core/lib/logstash/environment.rb | 1 + logstash-core/lib/logstash/java_pipeline.rb | 2 + logstash-core/lib/logstash/settings.rb | 1 + .../acked_queue_concurrent_stress_spec.rb | 2 +- .../logstash/api/modules/node_stats_spec.rb | 21 ++- .../instrument/wrapped_write_client_spec.rb | 6 +- .../spec/logstash/queue_factory_spec.rb | 1 + .../logstash/util/wrapped_acked_queue_spec.rb | 2 +- .../util/wrapped_synchronous_queue_spec.rb | 2 +- .../main/java/org/logstash/ConvertedMap.java | 122 +++++++++++++++- .../src/main/java/org/logstash/Event.java | 13 ++ .../logstash/ackedqueue/QueueFactoryExt.java | 36 +++-- .../org/logstash/ackedqueue/Settings.java | 1 - .../ackedqueue/ext/JRubyAckedQueueExt.java | 2 +- .../ext/JRubyWrappedAckedQueueExt.java | 32 ++++- .../common/SettingKeyDefinitions.java | 2 + .../execution/AbstractPipelineExt.java | 31 +++- .../execution/QueueReadClientBase.java | 22 ++- .../QueueReadClientBatchMetrics.java | 76 ++++++++++ .../logstash/ext/JrubyAckedReadClientExt.java | 11 +- .../ext/JrubyMemoryReadClientExt.java | 15 +- .../ext/JrubyWrappedSynchronousQueueExt.java | 32 ++++- .../instrument/metrics/MetricKeys.java | 15 ++ .../QueueReadClientBatchMetricsTest.java | 133 ++++++++++++++++++ .../ext/JrubyMemoryReadClientExtTest.java | 79 ++++++++++- .../metrics/MockNamespacedMetric.java | 101 +++++++++++++ qa/integration/specs/monitoring_api_spec.rb | 64 +++++++++ 29 files changed, 805 insertions(+), 45 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java create mode 100644 logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java create mode 100644 logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java diff --git a/config/logstash.yml b/config/logstash.yml index e4d008ca5cd..9658274fa64 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -49,6 +49,13 @@ # # pipeline.batch.delay: 50 # +# Set the pipeline's batch metrics reporting mode. It can be "disabled" to disable it. +# "minimal" to collect only 1% of the batches metrics, "full" to collect all batches. +# Default is "minimal". +# +# pipeline.batch.metrics.sampling_mode: "minimal" +pipeline.batch.metrics.sampling_mode: minimal +# # Force Logstash to exit during shutdown even if there are still inflight # events in memory. By default, logstash will refuse to quit until all # received events have been pushed to the outputs. diff --git a/logstash-core/lib/logstash/api/commands/stats.rb b/logstash-core/lib/logstash/api/commands/stats.rb index 5bf1b3e3a09..2ec44fe0afa 100644 --- a/logstash-core/lib/logstash/api/commands/stats.rb +++ b/logstash-core/lib/logstash/api/commands/stats.rb @@ -172,6 +172,23 @@ def plugin_stats(stats, plugin_type) end end + def refine_batch_metrics(stats) + { + :event_count => { + :average => { + # average return a FlowMetric which and we need to invoke getValue to obtain the map with metric details. + :lifetime => stats[:batch][:event_count][:average].value["lifetime"] ? stats[:batch][:event_count][:average].value["lifetime"].round : 0 + } + }, + :byte_size => { + :average => { + :lifetime => stats[:batch][:byte_size][:average].value["lifetime"] ? stats[:batch][:byte_size][:average].value["lifetime"].round : 0 + } + } + } + end + private :refine_batch_metrics + def report(stats, extended_stats = nil, opts = {}) ret = { :events => stats[:events], @@ -190,6 +207,7 @@ def report(stats, extended_stats = nil, opts = {}) :batch_delay => stats.dig(:config, :batch_delay), } } + ret[:batch] = refine_batch_metrics(stats) if stats.include?(:batch) ret[:dead_letter_queue] = stats[:dlq] if stats.include?(:dlq) # if extended_stats were provided, enrich the return value diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index d49d3d9b6fa..4611741a54c 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -86,6 +86,7 @@ def self.as_java_range(r) Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable, Setting::Password.new("api.ssl.keystore.password", nil, false).nullable, Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]), + Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), Setting::StringSetting.new("queue.type", "memory", true, ["persisted", "memory"]), Setting::BooleanSetting.new("queue.drain", false), Setting::Bytes.new("queue.page_capacity", "64mb"), diff --git a/logstash-core/lib/logstash/java_pipeline.rb b/logstash-core/lib/logstash/java_pipeline.rb index 11ce715bea2..600b689d0f1 100644 --- a/logstash-core/lib/logstash/java_pipeline.rb +++ b/logstash-core/lib/logstash/java_pipeline.rb @@ -267,6 +267,7 @@ def start_workers @preserve_event_order = preserve_event_order?(pipeline_workers) batch_size = settings.get("pipeline.batch.size") batch_delay = settings.get("pipeline.batch.delay") + batch_metric_sampling = settings.get("pipeline.batch.metrics.sampling_mode") max_inflight = batch_size * pipeline_workers @@ -287,6 +288,7 @@ def start_workers "pipeline.batch.size" => batch_size, "pipeline.batch.delay" => batch_delay, "pipeline.max_inflight" => max_inflight, + "batch_metric_sampling" => batch_metric_sampling, "pipeline.sources" => pipeline_source_details) @logger.info("Starting pipeline", pipeline_log_params) diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index c23455df930..5458d16e346 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -58,6 +58,7 @@ def self.included(base) "path.dead_letter_queue", "path.queue", "pipeline.batch.delay", + "pipeline.batch.metrics.sampling_mode", "pipeline.batch.size", "pipeline.id", "pipeline.reloadable", diff --git a/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb b/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb index ccd37f6bb4e..c7c1eae328a 100644 --- a/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb +++ b/logstash-core/spec/logstash/acked_queue_concurrent_stress_spec.rb @@ -38,7 +38,7 @@ end let(:queue) do - described_class.new(queue_settings) + described_class.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) end let(:writer_threads) do diff --git a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb index 48b73f80ce5..71468fdd63e 100644 --- a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb +++ b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb @@ -21,8 +21,13 @@ require "logstash/api/modules/node_stats" describe LogStash::Api::Modules::NodeStats do - # enable PQ to ensure PQ-related metrics are present - include_context "api setup", {"queue.type" => "persisted"} + + include_context "api setup", { + # enable PQ to ensure PQ-related metrics are present + "queue.type" => "persisted", + #enable batch metrics + "pipeline.batch.metrics.sampling_mode" => "full" + } include_examples "not found" extend ResourceDSLMethods @@ -142,6 +147,18 @@ "path" => String, "free_space_in_bytes" => Numeric } + }, + "batch" => { + "event_count" => { + "average" => { + "lifetime" => Numeric + } + }, + "byte_size" => { + "average" => { + "lifetime" => Numeric + } + } } } }, diff --git a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb index 1c1f9a1fb2d..3fa5bf325ad 100644 --- a/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb +++ b/logstash-core/spec/logstash/instrument/wrapped_write_client_spec.rb @@ -113,7 +113,7 @@ def threaded_read_client end context "WrappedSynchronousQueue" do - let(:queue) { LogStash::WrappedSynchronousQueue.new(1024) } + let(:queue) { LogStash::WrappedSynchronousQueue.new(1024, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) } before do read_client.set_events_metric(metric.namespace([:stats, :events])) @@ -136,7 +136,9 @@ def threaded_read_client .build end - let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings) } + let(:queue) do + LogStash::WrappedAckedQueue.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) + end before do read_client.set_events_metric(metric.namespace([:stats, :events])) diff --git a/logstash-core/spec/logstash/queue_factory_spec.rb b/logstash-core/spec/logstash/queue_factory_spec.rb index 540113412c4..7297e92dc43 100644 --- a/logstash-core/spec/logstash/queue_factory_spec.rb +++ b/logstash-core/spec/logstash/queue_factory_spec.rb @@ -31,6 +31,7 @@ LogStash::Setting::NumericSetting.new("queue.checkpoint.writes", 1024), LogStash::Setting::BooleanSetting.new("queue.checkpoint.retry", false), LogStash::Setting::StringSetting.new("pipeline.id", pipeline_id), + LogStash::Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), LogStash::Setting::PositiveIntegerSetting.new("pipeline.batch.size", 125), LogStash::Setting::PositiveIntegerSetting.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum) ] diff --git a/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb index 67093a78eea..19934c7892e 100644 --- a/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_acked_queue_spec.rb @@ -64,7 +64,7 @@ .build end - let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings) } + let(:queue) { LogStash::WrappedAckedQueue.new(queue_settings, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED) } after do queue.close diff --git a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb index 9e644195806..2f4b771e00d 100644 --- a/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb +++ b/logstash-core/spec/logstash/util/wrapped_synchronous_queue_spec.rb @@ -19,7 +19,7 @@ require "logstash/instrument/collector" describe LogStash::WrappedSynchronousQueue do - subject {LogStash::WrappedSynchronousQueue.new(5)} + subject {LogStash::WrappedSynchronousQueue.new(5, org.logstash.ackedqueue.QueueFactoryExt::BatchMetricMode::DISABLED)} describe "queue clients" do context "when requesting a write client" do diff --git a/logstash-core/src/main/java/org/logstash/ConvertedMap.java b/logstash-core/src/main/java/org/logstash/ConvertedMap.java index c9f0da606e9..c815679d4ea 100644 --- a/logstash-core/src/main/java/org/logstash/ConvertedMap.java +++ b/logstash-core/src/main/java/org/logstash/ConvertedMap.java @@ -21,14 +21,26 @@ package org.logstash; import java.io.Serializable; +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Collection; import java.util.HashMap; import java.util.IdentityHashMap; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; + +import org.jruby.RubyBignum; +import org.jruby.RubyBoolean; +import org.jruby.RubyFixnum; +import org.jruby.RubyFloat; import org.jruby.RubyHash; +import org.jruby.RubyNil; import org.jruby.RubyString; +import org.jruby.RubySymbol; +import org.jruby.ext.bigdecimal.RubyBigDecimal; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.ext.JrubyTimestampExtLibrary; /** *

This class is an internal API and behaves very different from a standard {@link Map}.

@@ -41,7 +53,7 @@ * intern pool to ensure identity match of equivalent strings. * For performance, we keep a global cache of strings that have been interned for use with {@link ConvertedMap}, * and encourage interning through {@link ConvertedMap#internStringForUseAsKey(String)} to avoid - * the performance pentalty of the global string intern pool. + * the performance penalty of the global string intern pool. */ public final class ConvertedMap extends IdentityHashMap { @@ -157,4 +169,112 @@ public Object unconvert() { private static String convertKey(final RubyString key) { return internStringForUseAsKey(key.asJavaString()); } + + public long estimateMemory() { + return values().stream() + .map(this::estimateMemory) + .mapToLong(Long::longValue) + .sum(); + } + + @SuppressWarnings({"rawtypes", "unchecked"}) + private long estimateMemory(Object o) { + if (o instanceof Boolean) { + return Byte.BYTES; + } + if (o instanceof Byte) { + return Byte.BYTES; + } + if (o instanceof Short) { + return Short.BYTES; + } + if (o instanceof Integer) { + return Integer.BYTES; + } + if (o instanceof Long) { + return Long.BYTES; + } + if (o instanceof Float) { + return Float.BYTES; + } + if (o instanceof Double) { + return Double.BYTES; + } + if (o instanceof Character) { + return Character.BYTES; + } + if (o instanceof String) { + return ((String) o).getBytes().length; + } + if (o instanceof RubyString) { + return ((RubyString) o).getBytes().length; + } + + if (o instanceof Collection) { + Collection c = (Collection) o; + long memory = 0L; + for (Object v : c) { + memory += estimateMemory(v); + } + return memory; + } + + if (o instanceof ConvertedMap) { + ConvertedMap c = (ConvertedMap) o; + return c.estimateMemory(); + } + + if (o instanceof Map) { + // this case shouldn't happen because all Map are converted to ConvertedMap + Map m = (Map) o; + long memory = 0L; + for (Map.Entry e : m.entrySet()) { + memory += estimateMemory(e.getKey()); + memory += estimateMemory(e.getValue()); + } + return memory; + } + if (o instanceof JrubyTimestampExtLibrary.RubyTimestamp) { + // wraps an java.time.Instant which is made of long and int + return Long.BYTES + Integer.BYTES; + } + if (o instanceof BigInteger) { + return ((BigInteger) o).toByteArray().length; + } + if (o instanceof BigDecimal) { + // BigInteger has 4 fields, one reference 2 ints (scale and precision) and a long. + return 8 + 2 * Integer.BYTES + Long.BYTES; + } + if (o instanceof RubyBignum) { + RubyBignum rbn = (RubyBignum) o; + return ((RubyFixnum) rbn.size()).getLongValue(); + } + if (o instanceof RubyBigDecimal) { + RubyBigDecimal rbd = (RubyBigDecimal) o; + // wraps a Java BigDecimal so we can return the size of that: + return estimateMemory(rbd.getValue()); + } + if (o instanceof RubyFixnum) { + // like an int value + return Integer.BYTES; + } + if (o instanceof RubyBoolean) { + return Byte.BYTES; + } + if (o instanceof RubyNil) { + return 8 + Integer.BYTES; // object reference, one int + } + if (o instanceof RubySymbol) { + return estimateMemory(((RubySymbol) o).asJavaString()); + } + if (o instanceof RubyFloat) { + return Double.BYTES; + } + + throw new IllegalArgumentException( + "Unsupported type encountered in estimateMemory: " + o.getClass().getName() + + ". Please ensure all objects passed to estimateMemory are of supported types. " + + "Refer to the ConvertedMap.estimateMemory method for the list of supported types." + ); + } } diff --git a/logstash-core/src/main/java/org/logstash/Event.java b/logstash-core/src/main/java/org/logstash/Event.java index e1e9f4db1f2..2a9f79ab9b5 100644 --- a/logstash-core/src/main/java/org/logstash/Event.java +++ b/logstash-core/src/main/java/org/logstash/Event.java @@ -529,6 +529,7 @@ private void initFailTag(final Object tag) { * and needs to be converted to a list before appending to it. * @param existing Existing Tag * @param tag Tag to add + * */ private void scalarTagFallback(final String existing, final String tag) { final List tags = new ArrayList<>(2); @@ -567,4 +568,16 @@ private static String getCanonicalFieldReference(final FieldReference field) { return path.stream().collect(Collectors.joining("][", "[", "]")); } } + + /** + * @return a byte size estimation of the event, based on the payloads carried by nested data structures, + * without considering the space needed by the JVM to represent the object itself. + * + * */ + public long estimateMemory() { + long total = 0; + total += data.estimateMemory(); + total += metadata.estimateMemory(); + return total; + } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index 6a10c2a3e7f..c44560ad7c6 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -46,6 +46,12 @@ @JRubyClass(name = "QueueFactory") public final class QueueFactoryExt extends RubyBasicObject { + public enum BatchMetricMode { + DISABLED, + MINIMAL, + FULL + } + /** * A static value to indicate Persistent Queue is enabled. */ @@ -71,6 +77,7 @@ public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { public static AbstractWrappedQueueExt create(final ThreadContext context, final IRubyObject recv, final IRubyObject settings) throws IOException { final String type = getSetting(context, settings, QUEUE_TYPE_CONTEXT_NAME).asJavaString(); + final BatchMetricMode batchMetricMode = decodeBatchMetricMode(context, settings); if (PERSISTED_TYPE.equals(type)) { final Settings queueSettings = extractQueueSettings(settings); final Path queuePath = Paths.get(queueSettings.getDirPath()); @@ -81,18 +88,14 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final Files.createDirectories(queuePath); } - return JRubyWrappedAckedQueueExt.create(context, queueSettings); + return JRubyWrappedAckedQueueExt.create(context, queueSettings, batchMetricMode); } else if (MEMORY_TYPE.equals(type)) { - return new JrubyWrappedSynchronousQueueExt( - context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS - ).initialize( - context, context.runtime.newFixnum( - getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) - .convertToInteger().getIntValue() - * getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) - .convertToInteger().getIntValue() - ) - ); + final int batchSize = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) + .convertToInteger().getIntValue(); + final int workers = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_WORKERS) + .convertToInteger().getIntValue(); + int queueSize = batchSize * workers; + return JrubyWrappedSynchronousQueueExt.create(context, queueSize, batchMetricMode); } else { throw context.runtime.newRaiseException( RubyUtil.CONFIGURATION_ERROR_CLASS, @@ -104,6 +107,16 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final } } + private static BatchMetricMode decodeBatchMetricMode(ThreadContext context, IRubyObject settings) { + final String batchMetricModeStr = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_METRICS) + .asJavaString(); + + if (batchMetricModeStr == null || batchMetricModeStr.isEmpty()) { + return BatchMetricMode.DISABLED; + } + return BatchMetricMode.valueOf(batchMetricModeStr.toUpperCase()); + } + private static IRubyObject getSetting(final ThreadContext context, final IRubyObject settings, final String name) { return settings.callMethod(context, "get_value", context.runtime.newString(name)); @@ -115,6 +128,7 @@ private static Settings extractQueueSettings(final IRubyObject settings) { getSetting(context, settings, PATH_QUEUE).asJavaString(), getSetting(context, settings, PIPELINE_ID).asJavaString() ); + return SettingsImpl.fileSettingsBuilder(queuePath.toString()) .elementClass(Event.class) .capacity(getSetting(context, settings, QUEUE_PAGE_CAPACITY).toJava(Integer.class)) diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java index 1623738659e..212945ef07f 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java @@ -90,6 +90,5 @@ interface Builder { Builder checkpointRetry(boolean checkpointRetry); Settings build(); - } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java index 35cb765dbc9..770824935ce 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java @@ -44,7 +44,7 @@ import org.logstash.ackedqueue.SettingsImpl; /** - * JRuby extension to wrap a persistent queue istance. + * JRuby extension to wrap a persistent queue instance. */ @JRubyClass(name = "AckedQueue") public final class JRubyAckedQueueExt extends RubyObject { diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java index d2d374b56f7..42d11da8875 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java @@ -21,6 +21,7 @@ package org.logstash.ackedqueue.ext; import java.io.IOException; +import java.util.Objects; import org.jruby.Ruby; import org.jruby.RubyBoolean; @@ -32,6 +33,7 @@ import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; import org.logstash.ackedqueue.Settings; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.execution.QueueReadClientBase; import org.logstash.ext.JRubyAbstractQueueWriteClientExt; @@ -48,9 +50,10 @@ public final class JRubyWrappedAckedQueueExt extends AbstractWrappedQueueExt { private static final long serialVersionUID = 1L; private JRubyAckedQueueExt queue; + private QueueFactoryExt.BatchMetricMode batchMetricMode; - @JRubyMethod(required=1) - public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject settings) throws IOException { + @JRubyMethod(required=2) + public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject settings, IRubyObject batchMetricMode) throws IOException { if (!JavaUtil.isJavaObject(settings)) { // We should never get here, but previously had an initialize method // that took 7 technically-optional ordered parameters. @@ -60,18 +63,33 @@ public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject s settings.getClass().getName(), settings)); } - this.queue = JRubyAckedQueueExt.create(JavaUtil.unwrapJavaObject(settings)); + + Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + if (!JavaUtil.isJavaObject(batchMetricMode)) { + throw new IllegalArgumentException( + String.format( + "Failed to instantiate JRubyWrappedAckedQueueExt with <%s:%s>", + batchMetricMode.getClass().getName(), + batchMetricMode)); + } + + + Settings javaSettings = JavaUtil.unwrapJavaObject(settings); + this.queue = JRubyAckedQueueExt.create(javaSettings); + + this.batchMetricMode = JavaUtil.unwrapJavaObject(batchMetricMode); this.queue.open(); return this; } - public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings) throws IOException { - return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings); + public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { + return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings, batchMetricMode); } - public JRubyWrappedAckedQueueExt(Ruby runtime, RubyClass metaClass, Settings settings) throws IOException { + public JRubyWrappedAckedQueueExt(Ruby runtime, RubyClass metaClass, Settings settings, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { super(runtime, metaClass); + this.batchMetricMode = Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); this.queue = JRubyAckedQueueExt.create(settings); this.queue.open(); } @@ -111,7 +129,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co @Override protected QueueReadClientBase getReadClient() { - return JrubyAckedReadClientExt.create(queue); + return JrubyAckedReadClientExt.create(queue, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java index 7c5c47e3162..1e9eabf0a61 100644 --- a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java +++ b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java @@ -28,6 +28,8 @@ public class SettingKeyDefinitions { public static final String PIPELINE_WORKERS = "pipeline.workers"; + public static final String PIPELINE_BATCH_METRICS = "pipeline.batch.metrics.sampling_mode"; + public static final String PIPELINE_BATCH_SIZE = "pipeline.batch.size"; public static final String PATH_QUEUE = "path.queue"; diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index ec26b9b0735..ef12af32cd6 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -317,8 +317,7 @@ public final IRubyObject openQueue(final ThreadContext context) { new IRubyObject[]{ STATS_KEY, PIPELINES_KEY, - pipelineId.convertToString().intern(), - EVENTS_KEY + pipelineId.convertToString().intern() } ) ) @@ -585,11 +584,39 @@ public final IRubyObject initializeFlowMetrics(final ThreadContext context) { this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, utilizationFlow); storeMetric(context, flowNamespace, utilizationFlow); + // Batch average byte size and count metrics + if (isBatchMetricsEnabled(context)) { + initializeBatchMetrics(context); + } + initializePqFlowMetrics(context, flowNamespace, uptimeMetric); initializePluginFlowMetrics(context, uptimeMetric); return context.nil; } + private void initializeBatchMetrics(ThreadContext context) { + final RubySymbol[] batchNamespace = buildNamespace(BATCH_KEY, BATCH_EVENT_COUNT_KEY); + final LongCounter batchEventsInCounter = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_TOTAL_EVENTS); + final LongCounter batchCounter = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_COUNT); + final FlowMetric documentsPerBatch = createFlowMetric(BATCH_AVERAGE_KEY, batchEventsInCounter, batchCounter); + this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, documentsPerBatch); + storeMetric(context, batchNamespace, documentsPerBatch); + + final RubySymbol[] batchSizeNamespace = buildNamespace(BATCH_KEY, BATCH_BYTE_SIZE_KEY); + final LongCounter totalBytes = initOrGetCounterMetric(context, buildNamespace(BATCH_KEY), BATCH_TOTAL_BYTES); + final FlowMetric byteSizePerBatch = createFlowMetric(BATCH_AVERAGE_KEY, totalBytes, batchCounter); + this.scopedFlowMetrics.register(ScopedFlowMetrics.Scope.WORKER, byteSizePerBatch); + storeMetric(context, batchSizeNamespace, byteSizePerBatch); + } + + private boolean isBatchMetricsEnabled(ThreadContext context) { + IRubyObject pipelineBatchMetricsSetting = getSetting(context, "pipeline.batch.metrics.sampling_mode"); + return !pipelineBatchMetricsSetting.isNil() && + QueueFactoryExt.BatchMetricMode.valueOf( + pipelineBatchMetricsSetting.asJavaString().toUpperCase() + ) != QueueFactoryExt.BatchMetricMode.DISABLED; + } + @JRubyMethod(name = "collect_flow_metrics") public final IRubyObject collectFlowMetrics(final ThreadContext context) { this.scopedFlowMetrics.captureAll(); diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java index 535cd838a0e..aba1377ca5d 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBase.java @@ -32,15 +32,19 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricKeys; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.counter.LongCounter; import java.io.IOException; +import java.util.Objects; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.TimeUnit; +import static org.logstash.instrument.metrics.MetricKeys.EVENTS_KEY; + /** * Common code shared by Persistent and In-Memory queues clients implementation * */ @@ -60,9 +64,17 @@ public abstract class QueueReadClientBase extends RubyObject implements QueueRea private transient LongCounter pipelineMetricOut; private transient LongCounter pipelineMetricFiltered; private transient TimerMetric pipelineMetricTime; + private final transient QueueReadClientBatchMetrics batchMetrics; protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass) { + this(runtime, metaClass, QueueFactoryExt.BatchMetricMode.DISABLED); + } + + protected QueueReadClientBase(final Ruby runtime, final RubyClass metaClass, + final QueueFactoryExt.BatchMetricMode batchMetricMode) { super(runtime, metaClass); + Objects.requireNonNull(batchMetricMode, "batchMetricMode must not be null"); + this.batchMetrics = new QueueReadClientBatchMetrics(batchMetricMode); } @JRubyMethod(name = "inflight_batches") @@ -86,10 +98,13 @@ public IRubyObject setEventsMetric(final IRubyObject metric) { @JRubyMethod(name = "set_pipeline_metric") public IRubyObject setPipelineMetric(final IRubyObject metric) { final AbstractNamespacedMetricExt namespacedMetric = (AbstractNamespacedMetricExt) metric; + ThreadContext context = metric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt eventsNamespace = namespacedMetric.namespace(context, EVENTS_KEY); synchronized(namespacedMetric.getMetric()) { - pipelineMetricOut = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.OUT_KEY); - pipelineMetricFiltered = LongCounter.fromRubyBase(namespacedMetric, MetricKeys.FILTERED_KEY); - pipelineMetricTime = TimerMetric.fromRubyBase(namespacedMetric, MetricKeys.DURATION_IN_MILLIS_KEY); + pipelineMetricOut = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.OUT_KEY); + pipelineMetricFiltered = LongCounter.fromRubyBase(eventsNamespace, MetricKeys.FILTERED_KEY); + pipelineMetricTime = TimerMetric.fromRubyBase(eventsNamespace, MetricKeys.DURATION_IN_MILLIS_KEY); + batchMetrics.setupMetrics(namespacedMetric); } return this; } @@ -193,6 +208,7 @@ public void startMetrics(QueueBatch batch) { // JTODO getId has been deprecated in JDK 19, when JDK 21 is the target version use threadId() instead long threadId = Thread.currentThread().getId(); inflightBatches.put(threadId, batch); + batchMetrics.updateBatchMetrics(batch); } @Override diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java new file mode 100644 index 00000000000..3ed58daaef2 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java @@ -0,0 +1,76 @@ +package org.logstash.execution; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jruby.runtime.ThreadContext; +import org.logstash.ackedqueue.QueueFactoryExt; +import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.counter.LongCounter; + +import java.security.SecureRandom; + +import static org.logstash.instrument.metrics.MetricKeys.*; + +class QueueReadClientBatchMetrics { + + private static final Logger LOG = LogManager.getLogger(QueueReadClientBatchMetrics.class); + + private final QueueFactoryExt.BatchMetricMode batchMetricMode; + + private LongCounter pipelineMetricBatchCount; + private LongCounter pipelineMetricBatchByteSize; + private LongCounter pipelineMetricBatchTotalEvents; + private final SecureRandom random = new SecureRandom(); + + public QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode batchMetricMode) { + this.batchMetricMode = batchMetricMode; + } + + public void setupMetrics(AbstractNamespacedMetricExt namespacedMetric) { + LOG.debug("setupMetrics called with mode: {}", batchMetricMode); + ThreadContext context = namespacedMetric.getRuntime().getCurrentContext(); + AbstractNamespacedMetricExt batchNamespace = namespacedMetric.namespace(context, BATCH_KEY); + if (batchMetricMode != QueueFactoryExt.BatchMetricMode.DISABLED) { + pipelineMetricBatchCount = LongCounter.fromRubyBase(batchNamespace, BATCH_COUNT); + pipelineMetricBatchTotalEvents = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_EVENTS); + pipelineMetricBatchByteSize = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_BYTES); + } + } + + public void updateBatchMetrics(QueueBatch batch) { + if (batch.events().isEmpty()) { + // avoid to increment batch count for empty batches + return; + } + + if (batchMetricMode == QueueFactoryExt.BatchMetricMode.DISABLED) { + return; + } + + boolean updateMetric = true; + if (batchMetricMode == QueueFactoryExt.BatchMetricMode.MINIMAL) { + // 1% chance to update metric + updateMetric = random.nextInt(100) < 2; + } + + if (updateMetric) { + updateBatchSizeMetric(batch); + } + } + + private void updateBatchSizeMetric(QueueBatch batch) { + try { + // if an error occurs in estimating the size of the batch, no counter has to be updated + long totalSize = 0L; + for (JrubyEventExtLibrary.RubyEvent rubyEvent : batch.events()) { + totalSize += rubyEvent.getEvent().estimateMemory(); + } + pipelineMetricBatchCount.increment(); + pipelineMetricBatchTotalEvents.increment(batch.filteredSize()); + pipelineMetricBatchByteSize.increment(totalSize); + } catch (IllegalArgumentException e) { + LOG.error("Failed to calculate batch byte size for metrics", e); + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java index 273fa6cb2b2..3f4f1b62a44 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyAckedReadClientExt.java @@ -28,6 +28,7 @@ import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; import org.logstash.ackedqueue.AckedReadBatch; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.ackedqueue.ext.JRubyAckedQueueExt; import org.logstash.execution.QueueBatch; import org.logstash.execution.QueueReadClient; @@ -49,12 +50,12 @@ public final class JrubyAckedReadClientExt extends QueueReadClientBase implement public static JrubyAckedReadClientExt create(final ThreadContext context, final IRubyObject recv, final IRubyObject queue) { return new JrubyAckedReadClientExt( - context.runtime, RubyUtil.ACKED_READ_CLIENT_CLASS, queue + context.runtime, RubyUtil.ACKED_READ_CLIENT_CLASS, queue, QueueFactoryExt.BatchMetricMode.DISABLED ); } - public static JrubyAckedReadClientExt create(IRubyObject queue) { - return new JrubyAckedReadClientExt(RubyUtil.RUBY, RubyUtil.ACKED_READ_CLIENT_CLASS, queue); + public static JrubyAckedReadClientExt create(IRubyObject queue, QueueFactoryExt.BatchMetricMode batchMetricMode) { + return new JrubyAckedReadClientExt(RubyUtil.RUBY, RubyUtil.ACKED_READ_CLIENT_CLASS, queue, batchMetricMode); } public JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass) { @@ -62,8 +63,8 @@ public JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass) { } private JrubyAckedReadClientExt(final Ruby runtime, final RubyClass metaClass, - final IRubyObject queue) { - super(runtime, metaClass); + final IRubyObject queue, final QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass, batchMetricMode); this.queue = (JRubyAckedQueueExt)queue; } diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java index 0a93a347c4d..8485d0864d1 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyMemoryReadClientExt.java @@ -26,6 +26,7 @@ import org.jruby.RubyClass; import org.jruby.anno.JRubyClass; import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.common.LsQueueUtils; import org.logstash.execution.MemoryReadBatch; import org.logstash.execution.QueueBatch; @@ -47,8 +48,9 @@ public JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass) { @SuppressWarnings("rawtypes") private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, - BlockingQueue queue, int batchSize, int waitForMillis) { - super(runtime, metaClass); + BlockingQueue queue, int batchSize, int waitForMillis, + QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass, batchMetricMode); this.queue = queue; this.batchSize = batchSize; this.waitForNanos = TimeUnit.NANOSECONDS.convert(waitForMillis, TimeUnit.MILLISECONDS); @@ -58,8 +60,15 @@ private JrubyMemoryReadClientExt(final Ruby runtime, final RubyClass metaClass, @SuppressWarnings("rawtypes") public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, int waitForMillis) { + return create(queue, batchSize, waitForMillis, QueueFactoryExt.BatchMetricMode.DISABLED); + } + + @SuppressWarnings("rawtypes") + public static JrubyMemoryReadClientExt create(BlockingQueue queue, int batchSize, + int waitForMillis, + QueueFactoryExt.BatchMetricMode batchMetricMode) { return new JrubyMemoryReadClientExt(RubyUtil.RUBY, - RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis); + RubyUtil.MEMORY_READ_CLIENT_CLASS, queue, batchSize, waitForMillis, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java index dbbfb97de5b..ebebae00cbe 100644 --- a/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ext/JrubyWrappedSynchronousQueueExt.java @@ -20,6 +20,7 @@ package org.logstash.ext; +import java.util.Objects; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.BlockingQueue; @@ -28,8 +29,11 @@ import org.jruby.RubyNumeric; import org.jruby.anno.JRubyClass; import org.jruby.anno.JRubyMethod; +import org.jruby.javasupport.JavaUtil; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.execution.QueueReadClientBase; @@ -42,20 +46,44 @@ public final class JrubyWrappedSynchronousQueueExt extends AbstractWrappedQueueE private static final long serialVersionUID = 1L; private transient BlockingQueue queue; + private QueueFactoryExt.BatchMetricMode batchMetricMode; public JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + private JrubyWrappedSynchronousQueueExt(final Ruby runtime, final RubyClass metaClass, + int size, QueueFactoryExt.BatchMetricMode batchMetricMode) { + super(runtime, metaClass); + this.queue = new ArrayBlockingQueue<>(size); + this.batchMetricMode = Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + } + @JRubyMethod @SuppressWarnings("unchecked") public JrubyWrappedSynchronousQueueExt initialize(final ThreadContext context, - IRubyObject size) { + IRubyObject size, + IRubyObject batchMetricMode) { + if (!JavaUtil.isJavaObject(batchMetricMode)) { + throw new IllegalArgumentException( + String.format( + "Failed to instantiate JrubyWrappedSynchronousQueueExt with <%s:%s>", + batchMetricMode.getClass().getName(), + batchMetricMode)); + } + int typedSize = ((RubyNumeric)size).getIntValue(); this.queue = new ArrayBlockingQueue<>(typedSize); + Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); + this.batchMetricMode = JavaUtil.unwrapJavaObject(batchMetricMode); return this; } + public static JrubyWrappedSynchronousQueueExt create(final ThreadContext context, int size, + QueueFactoryExt.BatchMetricMode batchMetricMode) { + return new JrubyWrappedSynchronousQueueExt(context.runtime, RubyUtil.WRAPPED_SYNCHRONOUS_QUEUE_CLASS, size, batchMetricMode); + } + @Override protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext context) { return JrubyMemoryWriteClientExt.create(queue); @@ -65,7 +93,7 @@ protected JRubyAbstractQueueWriteClientExt getWriteClient(final ThreadContext co protected QueueReadClientBase getReadClient() { // batch size and timeout are currently hard-coded to 125 and 50ms as values observed // to be reasonable tradeoffs between latency and throughput per PR #8707 - return JrubyMemoryReadClientExt.create(queue, 125, 50); + return JrubyMemoryReadClientExt.create(queue, 125, 50, batchMetricMode); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index 2730ab83749..15f540b3dca 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -116,4 +116,19 @@ private MetricKeys() { public static final RubySymbol WRITES_IN_KEY = RubyUtil.RUBY.newSymbol("writes_in"); + // Batch metrics keys + public static final RubySymbol BATCH_EVENT_COUNT_KEY = RubyUtil.RUBY.newSymbol("event_count"); + + public static final RubySymbol BATCH_AVERAGE_KEY = RubyUtil.RUBY.newSymbol("average"); + + public static final RubySymbol BATCH_KEY = RubyUtil.RUBY.newSymbol("batch"); + + public static final RubySymbol BATCH_COUNT = RubyUtil.RUBY.newSymbol("count"); + + public static final RubySymbol BATCH_TOTAL_EVENTS = RubyUtil.RUBY.newSymbol("total_events"); + + public static final RubySymbol BATCH_TOTAL_BYTES = RubyUtil.RUBY.newSymbol("total_bytes"); + + public static final RubySymbol BATCH_BYTE_SIZE_KEY = RubyUtil.RUBY.newSymbol("byte_size"); + } diff --git a/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java b/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java new file mode 100644 index 00000000000..588064e41af --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/execution/QueueReadClientBatchMetricsTest.java @@ -0,0 +1,133 @@ +package org.logstash.execution; + +import static org.hamcrest.MatcherAssert.assertThat; +import org.jruby.RubyArray; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.junit.Before; +import org.junit.Test; +import org.logstash.Event; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; +import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.counter.LongCounter; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.List; + +import static org.junit.Assert.assertEquals; + +public class QueueReadClientBatchMetricsTest { + + public static final class MockQueueBatch implements QueueBatch { + + private final long processingTimeNanos; + private final List events; + + public MockQueueBatch(long processingTimeNanos, JrubyEventExtLibrary.RubyEvent... events) { + this.processingTimeNanos = processingTimeNanos; + this.events = Arrays.stream(events).toList(); + } + + @Override + @SuppressWarnings("unchecked") + public RubyArray to_a() { + List list = new ArrayList<>(events); + return (RubyArray) RubyUtil.RUBY.newArray(list); + } + + @Override + @SuppressWarnings("unchecked") + public Collection events() { + return to_a(); + } + + @Override + public void close() throws IOException { + // no-op + } + + @Override + public int filteredSize() { + return events.size(); + } + + public long getProcessingTimeNanos() { + return processingTimeNanos; + } + } + + private AbstractNamespacedMetricExt metric; + private QueueReadClientBatchMetrics sut; + private LongCounter batchCounter; + private LongCounter batchByteSizeCounter; + private JrubyEventExtLibrary.RubyEvent rubyEvent; + + @Before + public void setUp() { + metric = MockNamespacedMetric.create(); + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.FULL); + sut.setupMetrics(metric); + + ThreadContext context = metric.getRuntime().getCurrentContext(); + batchCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_COUNT); + batchByteSizeCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_TOTAL_BYTES); + + rubyEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + } + + @Test + public void givenEmptyBatchAndFullMetricsWhenUpdateBatchMetricsThenNoMetricsAreUpdated() { + QueueBatch emptyBatch = new MockQueueBatch(10); + + sut.updateBatchMetrics(emptyBatch); + + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyBatchAndFullMetricsWhenUpdateBatchMetricsThenMetricsAreUpdated() { + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + final long expectedBatchByteSize = rubyEvent.getEvent().estimateMemory(); + + sut.updateBatchMetrics(batch); + + assertEquals(1L, batchCounter.getValue().longValue()); + assertEquals(expectedBatchByteSize, batchByteSizeCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyBatchesAndMinimalMetricsThenMetricsAreUpdated() { + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.MINIMAL); + sut.setupMetrics(metric); + + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + final long expectedBatchByteSize = rubyEvent.getEvent().estimateMemory(); + + for (int i = 0; i < 200; i++) { + sut.updateBatchMetrics(batch); + } + sut.updateBatchMetrics(batch); + + assertThat(batchCounter.getValue(), org.hamcrest.Matchers.greaterThan(1L)); + assertThat(batchByteSizeCounter.getValue(), org.hamcrest.Matchers.greaterThan(expectedBatchByteSize)); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadAndMetricIsDisabledThenBatchCounterMetricIsNotUpdated() { + sut = new QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode.DISABLED); + sut.setupMetrics(metric); + QueueBatch batch = new MockQueueBatch(10, rubyEvent); + + sut.updateBatchMetrics(batch); + + assertEquals(0L, batchCounter.getValue().longValue()); + assertEquals(0L, batchByteSizeCounter.getValue().longValue()); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java index dff3c4845a3..386aa628f2b 100644 --- a/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java +++ b/logstash-core/src/test/java/org/logstash/ext/JrubyMemoryReadClientExtTest.java @@ -25,26 +25,50 @@ import java.util.concurrent.BlockingQueue; import org.jruby.RubyHash; import org.jruby.runtime.ThreadContext; +import org.junit.Before; import org.junit.Test; +import org.logstash.Event; import org.logstash.RubyTestBase; +import org.logstash.RubyUtil; +import org.logstash.ackedqueue.QueueFactoryExt; import org.logstash.execution.QueueBatch; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.MetricKeys; +import org.logstash.instrument.metrics.MockNamespacedMetric; +import org.logstash.instrument.metrics.counter.LongCounter; import static org.hamcrest.CoreMatchers.is; import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.assertEquals; /** * Tests for {@link JrubyMemoryReadClientExt}. */ public final class JrubyMemoryReadClientExtTest extends RubyTestBase { + private JrubyEventExtLibrary.RubyEvent testEvent; + private BlockingQueue queue; + private AbstractNamespacedMetricExt metric; + private LongCounter batchCounter; + private LongCounter batchByteSizeCounter; + + @Before + public void setUp() { + testEvent = JrubyEventExtLibrary.RubyEvent.newRubyEvent(RubyUtil.RUBY, new Event()); + queue = new ArrayBlockingQueue<>(10); + metric = MockNamespacedMetric.create(); + ThreadContext context = metric.getRuntime().getCurrentContext(); + batchCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_COUNT); + batchByteSizeCounter = LongCounter.fromRubyBase(metric.namespace(context, MetricKeys.BATCH_KEY), MetricKeys.BATCH_TOTAL_BYTES); + } + @Test @SuppressWarnings("deprecation") public void testInflightBatchesTracking() throws InterruptedException, IOException { - final BlockingQueue queue = - new ArrayBlockingQueue<>(10); final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50); final ThreadContext context = client.getRuntime().getCurrentContext(); + client.setPipelineMetric(metric); final QueueBatch batch = client.readBatch(); final RubyHash inflight = client.rubyGetInflightBatches(context); assertThat(inflight.size(), is(1)); @@ -53,4 +77,55 @@ public void testInflightBatchesTracking() throws InterruptedException, IOExcepti client.closeBatch(batch); assertThat(client.rubyGetInflightBatches(context).size(), is(0)); } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadThenBatchCounterMetricIsUpdated() throws InterruptedException { + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(1L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadAndMetricIsDisabledThenBatchCounterMetricIsNotUpdated() throws InterruptedException { + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.DISABLED); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenEmptyQueueWhenEmptyBatchIsReadAndMetricIsFullyCollectedThenBatchCounterMetricIsNotUpdated() throws InterruptedException { + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(0, batch.filteredSize()); + assertEquals(0L, batchCounter.getValue().longValue()); + } + + @Test + public void givenNonEmptyQueueWhenBatchIsReadThenBatchByteSizeMetricIsUpdated() throws InterruptedException { + final long expectedBatchByteSize = testEvent.getEvent().estimateMemory(); + queue.add(testEvent); + + final JrubyMemoryReadClientExt client = JrubyMemoryReadClientExt.create(queue, 5, 50, + QueueFactoryExt.BatchMetricMode.FULL); + client.setPipelineMetric(metric); + + final QueueBatch batch = client.readBatch(); + assertEquals(1, batch.filteredSize()); + assertEquals(expectedBatchByteSize, batchByteSizeCounter.getValue().longValue()); + } } diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java new file mode 100644 index 00000000000..91ea0e49d01 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -0,0 +1,101 @@ +package org.logstash.instrument.metrics; + +import org.jruby.Ruby; +import org.jruby.RubyArray; +import org.jruby.RubyClass; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; +import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +/** + * Trivial implementation of AbstractNamespacedMetricExt where each abstract creation + * metric is implemented by pooling metric instances by name. + * */ +@SuppressWarnings({"rawtypes", "serializable"}) +public class MockNamespacedMetric extends AbstractNamespacedMetricExt { + + private static final long serialVersionUID = -6507123659910450215L; + + private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); + + public static MockNamespacedMetric create() { + return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); + } + + MockNamespacedMetric(final Ruby runtime, final RubyClass metaClass) { + super(runtime, metaClass); + } + + @Override + protected IRubyObject getGauge(ThreadContext context, IRubyObject key, IRubyObject value) { + return null; + } + + @Override + protected RubyArray getNamespaceName(ThreadContext context) { + return null; + } + + @Override + protected IRubyObject getCounter(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LongCounter::new)); + } + + @Override + protected IRubyObject getTimer(ThreadContext context, IRubyObject key) { + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), TimerMetric::create)); + } + + @Override + protected IRubyObject doTime(ThreadContext context, IRubyObject key, Block block) { + return null; + } + + @Override + protected IRubyObject doReportTime(ThreadContext context, IRubyObject key, IRubyObject duration) { + return null; + } + + @Override + protected IRubyObject doIncrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + protected IRubyObject doDecrement(ThreadContext context, IRubyObject[] args) { + return null; + } + + @Override + public AbstractMetricExt getMetric() { + return NullMetricExt.create(); + } + + @Override + protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { + return this; + } + + @Override + protected IRubyObject getCollector(ThreadContext context) { + return null; + } + + private static void requireRubySymbol(IRubyObject value, String paramName) { + if (!(value instanceof RubySymbol)) { + throw new IllegalArgumentException(paramName + " must be a RubySymbol instead was: " + value.getClass()); + } + } +} \ No newline at end of file diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index 19dabc16f0d..a77c113ca78 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -240,6 +240,70 @@ end end + context "when pipeline.batch.metrics.sampling_mode is set to 'full'" do + let(:settings_overrides) do + super().merge({'pipeline.batch.metrics.sampling_mode' => 'full'}) + end + + it "can retrieve batch stats" do + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + batch_stats = result.fetch("pipelines").fetch(pipeline_id).fetch("batch") + expect(batch_stats).not_to be_nil + + expect(batch_stats["event_count"]).not_to be_nil + expect(batch_stats["event_count"]["average"]).not_to be_nil + expect(batch_stats["event_count"]["average"]["lifetime"]).not_to be_nil + expect(batch_stats["event_count"]["average"]["lifetime"]).to be_a_kind_of(Numeric) + expect(batch_stats["event_count"]["average"]["lifetime"]).to be > 0 + + expect(batch_stats["byte_size"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]["lifetime"]).not_to be_nil + expect(batch_stats["byte_size"]["average"]["lifetime"]).to be_a_kind_of(Numeric) + expect(batch_stats["byte_size"]["average"]["lifetime"]).to be > 0 + end + end + end + + context "when pipeline.batch.metrics.sampling_mode is set to 'disabled'" do + let(:settings_overrides) do + super().merge({'pipeline.batch.metrics.sampling_mode' => 'disabled'}) + end + + it "no batch stats metrics are available" do + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_stats can fail if the stats subsystem isn't ready + result = logstash_service.monitoring_api.node_stats rescue nil + expect(result).not_to be_nil + # we use fetch here since we want failed fetches to raise an exception + # and trigger the retry block + pipeline_stats = result.fetch("pipelines").fetch(pipeline_id) + expect(pipeline_stats).not_to include("batch") + end + end + end + it "retrieves the pipeline flow statuses" do logstash_service = @fixture.get_service("logstash") logstash_service.start_with_stdin From 6803cc39a9b11dc8a6d6c189c9585cd2eabef724 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 23 Sep 2025 18:00:27 +0200 Subject: [PATCH 16/44] Remove uncommented line erroneously committed. (#18204) --- config/logstash.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/config/logstash.yml b/config/logstash.yml index 9658274fa64..472c421b7cf 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -54,7 +54,6 @@ # Default is "minimal". # # pipeline.batch.metrics.sampling_mode: "minimal" -pipeline.batch.metrics.sampling_mode: minimal # # Force Logstash to exit during shutdown even if there are still inflight # events in memory. By default, logstash will refuse to quit until all From 9554ec48e6d8c1a9f86a35923f1b71ab5cbdc075 Mon Sep 17 00:00:00 2001 From: Rob Bavey Date: Tue, 23 Sep 2025 15:03:37 -0400 Subject: [PATCH 17/44] Update logstash_project_board.yml (#18116) Send to Ingest->Logstash Project Board --- .github/workflows/logstash_project_board.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/logstash_project_board.yml b/.github/workflows/logstash_project_board.yml index 48858610f53..7a62e6416d0 100644 --- a/.github/workflows/logstash_project_board.yml +++ b/.github/workflows/logstash_project_board.yml @@ -18,6 +18,6 @@ jobs: clientMutationId } } - projectid: "PVT_kwDOAGc3Zs0SEg" + projectid: "PVT_kwDOAGc3Zs4AMlnl" contentid: ${{ github.event.issue.node_id }} GITHUB_TOKEN: ${{ secrets.PROJECT_TOKEN }} From 9f9ee562c8d95131223aa2b57465d67bbdeb65f8 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Tue, 23 Sep 2025 12:04:26 -0700 Subject: [PATCH 18/44] GH action for updating logstash version (#18035) * GH action for updating logstash version This commit adds a GH action for bumping the logstash version. It uses the updatescli based on a suggestion for how the beats team is doing file updates. There is a workflow_dispatch trigger that accepts a new logstash version. Both the versions.yml and lock file are updated. If it is a branch where we are not vendoring a lockfile that file is skipped without failing the workflow step. * use stub workflow to test * Separate step when modifying * dont track temp updatecli file and install action * restore stub * Refactor updatecli workflow use pattern suggested in code review * Use test stub to validate changes * fix inputs section * fix branch * dont fail if file is not present * try with no global conditionals * file condition * name conditional * split into separate pipeline files * remove unused scmid * no explicit pr action? * add back PR * try sources pattern * explicitly connect target with scmid * ensure lock file lookup executes from checkout * Cleanup * test * restore LOGSTASH_BRANCH * codereview feedback --- .ci/updatecli/bump-logstash-version.yml | 81 +++++++++++++++++++++++++ .ci/updatecli/values.d/scm.yml | 3 + .github/workflows/bump-logstash.yml | 31 ++++++++++ .updatecli-version | 1 + 4 files changed, 116 insertions(+) create mode 100644 .ci/updatecli/bump-logstash-version.yml create mode 100644 .ci/updatecli/values.d/scm.yml create mode 100644 .github/workflows/bump-logstash.yml create mode 100644 .updatecli-version diff --git a/.ci/updatecli/bump-logstash-version.yml b/.ci/updatecli/bump-logstash-version.yml new file mode 100644 index 00000000000..a100a19aeb8 --- /dev/null +++ b/.ci/updatecli/bump-logstash-version.yml @@ -0,0 +1,81 @@ +--- +name: Update logstash version files +pipelineid: "logstash/version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "LOGSTASH_BRANCH" }}' + commitusingapi: true + force: false + +actions: + default: + title: 'Bump logstash version {{ requiredEnv "LOGSTASH_VERSION" }}' + kind: github/pullrequest + scmid: default + spec: + automerge: false + labels: + - automation + description: |- + ### What + Update logstash version + +sources: + lock_file_exists: + kind: shell + scmid: default + spec: + command: test -f Gemfile.jruby-3.1.lock.release + +targets: + update_logstash_version: + name: Update logstash version in versions.yml + kind: yaml + disablesourceinput: true + scmid: default + spec: + file: versions.yml + key: $.logstash + value: '{{ requiredEnv "LOGSTASH_VERSION" }}' + + update_logstash_core_version: + name: Update logstash-core version in versions.yml + kind: yaml + disablesourceinput: true + scmid: default + spec: + file: versions.yml + key: $.logstash-core + value: '{{ requiredEnv "LOGSTASH_VERSION" }}' + + update_gemfile_lock_dependency: + name: Update logstash-core dependency in lockfile + kind: file + disablesourceinput: true + scmid: default + dependson: + - 'source#lock_file_exists' + spec: + file: Gemfile.jruby-3.1.lock.release + matchpattern: 'logstash-core \(= [0-9]+\.[0-9]+\.[0-9]+' + replacepattern: 'logstash-core (= {{ requiredEnv "LOGSTASH_VERSION" }}' + + update_gemfile_lock_spec: + name: Update logstash-core spec in lockfile + kind: file + disablesourceinput: true + scmid: default + dependson: + - 'source#lock_file_exists' + spec: + file: Gemfile.jruby-3.1.lock.release + matchpattern: 'logstash-core \([0-9]+\.[0-9]+\.[0-9]+-java\)' + replacepattern: 'logstash-core ({{ requiredEnv "LOGSTASH_VERSION" }}-java)' \ No newline at end of file diff --git a/.ci/updatecli/values.d/scm.yml b/.ci/updatecli/values.d/scm.yml new file mode 100644 index 00000000000..62b6e5c249e --- /dev/null +++ b/.ci/updatecli/values.d/scm.yml @@ -0,0 +1,3 @@ +scm: + owner: elastic + repository: logstash diff --git a/.github/workflows/bump-logstash.yml b/.github/workflows/bump-logstash.yml new file mode 100644 index 00000000000..6679f1ed618 --- /dev/null +++ b/.github/workflows/bump-logstash.yml @@ -0,0 +1,31 @@ +name: bump-logstash-version + +on: + workflow_dispatch: + inputs: + logstash_version: + description: 'Logstash version (example: 9.1.4)' + required: true + type: string + logstash_branch: + description: 'Logstash branch (example: 9.1)' + required: true + type: string + +jobs: + bump: + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/bump-logstash-version.yml --values .ci/updatecli/values.d/scm.yml + version-file: .updatecli-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOGSTASH_BRANCH: "${{ github.event.inputs.logstash_branch }}" + LOGSTASH_VERSION: "${{ github.event.inputs.logstash_version }}" diff --git a/.updatecli-version b/.updatecli-version new file mode 100644 index 00000000000..5c5d5571187 --- /dev/null +++ b/.updatecli-version @@ -0,0 +1 @@ +v0.104.0 \ No newline at end of file From 50fde260e32184d07444a34ab6604656a843c19d Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Wed, 24 Sep 2025 10:05:40 +0200 Subject: [PATCH 19/44] Cover the warn password policy usage of ValidatedPassword setting (#18203) --- logstash-core/spec/logstash/settings_spec.rb | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/logstash-core/spec/logstash/settings_spec.rb b/logstash-core/spec/logstash/settings_spec.rb index 60f7d8848a3..bc1b704c4cb 100644 --- a/logstash-core/spec/logstash/settings_spec.rb +++ b/logstash-core/spec/logstash/settings_spec.rb @@ -299,6 +299,25 @@ expect(LogStash::Setting::ValidatedPassword.new("test.validated.password", password, password_policies)).to_not be_nil end end + + context "containing a mode WARN policy" do + before :each do + # Needs to mock the logger method at LogStash::Settings instead of LogStash::Setting::ValidatedPassword + # else the LOGGABLE_PROXY hide the mock itself. + allow(LogStash::Settings).to receive(:logger).at_least(:once).and_return(mock_logger) + allow(mock_logger).to receive(:warn) + end + let(:mock_logger) { double("logger") } + let(:password_policies) { super().merge({ "mode": "WARN" }) } + + it "logs a warning on validation failure" do + password = LogStash::Util::Password.new("Password!") + expect { + LogStash::Setting::ValidatedPassword.new("test.validated.password", password, password_policies) + }.not_to raise_error + expect(mock_logger).to have_received(:warn).with(a_string_including("Password must contain at least one digit between 0 and 9.")) + end + end end context "placeholders in nested logstash.yml" do From 62bf9b3fd657cbb20266a23da8a35c572cd9e46a Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Wed, 24 Sep 2025 13:47:14 -0700 Subject: [PATCH 20/44] Replace buildkite jdk version check w/GH action (#17945) * Replace buildkite jdk version check w/GH action Previously a weekly buildkite job would check if there is a newer JDK available than the one vendored in logstash. This job would fail until someone manually updated the JDK version in the `versions.yml` file in logstash. This commit reduces manual intervention by moving the responsibility to a GH action where there are patterns for creating PRs back to logstash. The action should run weekly and also be available in the actions UI. It is locked down with the same permissions as the version bump action. * Refactor to use updatecli * Remove gradle task in favor of updatecli * remove unused file * Dynamically generate java version lookup * make pipeline name unique' ' --- .../jdk_availability_check_pipeline.yml | 14 ----- .ci/updatecli/bump-java-version.yml | 56 +++++++++++++++++ .github/workflows/bump-java-version.yml | 30 ++++++---- build.gradle | 24 -------- catalog-info.yaml | 60 ------------------- ci/check_jdk_version_availability.sh | 7 --- 6 files changed, 74 insertions(+), 117 deletions(-) delete mode 100644 .buildkite/jdk_availability_check_pipeline.yml create mode 100644 .ci/updatecli/bump-java-version.yml delete mode 100755 ci/check_jdk_version_availability.sh diff --git a/.buildkite/jdk_availability_check_pipeline.yml b/.buildkite/jdk_availability_check_pipeline.yml deleted file mode 100644 index 3fa826564e0..00000000000 --- a/.buildkite/jdk_availability_check_pipeline.yml +++ /dev/null @@ -1,14 +0,0 @@ -steps: - - label: "JDK Availability check" - key: "jdk-availability-check" - agents: - image: "docker.elastic.co/ci-agent-images/platform-ingest/buildkite-agent-logstash-ci" - cpu: "4" - memory: "6Gi" - ephemeralStorage: "100Gi" - command: | - set -euo pipefail - - source .buildkite/scripts/common/container-agent.sh - export GRADLE_OPTS="-Xmx2g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info" - ci/check_jdk_version_availability.sh \ No newline at end of file diff --git a/.ci/updatecli/bump-java-version.yml b/.ci/updatecli/bump-java-version.yml new file mode 100644 index 00000000000..62f48760d33 --- /dev/null +++ b/.ci/updatecli/bump-java-version.yml @@ -0,0 +1,56 @@ +--- +name: Update java version file +pipelineid: "logstash/jdk-version-updates-{{ requiredEnv "LOGSTASH_BRANCH" }}" + +scms: + default: + kind: github + spec: + user: '{{ requiredEnv "GITHUB_ACTOR" }}' + username: '{{ requiredEnv "GITHUB_ACTOR" }}' + owner: '{{ .scm.owner }}' + repository: '{{ .scm.repository }}' + token: '{{ requiredEnv "GITHUB_TOKEN" }}' + branch: '{{ requiredEnv "LOGSTASH_BRANCH" }}' + commitusingapi: true + force: false + +sources: + jdk_major: + kind: yaml + spec: + file: "versions.yml" + key: "$.bundled_jdk.revision" + transformers: + - findsubmatch: + pattern: '^(\d+)\.\d+\.\d+$' + captureindex: 1 + + latest_jdk_version: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' + key: 'version' + + latest_jdk_build: + kind: json + spec: + file: 'https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_{{ source "jdk_major" }}_linux' + key: 'revision' + +targets: + update_jdk_revision: + name: "Update JDK revision" + kind: yaml + sourceid: latest_jdk_version + spec: + file: versions.yml + key: $.bundled_jdk.revision + + update_jdk_build: + name: "Update JDK build" + kind: yaml + sourceid: latest_jdk_build + spec: + file: versions.yml + key: $.bundled_jdk.build \ No newline at end of file diff --git a/.github/workflows/bump-java-version.yml b/.github/workflows/bump-java-version.yml index ac66267e441..818faf236f0 100644 --- a/.github/workflows/bump-java-version.yml +++ b/.github/workflows/bump-java-version.yml @@ -1,19 +1,25 @@ -name: Stub GH action for devoping new workflows [STUB] +name: bump-java-version + on: + schedule: + # Run weekly on Mondays at midnight UTC + - cron: '0 0 * * 1' workflow_dispatch: - pull_request: - types: [opened, synchronize, reopened] -permissions: - pull-requests: write - contents: write jobs: - stub_job_name: - name: Stub Job + bump: + permissions: + contents: write + pull-requests: write runs-on: ubuntu-latest steps: - - name: Stub step - run: | - echo "Stub to iterate via PR" - \ No newline at end of file + - uses: actions/checkout@v5 + + - uses: elastic/oblt-actions/updatecli/run@v1 + with: + command: apply --config .ci/updatecli/bump-java-version.yml --values .ci/updatecli/values.d/scm.yml + version-file: .updatecli-version + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + LOGSTASH_BRANCH: ${{ github.ref_name }} diff --git a/build.gradle b/build.gradle index b6987700c46..923bdc49151 100644 --- a/build.gradle +++ b/build.gradle @@ -748,23 +748,6 @@ class JDKDetails { return createElasticCatalogDownloadUrl() } - // throws an error iff local version in versions.yml doesn't match the latest from JVM catalog. - void checkLocalVersionMatchingLatest() { - // retrieve the metadata from remote - def url = "https://jvm-catalog.elastic.co/jdk/latest_adoptiumjdk_${major}_${osName}" - def catalogMetadataUrl = URI.create(url).toURL() - def catalogConnection = catalogMetadataUrl.openConnection() - catalogConnection.requestMethod = 'GET' - assert catalogConnection.responseCode == 200 - - def metadataRetrieved = catalogConnection.content.text - def catalogMetadata = new JsonSlurper().parseText(metadataRetrieved) - - if (catalogMetadata.version != revision || catalogMetadata.revision != build) { - throw new GradleException("Found new jdk version. Please update version.yml to ${catalogMetadata.version} build ${catalogMetadata.revision}") - } - } - private String createElasticCatalogDownloadUrl() { // Ask details to catalog https://jvm-catalog.elastic.co/jdk and return the url to download the JDK @@ -874,13 +857,6 @@ tasks.register("downloadJdk", Download) { } } -tasks.register("checkNewJdkVersion") { - // use Linux x86_64 as canary platform - def jdkDetails = new JDKDetails(gradle.ext.versions.bundled_jdk, "linux", "x86_64") - // throws Gradle exception if local and remote doesn't match - jdkDetails.checkLocalVersionMatchingLatest() -} - tasks.register("deleteLocalJdk", Delete) { // CLI project properties: -Pjdk_bundle_os=[windows|linux|darwin] String osName = selectOsType() diff --git a/catalog-info.yaml b/catalog-info.yaml index b1fada97a77..27350e8ef76 100644 --- a/catalog-info.yaml +++ b/catalog-info.yaml @@ -33,7 +33,6 @@ spec: - resource:logstash-windows-jdk-matrix-pipeline - resource:logstash-benchmark-pipeline - resource:logstash-health-report-tests-pipeline - - resource:logstash-jdk-availability-check-pipeline # *********************************** # Declare serverless IT pipeline @@ -795,62 +794,3 @@ spec: branch: main cronline: 30 20 * * * message: Daily trigger of Health Report Tests Pipeline - -# ******************************* -# SECTION END: Health Report Tests pipeline -# ******************************* - -# *********************************** -# Declare JDK check pipeline -# *********************************** ---- -# yaml-language-server: $schema=https://gist.githubusercontent.com/elasticmachine/988b80dae436cafea07d9a4a460a011d/raw/rre.schema.json -apiVersion: backstage.io/v1alpha1 -kind: Resource -metadata: - name: logstash-jdk-availability-check-pipeline - description: ":logstash: check availability of new JDK version" -spec: - type: buildkite-pipeline - owner: group:logstash - system: platform-ingest - implementation: - apiVersion: buildkite.elastic.dev/v1 - kind: Pipeline - metadata: - name: logstash-jdk-availability-check-pipeline - spec: - repository: elastic/logstash - pipeline_file: ".buildkite/jdk_availability_check_pipeline.yml" - maximum_timeout_in_minutes: 10 - provider_settings: - trigger_mode: none # don't trigger jobs from github activity - env: - ELASTIC_SLACK_NOTIFICATIONS_ENABLED: 'true' - SLACK_NOTIFICATIONS_CHANNEL: '#logstash-build' - SLACK_NOTIFICATIONS_ON_SUCCESS: 'false' - SLACK_NOTIFICATIONS_SKIP_FOR_RETRIES: 'true' - teams: - logstash: - access_level: MANAGE_BUILD_AND_READ - ingest-eng-prod: - access_level: MANAGE_BUILD_AND_READ - everyone: - access_level: READ_ONLY - schedules: - Weekly JDK availability check (main): - branch: main - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - Weekly JDK availability check (8.19): - branch: "8.19" - cronline: 0 2 * * 1 # every Monday@2AM UTC - message: Weekly trigger of JDK update availability pipeline per branch - env: - PIPELINES_TO_TRIGGER: 'logstash-jdk-availability-check-pipeline' - -# ******************************* -# SECTION END: JDK check pipeline -# ******************************* diff --git a/ci/check_jdk_version_availability.sh b/ci/check_jdk_version_availability.sh deleted file mode 100755 index 2ce40dc7b2f..00000000000 --- a/ci/check_jdk_version_availability.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -export GRADLE_OPTS="-Xmx4g -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" - -echo "Checking local JDK version against latest remote from JVM catalog" -./gradlew checkNewJdkVersion \ No newline at end of file From 21ec30c35fa4f2d02f9d5ae4c534ee6184405d29 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Fri, 26 Sep 2025 09:21:23 +0200 Subject: [PATCH 21/44] Implements current batch event count and byte size metrics (#18160) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exposes the current batch size in terms of events and estimated memory consumption. Introduce a new gauge metric to collect list of values, used a a couple, the first element count of events in the batch and the second one is the estimated memory occupation of the batch. This list is later grabbed in the API layer to populate the two current values for `event_count` and `batch_size`. Co-authored-by: João Duarte --- .../lib/logstash/api/commands/stats.rb | 6 +++++ .../logstash/api/modules/node_stats_spec.rb | 2 ++ .../execution/AbstractPipelineExt.java | 1 + .../QueueReadClientBatchMetrics.java | 18 ++++++++++----- .../instrument/metrics/MetricExt.java | 6 ++--- .../instrument/metrics/MetricKeys.java | 2 ++ .../metrics/NamespacedMetricExt.java | 5 ++++- .../metrics/gauge/LazyDelegatingGauge.java | 22 +++++++++++++++++++ .../metrics/MockNamespacedMetric.java | 5 ++++- qa/integration/specs/monitoring_api_spec.rb | 6 +++++ 10 files changed, 62 insertions(+), 11 deletions(-) diff --git a/logstash-core/lib/logstash/api/commands/stats.rb b/logstash-core/lib/logstash/api/commands/stats.rb index 2ec44fe0afa..7ae782fadf1 100644 --- a/logstash-core/lib/logstash/api/commands/stats.rb +++ b/logstash-core/lib/logstash/api/commands/stats.rb @@ -173,14 +173,20 @@ def plugin_stats(stats, plugin_type) end def refine_batch_metrics(stats) + # current is a tuple of [event_count, byte_size] store the reference locally to avoid repeatedly + # reading and retrieve unrelated values + current_data_point = stats[:batch][:current] { :event_count => { + # current_data_point is an instance of org.logstash.instrument.metrics.gauge.LazyDelegatingGauge so need to invoke getValue() to obtain the actual value + :current => current_data_point.value[0], :average => { # average return a FlowMetric which and we need to invoke getValue to obtain the map with metric details. :lifetime => stats[:batch][:event_count][:average].value["lifetime"] ? stats[:batch][:event_count][:average].value["lifetime"].round : 0 } }, :byte_size => { + :current => current_data_point.value[1], :average => { :lifetime => stats[:batch][:byte_size][:average].value["lifetime"] ? stats[:batch][:byte_size][:average].value["lifetime"].round : 0 } diff --git a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb index 71468fdd63e..136c1349dbf 100644 --- a/logstash-core/spec/logstash/api/modules/node_stats_spec.rb +++ b/logstash-core/spec/logstash/api/modules/node_stats_spec.rb @@ -150,11 +150,13 @@ }, "batch" => { "event_count" => { + "current" => Numeric, "average" => { "lifetime" => Numeric } }, "byte_size" => { + "current" => Numeric, "average" => { "lifetime" => Numeric } diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index ef12af32cd6..5bf98abff19 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -92,6 +92,7 @@ import org.logstash.instrument.metrics.MetricType; import org.logstash.instrument.metrics.NullMetricExt; import org.logstash.instrument.metrics.UpScaledMetric; +import org.logstash.instrument.metrics.gauge.TextGauge; import org.logstash.instrument.metrics.timer.TimerMetric; import org.logstash.instrument.metrics.UptimeMetric; import org.logstash.instrument.metrics.counter.LongCounter; diff --git a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java index 3ed58daaef2..a91cdf5dedf 100644 --- a/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java +++ b/logstash-core/src/main/java/org/logstash/execution/QueueReadClientBatchMetrics.java @@ -7,8 +7,10 @@ import org.logstash.ext.JrubyEventExtLibrary; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.gauge.LazyDelegatingGauge; import java.security.SecureRandom; +import java.util.Arrays; import static org.logstash.instrument.metrics.MetricKeys.*; @@ -22,6 +24,7 @@ class QueueReadClientBatchMetrics { private LongCounter pipelineMetricBatchByteSize; private LongCounter pipelineMetricBatchTotalEvents; private final SecureRandom random = new SecureRandom(); + private LazyDelegatingGauge currentBatchDimensions; public QueueReadClientBatchMetrics(QueueFactoryExt.BatchMetricMode batchMetricMode) { this.batchMetricMode = batchMetricMode; @@ -35,16 +38,18 @@ public void setupMetrics(AbstractNamespacedMetricExt namespacedMetric) { pipelineMetricBatchCount = LongCounter.fromRubyBase(batchNamespace, BATCH_COUNT); pipelineMetricBatchTotalEvents = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_EVENTS); pipelineMetricBatchByteSize = LongCounter.fromRubyBase(batchNamespace, BATCH_TOTAL_BYTES); + currentBatchDimensions = LazyDelegatingGauge.fromRubyBase(batchNamespace, BATCH_CURRENT_KEY); } } public void updateBatchMetrics(QueueBatch batch) { - if (batch.events().isEmpty()) { - // avoid to increment batch count for empty batches + if (batchMetricMode == QueueFactoryExt.BatchMetricMode.DISABLED) { return; } - if (batchMetricMode == QueueFactoryExt.BatchMetricMode.DISABLED) { + if (batch.events().isEmpty()) { + // don't update averages for empty batches, but set current back to zero + currentBatchDimensions.set(Arrays.asList(0L, 0L)); return; } @@ -62,13 +67,14 @@ public void updateBatchMetrics(QueueBatch batch) { private void updateBatchSizeMetric(QueueBatch batch) { try { // if an error occurs in estimating the size of the batch, no counter has to be updated - long totalSize = 0L; + long totalByteSize = 0L; for (JrubyEventExtLibrary.RubyEvent rubyEvent : batch.events()) { - totalSize += rubyEvent.getEvent().estimateMemory(); + totalByteSize += rubyEvent.getEvent().estimateMemory(); } pipelineMetricBatchCount.increment(); pipelineMetricBatchTotalEvents.increment(batch.filteredSize()); - pipelineMetricBatchByteSize.increment(totalSize); + pipelineMetricBatchByteSize.increment(totalByteSize); + currentBatchDimensions.set(Arrays.asList(batch.filteredSize(), totalByteSize)); } catch (IllegalArgumentException e) { LOG.error("Failed to calculate batch byte size for metrics", e); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java index 1303e1a753a..f842500e19d 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java @@ -44,15 +44,15 @@ public final class MetricExt extends AbstractSimpleMetricExt { private static final long serialVersionUID = 1L; - public static final RubySymbol COUNTER = RubyUtil.RUBY.newSymbol("counter"); + // These two metric type symbols need to be package-private because used in NamespacedMetricExt + static final RubySymbol COUNTER = RubyUtil.RUBY.newSymbol("counter"); + static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubyFixnum ONE = RubyUtil.RUBY.newFixnum(1); private static final RubySymbol INCREMENT = RubyUtil.RUBY.newSymbol("increment"); private static final RubySymbol DECREMENT = RubyUtil.RUBY.newSymbol("decrement"); - - private static final RubySymbol GAUGE = RubyUtil.RUBY.newSymbol("gauge"); private static final RubySymbol TIMER = RubyUtil.RUBY.newSymbol("timer"); private static final RubySymbol SET = RubyUtil.RUBY.newSymbol("set"); private static final RubySymbol GET = RubyUtil.RUBY.newSymbol("get"); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java index 15f540b3dca..e3043517282 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricKeys.java @@ -131,4 +131,6 @@ private MetricKeys() { public static final RubySymbol BATCH_BYTE_SIZE_KEY = RubyUtil.RUBY.newSymbol("byte_size"); + public static final RubySymbol BATCH_CURRENT_KEY = RubyUtil.RUBY.newSymbol("current"); + } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java index 8a77b4e2f03..388cfa7de7b 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java @@ -76,7 +76,10 @@ protected IRubyObject getCounter(final ThreadContext context, final IRubyObject @Override protected IRubyObject getGauge(final ThreadContext context, final IRubyObject key, final IRubyObject value) { - return metric.gauge(context, namespaceName, key, value); + metric.gauge(context, namespaceName, key, value); + return collector(context).callMethod( + context, "get", new IRubyObject[]{namespaceName, key, MetricExt.GAUGE} + ); } @Override diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java index cf9f30ad8bf..10c10fceef5 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/gauge/LazyDelegatingGauge.java @@ -23,10 +23,16 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jruby.RubyHash; +import org.jruby.RubySymbol; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; import org.logstash.ext.JrubyTimestampExtLibrary.RubyTimestamp; import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; import org.logstash.instrument.metrics.MetricType; +import java.util.Arrays; import java.util.List; import java.util.Optional; @@ -39,11 +45,27 @@ public class LazyDelegatingGauge extends AbstractMetric implements Gauge private static final Logger LOGGER = LogManager.getLogger(LazyDelegatingGauge.class); + public static final LazyDelegatingGauge DUMMY_GAUGE = new LazyDelegatingGauge("dummy"); + protected final String key; @SuppressWarnings("rawtypes") private GaugeMetric lazyMetric; + + public static LazyDelegatingGauge fromRubyBase(final AbstractNamespacedMetricExt metric, final RubySymbol key) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + // just initialize an empty gauge + final IRubyObject gauge = metric.gauge(context, key, context.runtime.newArray(context.runtime.newString("undefined"), context.runtime.newString("undefined"))); + final LazyDelegatingGauge javaGauge; + if (LazyDelegatingGauge.class.isAssignableFrom(gauge.getJavaClass())) { + javaGauge = gauge.toJava(LazyDelegatingGauge.class); + } else { + javaGauge = DUMMY_GAUGE; + } + return javaGauge; + } + /** * Constructor - null initial value * diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java index 91ea0e49d01..2f784fa9119 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -9,6 +9,7 @@ import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; import org.logstash.instrument.metrics.counter.LongCounter; +import org.logstash.instrument.metrics.gauge.LazyDelegatingGauge; import org.logstash.instrument.metrics.timer.TimerMetric; import java.util.Objects; @@ -36,7 +37,9 @@ public static MockNamespacedMetric create() { @Override protected IRubyObject getGauge(ThreadContext context, IRubyObject key, IRubyObject value) { - return null; + Objects.requireNonNull(key); + requireRubySymbol(key, "key"); + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), LazyDelegatingGauge::new)); } @Override diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index a77c113ca78..4c3e43a8245 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -269,11 +269,17 @@ expect(batch_stats["event_count"]["average"]["lifetime"]).to be_a_kind_of(Numeric) expect(batch_stats["event_count"]["average"]["lifetime"]).to be > 0 + expect(batch_stats["event_count"]["current"]).not_to be_nil + expect(batch_stats["event_count"]["current"]).to be >= 0 + expect(batch_stats["byte_size"]).not_to be_nil expect(batch_stats["byte_size"]["average"]).not_to be_nil expect(batch_stats["byte_size"]["average"]["lifetime"]).not_to be_nil expect(batch_stats["byte_size"]["average"]["lifetime"]).to be_a_kind_of(Numeric) expect(batch_stats["byte_size"]["average"]["lifetime"]).to be > 0 + + expect(batch_stats["byte_size"]["current"]).not_to be_nil + expect(batch_stats["byte_size"]["current"]).to be >= 0 end end end From 45320546c3ecd84f823fb30dfd8612d247797ab2 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Fri, 26 Sep 2025 12:47:50 -0700 Subject: [PATCH 22/44] [Docs] Add hyphen to the pipeline ID restriction description. (#18216) * [Docs] Add hyphen to the pipeline ID restriction description. * Apply suggestions from code review Apply Oxford comma. Co-authored-by: Rob Bavey --------- Co-authored-by: Rob Bavey --- docs/reference/configuring-centralized-pipelines.md | 2 +- x-pack/lib/config_management/bootstrap_check.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/configuring-centralized-pipelines.md b/docs/reference/configuring-centralized-pipelines.md index fc56a4a4f97..04ec13dd65c 100644 --- a/docs/reference/configuring-centralized-pipelines.md +++ b/docs/reference/configuring-centralized-pipelines.md @@ -144,7 +144,7 @@ This setting can be used only if `xpack.management.elasticsearch.ssl.certificate ## Wildcard support in pipeline ID [wildcard-in-pipeline-id] -Pipeline IDs must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers. You can use `*` in `xpack.management.pipeline.id` to match any number of letters, underscores, dashes, and numbers. +Pipeline IDs must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens and numbers. You can use `*` in `xpack.management.pipeline.id` to match any number of letters, underscores, dashes, hyphens, and numbers. ```shell xpack.management.pipeline.id: ["*logs", "*apache*", "tomcat_log"] diff --git a/x-pack/lib/config_management/bootstrap_check.rb b/x-pack/lib/config_management/bootstrap_check.rb index 89b675d0bb1..06067fe45d4 100644 --- a/x-pack/lib/config_management/bootstrap_check.rb +++ b/x-pack/lib/config_management/bootstrap_check.rb @@ -14,7 +14,7 @@ module ConfigManagement class BootstrapCheck include LogStash::Util::Loggable - # pipeline ID must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers + # pipeline ID must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens, and numbers # wildcard character `*` is also acceptable and follows globbing rules PIPELINE_ID_PATTERN = %r{\A[a-z_*][a-z_\-0-9*]*\Z}i @@ -43,7 +43,7 @@ def self.check(settings) invalid_patterns = pipeline_ids.reject { |entry| PIPELINE_ID_PATTERN =~ entry } if invalid_patterns.any? - raise LogStash::BootstrapCheckError, "Pipeline id in `xpack.management.pipeline.id` must begin with a letter or underscore and contain only letters, underscores, dashes, and numbers. The asterisk wildcard `*` can also be used. Invalid ids: #{invalid_patterns.join(', ')}" + raise LogStash::BootstrapCheckError, "Pipeline id in `xpack.management.pipeline.id` must begin with a letter or underscore and contain only letters, underscores, dashes, hyphens, and numbers. The asterisk wildcard `*` can also be used. Invalid ids: #{invalid_patterns.join(', ')}" end duplicate_ids = find_duplicate_ids(pipeline_ids) From 7c7504e1a5966d9ef933e47dfcdd49aaff802339 Mon Sep 17 00:00:00 2001 From: Lisa Cawley Date: Fri, 26 Sep 2025 13:33:54 -0700 Subject: [PATCH 23/44] [DOCS] Fix substitution variables (#18224) --- docs/release-notes/index.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index 38d7633bf77..ad0bb983932 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -31,7 +31,7 @@ To check for security updates, go to [Security announcements for the Elastic sta ### Fixes [logstash-9.1.4-fixes] -* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {ls} plugin activity on the {ls} Integration dashboards [#18090](https://github.com/elastic/logstash/pull/18090) +* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {{ls}} plugin activity on the {{ls}} Integration dashboards [#18090](https://github.com/elastic/logstash/pull/18090) * Improve logstash release artifacts file metadata: mtime is preserved when buiilding tar archives [#18091](https://github.com/elastic/logstash/pull/18091) @@ -182,8 +182,8 @@ The Elasticsearch Input now provides [support](https://github.com/logstash-plugi ### Fixes [logstash-9.0.7-fixes] -* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {ls} plugin activity on the {ls} Integration dashboards. [#18089](https://github.com/elastic/logstash/pull/18089) -* Improve logstash release artifacts file metadata: mtime is preserved when buiilding tar archives. [#18111](https://github.com/elastic/logstash/pull/18111) +* Gauge type metrics, such as current and peak connection counts of Elastic Agent, are now available in the `_node/stats` API response when the `vertices=true` parameter is included. These metrics are particularly useful for monitoring {{ls}} plugin activity on the {{ls}} Integration dashboards. [#18089](https://github.com/elastic/logstash/pull/18089) +* Improve logstash release artifacts file metadata: mtime is preserved when building tar archives. [#18111](https://github.com/elastic/logstash/pull/18111) ### Plugins [logstash-plugin-9.0.7-changes] @@ -387,4 +387,4 @@ Check out the [security advisory](https://discuss.elastic.co/c/announcements/sec **Tcp Output - 7.0.0** -* Remove deprecated SSL settings [#58](https://github.com/logstash-plugins/logstash-output-tcp/pull/58) \ No newline at end of file +* Remove deprecated SSL settings [#58](https://github.com/logstash-plugins/logstash-output-tcp/pull/58) From dd371da95691b4fab4d40a8944d06f59c49f52c5 Mon Sep 17 00:00:00 2001 From: Rye Biesemeyer Date: Mon, 29 Sep 2025 05:59:10 -0700 Subject: [PATCH 24/44] PQ: Add support for event-level compression using ZStandard (ZSTD) (#18121) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * noop: add pq compression-codec with no-op implementation * pq: add support for event compression using zstd Adds non-breaking support for event compression to the persisted queue, as configured by a new per-pipeline setting `queue.compression`, which supports: - `none` (default): no compression is performed, but if compressed events are encountered in the queue they will be decompressed - `speed`: compression optimized for speed - `balanced`: compression balancing speed against result size - `size`: compression optimized for maximum reduction of size - `disabled`: compression support entirely disabled; if a pipeline is run in this configuration against a PQ that already contains unacked compressed events, the pipeline WILL crash. To accomplish this, we then provide an abstract base implementation of the CompressionCodec whose decode method is capable of _detecting_ and decoding zstd-encoded payload while letting other payloads through unmodified. The detection is done with an operation on the first four bytes of the payload, so no additional context is needed. An instance of this zstd-aware compression codec is provided with a pass-through encode operation when configured with `queue.compression: none`, which is the default, ensuring that by default logstash is able to decode any event that had previously been written. We provide an additional implementation that is capable of _encoding_ events with a configurable goal: speed, size, or a balance of the two. * license: add notice for `com.github.luben:zstd-jni` * pq: log compression encode/decode from the codec * Apply docs suggestions from code review Co-authored-by: João Duarte * remove CleanerThreadLocal utility * license: add mapping for com.github.luben:zstd-jni * Apply suggestions from code review Co-authored-by: Rob Bavey --------- Co-authored-by: João Duarte Co-authored-by: Rob Bavey --- config/logstash.yml | 4 + docker/data/logstash/env2yaml/env2yaml.go | 1 + docs/reference/logstash-settings-file.md | 1 + docs/reference/persistent-queues.md | 11 + logstash-core/build.gradle | 1 + logstash-core/lib/logstash/environment.rb | 1 + logstash-core/lib/logstash/settings.rb | 1 + .../spec/logstash/queue_factory_spec.rb | 1 + .../AbstractZstdAwareCompressionCodec.java | 41 +++ .../logstash/ackedqueue/CompressionCodec.java | 58 +++++ .../java/org/logstash/ackedqueue/Queue.java | 8 +- .../logstash/ackedqueue/QueueFactoryExt.java | 12 + .../org/logstash/ackedqueue/Settings.java | 4 + .../org/logstash/ackedqueue/SettingsImpl.java | 17 ++ .../ackedqueue/ZstdAwareCompressionCodec.java | 18 ++ .../ZstdEnabledCompressionCodec.java | 41 +++ .../common/SettingKeyDefinitions.java | 2 + .../ackedqueue/CompressionCodecTest.java | 239 ++++++++++++++++++ .../ackedqueue/ImmutableByteArrayBarrier.java | 20 ++ .../mixed-compression-queue-data-dir.md | 141 +++++++++++ .../mixed-compression-queue-data-dir.tar.gz | Bin 0 -> 9467 bytes qa/integration/fixtures/pq_drain_spec.yml | 3 + qa/integration/specs/pq_drain_spec.rb | 142 +++++++++++ .../src/main/resources/licenseMapping.csv | 1 + .../com.github.luben!zstd-jni-NOTICE.txt | 26 ++ 25 files changed, 792 insertions(+), 2 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java create mode 100644 logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java create mode 100644 logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java create mode 100644 qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md create mode 100644 qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz create mode 100644 qa/integration/fixtures/pq_drain_spec.yml create mode 100644 qa/integration/specs/pq_drain_spec.rb create mode 100644 tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt diff --git a/config/logstash.yml b/config/logstash.yml index 472c421b7cf..37743875629 100644 --- a/config/logstash.yml +++ b/config/logstash.yml @@ -229,6 +229,10 @@ # # queue.checkpoint.writes: 1024 # +# If using queue.type: persisted, the compression goal. Valid values are `none`, `speed`, `balanced`, and `size`. +# The default `none` is able to decompress previously-written events, even if they were compressed. +# +# queue.compression: none # # ------------ Dead-Letter Queue Settings -------------- # Flag to turn on dead-letter queue. diff --git a/docker/data/logstash/env2yaml/env2yaml.go b/docker/data/logstash/env2yaml/env2yaml.go index 95fc569b236..d1e976bbab1 100644 --- a/docker/data/logstash/env2yaml/env2yaml.go +++ b/docker/data/logstash/env2yaml/env2yaml.go @@ -58,6 +58,7 @@ var validSettings = []string{ "queue.checkpoint.acks", "queue.checkpoint.writes", "queue.checkpoint.interval", // remove it for #17155 + "queue.compression", "queue.drain", "dead_letter_queue.enable", "dead_letter_queue.max_bytes", diff --git a/docs/reference/logstash-settings-file.md b/docs/reference/logstash-settings-file.md index 7bcce3c853a..419e1e3f5ae 100644 --- a/docs/reference/logstash-settings-file.md +++ b/docs/reference/logstash-settings-file.md @@ -68,6 +68,7 @@ The `logstash.yml` file includes these settings. | `queue.checkpoint.acks` | The maximum number of ACKed events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.acks: 0` to set this value to unlimited. | 1024 | | `queue.checkpoint.writes` | The maximum number of written events before forcing a checkpoint when persistent queues are enabled (`queue.type: persisted`). Specify `queue.checkpoint.writes: 0` to set this value to unlimited. | 1024 | | `queue.checkpoint.retry` | When enabled, Logstash will retry four times per attempted checkpoint write for any checkpoint writes that fail. Any subsequent errors are not retried. This is a workaround for failed checkpoint writes that have been seen only on Windows platform, filesystems with non-standard behavior such as SANs and is not recommended except in those specific circumstances. (`queue.type: persisted`) | `true` | +| `queue.compression` {applies_to}`stack: ga 9.2` | Set a persisted queue compression level, which allows the pipeline to reduce the event size on disk at the cost of CPU usage. Possible values are `speed`, `balanced`, and `size`. | `none` | | `queue.drain` | When enabled, Logstash waits until the persistent queue (`queue.type: persisted`) is drained before shutting down. | `false` | | `dead_letter_queue.enable` | Flag to instruct Logstash to enable the DLQ feature supported by plugins. | `false` | | `dead_letter_queue.max_bytes` | The maximum size of each dead letter queue. Entries will be dropped if they would increase the size of the dead letter queue beyond this setting. | `1024mb` | diff --git a/docs/reference/persistent-queues.md b/docs/reference/persistent-queues.md index 8c2bed6639d..b10c1179817 100644 --- a/docs/reference/persistent-queues.md +++ b/docs/reference/persistent-queues.md @@ -84,6 +84,17 @@ If you want to define values for a specific pipeline, use [`pipelines.yml`](/ref `queue.checkpoint.interval` {applies_to}`stack: deprecated 9.1` : Sets the interval in milliseconds when a checkpoint is forced on the head page. Default is `1000`. Set to `0` to eliminate periodic checkpoints. +`queue.compression` {applies_to}`stack: ga 9.2` +: Sets the event compression level for use with the Persisted Queue. Default is `none`. Possible values are: + * `none`: does not perform compression, but reads compressed events + * `speed`: optimize for fastest compression operation + * `size`: optimize for smallest possible size on disk, spending more CPU + * `balanced`: a balance between the `speed` and `size` settings +:::{important} +Compression can be enabled for an existing PQ, but once compressed elements have been added to a PQ, that PQ cannot be read by previous Logstash releases that did not support compression. +If you need to downgrade Logstash after enabling the PQ, you will need to either delete the PQ or run the pipeline with `queue.drain: true` first to ensure that no compressed elements remain. +::: + ## Configuration notes [pq-config-notes] Every situation and environment is different, and the "ideal" configuration varies. If you optimize for performance, you may increase your risk of losing data. If you optimize for data protection, you may impact performance. diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index 5c0db0f5a3f..a45e168f008 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -239,6 +239,7 @@ dependencies { implementation 'commons-codec:commons-codec:1.17.0' // transitively required by httpclient // Jackson version moved to versions.yml in the project root (the JrJackson version is there too) implementation "com.fasterxml.jackson.core:jackson-core:${jacksonVersion}" + implementation "com.github.luben:zstd-jni:1.5.7-4" api "com.fasterxml.jackson.core:jackson-databind:${jacksonDatabindVersion}" api "com.fasterxml.jackson.core:jackson-annotations:${jacksonVersion}" implementation 'org.codehaus.janino:janino:3.1.0' diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index 4611741a54c..a04b3b9b0b4 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -96,6 +96,7 @@ def self.as_java_range(r) Setting::NumericSetting.new("queue.checkpoint.writes", 1024), # 0 is unlimited Setting::NumericSetting.new("queue.checkpoint.interval", 1000), # remove it for #17155 Setting::BooleanSetting.new("queue.checkpoint.retry", true), + Setting::StringSetting.new("queue.compression", "none", true, %w(none speed balanced size disabled)), Setting::BooleanSetting.new("dead_letter_queue.enable", false), Setting::Bytes.new("dead_letter_queue.max_bytes", "1024mb"), Setting::NumericSetting.new("dead_letter_queue.flush_interval", 5000), diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index 5458d16e346..81d208cb151 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -70,6 +70,7 @@ def self.included(base) "queue.checkpoint.interval", # remove it for #17155 "queue.checkpoint.writes", "queue.checkpoint.retry", + "queue.compression", "queue.drain", "queue.max_bytes", "queue.max_events", diff --git a/logstash-core/spec/logstash/queue_factory_spec.rb b/logstash-core/spec/logstash/queue_factory_spec.rb index 7297e92dc43..56e54dabb88 100644 --- a/logstash-core/spec/logstash/queue_factory_spec.rb +++ b/logstash-core/spec/logstash/queue_factory_spec.rb @@ -30,6 +30,7 @@ LogStash::Setting::NumericSetting.new("queue.checkpoint.acks", 1024), LogStash::Setting::NumericSetting.new("queue.checkpoint.writes", 1024), LogStash::Setting::BooleanSetting.new("queue.checkpoint.retry", false), + LogStash::Setting::StringSetting.new("queue.compression", "none", true, %w(none speed balanced size disabled)), LogStash::Setting::StringSetting.new("pipeline.id", pipeline_id), LogStash::Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), LogStash::Setting::PositiveIntegerSetting.new("pipeline.batch.size", 125), diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java new file mode 100644 index 00000000000..1cb54c98a77 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java @@ -0,0 +1,41 @@ +package org.logstash.ackedqueue; + +import com.github.luben.zstd.Zstd; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +/** + * Subclasses of {@link AbstractZstdAwareCompressionCodec} are {@link CompressionCodec}s that are capable + * of detecting and decompressing deflate-compressed events. When decoding byte sequences that are NOT + * deflate-compressed, the given bytes are emitted verbatim. + */ +abstract class AbstractZstdAwareCompressionCodec implements CompressionCodec { + // log from the concrete class + protected final Logger logger = LogManager.getLogger(this.getClass()); + + @Override + public byte[] decode(byte[] data) { + if (!isZstd(data)) { + return data; + } + try { + final byte[] decoded = Zstd.decompress(data); + logger.trace("decoded {} -> {}", data.length, decoded.length); + return decoded; + } catch (Exception e) { + throw new RuntimeException("Exception while decoding", e); + } + } + + private static final byte[] ZSTD_FRAME_MAGIC = { (byte) 0x28, (byte) 0xB5, (byte) 0x2F, (byte) 0xFD }; + + static boolean isZstd(byte[] data) { + if (data.length < 4) { return false; } + + for (int i = 0; i < 4; i++) { + if (data[i] != ZSTD_FRAME_MAGIC[i]) { return false; } + } + + return true; + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java new file mode 100644 index 00000000000..b1f99cf9980 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java @@ -0,0 +1,58 @@ +package org.logstash.ackedqueue; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public interface CompressionCodec { + Logger LOGGER = LogManager.getLogger(CompressionCodec.class); + + byte[] encode(byte[] data); + byte[] decode(byte[] data); + + /** + * The {@link CompressionCodec#NOOP} is a {@link CompressionCodec} that + * does nothing when encoding and decoding. It is only meant to be activated + * as a safety-latch in the event of compression being broken. + */ + CompressionCodec NOOP = new CompressionCodec() { + @Override + public byte[] encode(byte[] data) { + return data; + } + + @Override + public byte[] decode(byte[] data) { + return data; + } + }; + + static CompressionCodec fromConfigValue(final String configValue) { + return fromConfigValue(configValue, LOGGER); + } + + static CompressionCodec fromConfigValue(final String configValue, final Logger logger) { + return switch (configValue) { + case "disabled" -> { + logger.warn("compression support has been disabled"); + yield CompressionCodec.NOOP; + } + case "none" -> { + logger.info("compression support is enabled (read-only)"); + yield ZstdAwareCompressionCodec.getInstance(); + } + case "speed" -> { + logger.info("compression support is enabled (goal: speed)"); + yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.SPEED); + } + case "balanced" -> { + logger.info("compression support is enabled (goal: balanced)"); + yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.BALANCED); + } + case "size" -> { + logger.info("compression support is enabled (goal: size)"); + yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.SIZE); + } + default -> throw new IllegalArgumentException(String.format("Unsupported compression setting `%s`", configValue)); + }; + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java index 691987793c5..ceace485888 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java @@ -83,6 +83,7 @@ public final class Queue implements Closeable { // deserialization private final Class elementClass; private final Method deserializeMethod; + private final CompressionCodec compressionCodec; // thread safety private final ReentrantLock lock = new ReentrantLock(); @@ -112,6 +113,7 @@ public Queue(Settings settings) { this.maxBytes = settings.getQueueMaxBytes(); this.checkpointIO = new FileCheckpointIO(dirPath, settings.getCheckpointRetry()); this.elementClass = settings.getElementClass(); + this.compressionCodec = settings.getCompressionCodec(); this.tailPages = new ArrayList<>(); this.unreadTailPages = new ArrayList<>(); this.closed = new AtomicBoolean(true); // not yet opened @@ -414,7 +416,8 @@ public long write(Queueable element) throws IOException { throw new QueueRuntimeException(QueueExceptionMessages.CANNOT_WRITE_TO_CLOSED_QUEUE); } - byte[] data = element.serialize(); + byte[] serializedBytes = element.serialize(); + byte[] data = compressionCodec.encode(serializedBytes); // the write strategy with regard to the isFull() state is to assume there is space for this element // and write it, then after write verify if we just filled the queue and wait on the notFull condition @@ -767,7 +770,8 @@ public CheckpointIO getCheckpointIO() { */ public Queueable deserialize(byte[] bytes) { try { - return (Queueable)this.deserializeMethod.invoke(this.elementClass, bytes); + byte[] decodedBytes = compressionCodec.decode(bytes); + return (Queueable)this.deserializeMethod.invoke(this.elementClass, decodedBytes); } catch (IllegalAccessException|InvocationTargetException e) { throw new QueueRuntimeException("deserialize invocation error", e); } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index c44560ad7c6..6d64bd4d9e4 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -24,6 +24,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import org.jruby.Ruby; import org.jruby.RubyBasicObject; import org.jruby.RubyClass; @@ -69,6 +72,8 @@ public enum BatchMetricMode { private static final long serialVersionUID = 1L; + private static final Logger LOGGER = LogManager.getLogger(QueueFactoryExt.class); + public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } @@ -137,6 +142,13 @@ private static Settings extractQueueSettings(final IRubyObject settings) { .checkpointMaxAcks(getSetting(context, settings, QUEUE_CHECKPOINT_ACKS).toJava(Integer.class)) .checkpointRetry(getSetting(context, settings, QUEUE_CHECKPOINT_RETRY).isTrue()) .queueMaxBytes(getSetting(context, settings, QUEUE_MAX_BYTES).toJava(Integer.class)) + .compressionCodec(extractConfiguredCodec(settings)) .build(); } + + private static CompressionCodec extractConfiguredCodec(final IRubyObject settings) { + final ThreadContext context = settings.getRuntime().getCurrentContext(); + final String compressionSetting = getSetting(context, settings, QUEUE_COMPRESSION).asJavaString(); + return CompressionCodec.fromConfigValue(compressionSetting, LOGGER); + } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java index 212945ef07f..36d4b60e3b2 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java @@ -44,6 +44,8 @@ public interface Settings { boolean getCheckpointRetry(); + CompressionCodec getCompressionCodec(); + /** * Validate and return the settings, or throw descriptive {@link QueueRuntimeException} * @param settings the settings to validate @@ -89,6 +91,8 @@ interface Builder { Builder checkpointRetry(boolean checkpointRetry); + Builder compressionCodec(CompressionCodec compressionCodec); + Settings build(); } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java index bc191f44a32..923217af366 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java @@ -31,6 +31,7 @@ public class SettingsImpl implements Settings { private final int checkpointMaxAcks; private final int checkpointMaxWrites; private final boolean checkpointRetry; + private final CompressionCodec compressionCodec; public static Builder builder(final Settings settings) { return new BuilderImpl(settings); @@ -49,6 +50,7 @@ private SettingsImpl(final BuilderImpl builder) { this.checkpointMaxAcks = builder.checkpointMaxAcks; this.checkpointMaxWrites = builder.checkpointMaxWrites; this.checkpointRetry = builder.checkpointRetry; + this.compressionCodec = builder.compressionCodec; } @Override @@ -91,6 +93,11 @@ public boolean getCheckpointRetry() { return this.checkpointRetry; } + @Override + public CompressionCodec getCompressionCodec() { + return this.compressionCodec; + } + /** * Default implementation for Setting's Builder * */ @@ -140,6 +147,8 @@ private static final class BuilderImpl implements Builder { private boolean checkpointRetry; + private CompressionCodec compressionCodec; + private BuilderImpl(final String dirForFiles) { this.dirForFiles = dirForFiles; this.elementClass = null; @@ -148,6 +157,7 @@ private BuilderImpl(final String dirForFiles) { this.maxUnread = DEFAULT_MAX_UNREAD; this.checkpointMaxAcks = DEFAULT_CHECKPOINT_MAX_ACKS; this.checkpointMaxWrites = DEFAULT_CHECKPOINT_MAX_WRITES; + this.compressionCodec = CompressionCodec.NOOP; this.checkpointRetry = false; } @@ -160,6 +170,7 @@ private BuilderImpl(final Settings settings) { this.checkpointMaxAcks = settings.getCheckpointMaxAcks(); this.checkpointMaxWrites = settings.getCheckpointMaxWrites(); this.checkpointRetry = settings.getCheckpointRetry(); + this.compressionCodec = settings.getCompressionCodec(); } @Override @@ -204,6 +215,12 @@ public Builder checkpointRetry(final boolean checkpointRetry) { return this; } + @Override + public Builder compressionCodec(CompressionCodec compressionCodec) { + this.compressionCodec = compressionCodec; + return this; + } + @Override public Settings build() { return Settings.ensureValid(new SettingsImpl(this)); diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java new file mode 100644 index 00000000000..f82b4b75f2e --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java @@ -0,0 +1,18 @@ +package org.logstash.ackedqueue; + +/** + * A {@link ZstdAwareCompressionCodec} is an {@link CompressionCodec} that can decode deflate-compressed + * bytes, but performs no compression when encoding. + */ +class ZstdAwareCompressionCodec extends AbstractZstdAwareCompressionCodec { + private static final ZstdAwareCompressionCodec INSTANCE = new ZstdAwareCompressionCodec(); + + static ZstdAwareCompressionCodec getInstance() { + return INSTANCE; + } + + @Override + public byte[] encode(byte[] data) { + return data; + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java new file mode 100644 index 00000000000..fa5a22b3ee6 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java @@ -0,0 +1,41 @@ +package org.logstash.ackedqueue; + +import com.github.luben.zstd.Zstd; + +/** + * A {@link ZstdEnabledCompressionCodec} is a {@link CompressionCodec} that can decode deflate-compressed + * bytes and performs deflate compression when encoding. + */ +class ZstdEnabledCompressionCodec extends AbstractZstdAwareCompressionCodec implements CompressionCodec { + public enum Goal { + FASTEST(-7), + SPEED(-1), + BALANCED(3), + HIGH(14), + SIZE(22), + ; + + private int internalLevel; + + Goal(final int internalLevel) { + this.internalLevel = internalLevel; + } + } + + private final int internalLevel; + + ZstdEnabledCompressionCodec(final Goal internalLevel) { + this.internalLevel = internalLevel.internalLevel; + } + + @Override + public byte[] encode(byte[] data) { + try { + final byte[] encoded = Zstd.compress(data, internalLevel); + logger.trace("encoded {} -> {}", data.length, encoded.length); + return encoded; + } catch (Exception e) { + throw new RuntimeException("Exception while encoding", e); + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java index 1e9eabf0a61..b6a426db343 100644 --- a/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java +++ b/logstash-core/src/main/java/org/logstash/common/SettingKeyDefinitions.java @@ -45,4 +45,6 @@ public class SettingKeyDefinitions { public static final String QUEUE_CHECKPOINT_RETRY = "queue.checkpoint.retry"; public static final String QUEUE_MAX_BYTES = "queue.max_bytes"; + + public static final String QUEUE_COMPRESSION = "queue.compression"; } \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java new file mode 100644 index 00000000000..d1a154bd3d3 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java @@ -0,0 +1,239 @@ +package org.logstash.ackedqueue; + +import com.github.luben.zstd.Zstd; +import org.apache.logging.log4j.Logger; +import org.junit.Test; +import org.mockito.Mockito; + +import java.security.SecureRandom; +import java.util.Arrays; + +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThrows; +import static org.mockito.Matchers.argThat; + +public class CompressionCodecTest { + static final ImmutableByteArrayBarrier RAW_BYTES = new ImmutableByteArrayBarrier(( + "this is a string of text with repeated substrings that is designed to be "+ + "able to be compressed into a string that is smaller than the original input "+ + "so that we can assert that the compression codecs compress strings to be "+ + "smaller than their uncompressed representations").getBytes()); + static final ImmutableByteArrayBarrier COMPRESSED_MINIMAL = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), -1)); + static final ImmutableByteArrayBarrier COMPRESSED_DEFAULT = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 3)); + static final ImmutableByteArrayBarrier COMPRESSED_MAXIMUM = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 22)); + + private final CompressionCodec codecDisabled = CompressionCodec.fromConfigValue("disabled"); + private final CompressionCodec codecNone = CompressionCodec.fromConfigValue("none"); + private final CompressionCodec codecSpeed = CompressionCodec.fromConfigValue("speed"); + private final CompressionCodec codecBalanced = CompressionCodec.fromConfigValue("balanced"); + private final CompressionCodec codecSize = CompressionCodec.fromConfigValue("size"); + + @Test + public void testDisabledCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled"); + assertDecodesRaw(compressionCodec); + + // ensure true pass-through when compression is disabled, even if the payload looks like ZSTD + assertThat(compressionCodec.decode(COMPRESSED_MINIMAL.bytes()), is(equalTo(COMPRESSED_MINIMAL.bytes()))); + assertThat(compressionCodec.decode(COMPRESSED_DEFAULT.bytes()), is(equalTo(COMPRESSED_DEFAULT.bytes()))); + assertThat(compressionCodec.decode(COMPRESSED_MAXIMUM.bytes()), is(equalTo(COMPRESSED_MAXIMUM.bytes()))); + } + + @Test + public void testDisabledCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled"); + // ensure true pass-through when compression is disabled + assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + @Test + public void testDisabledCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("disabled", mockLogger); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("compression support", "disabled"))); + } + + @Test + public void testNoneCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none"); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testNoneCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none"); + assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + @Test + public void testNoneCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("none", mockLogger); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "read-only"))); + } + + @Test + public void testSpeedCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed"); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testSpeedCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed"); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testSpeedCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("speed", mockLogger); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "speed"))); + } + + @Test + public void testBalancedCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced"); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testBalancedCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced"); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testBalancedCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("balanced", mockLogger); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "balanced"))); + } + + @Test + public void testSizeCompressionCodecDecodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size"); + assertDecodesRaw(compressionCodec); + assertDecodesDeflateAnyLevel(compressionCodec); + assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); + } + + @Test + public void testSizeCompressionCodecEncodes() throws Exception { + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size"); + assertEncodesSmallerRoundTrip(compressionCodec); + } + + @Test + public void testSizeCompressionCodecLogging() throws Exception { + final Logger mockLogger = Mockito.mock(Logger.class); + CompressionCodec.fromConfigValue("size", mockLogger); + Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "size"))); + } + + @Test(timeout=1000) + public void testCompressionCodecDecodeTailTruncated() throws Exception { + final byte[] truncatedInput = copyWithTruncatedTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(truncatedInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), containsString("Data corruption detected")); + } + + byte[] copyWithTruncatedTail(final byte[] input, final int tailSize) { + int startIndex = (input.length < tailSize) ? 0 : input.length - tailSize; + + final byte[] result = Arrays.copyOf(input, input.length); + Arrays.fill(result, startIndex, result.length, (byte) 0); + + return result; + } + + @Test(timeout=1000) + public void testCompressionCodecDecodeTailScrambled() throws Exception { + final byte[] scrambledInput = copyWithScrambledTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(scrambledInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), anyOf(containsString("Data corruption detected"), containsString("Destination buffer is too small"))); + } + + byte[] copyWithScrambledTail(final byte[] input, final int tailSize) { + final SecureRandom secureRandom = new SecureRandom(); + int startIndex = (input.length < tailSize) ? 0 : input.length - tailSize; + + byte[] randomBytes = new byte[input.length - startIndex]; + secureRandom.nextBytes(randomBytes); + + final byte[] result = Arrays.copyOf(input, input.length); + System.arraycopy(randomBytes, 0, result, startIndex, randomBytes.length); + + return result; + } + + @Test(timeout=1000) + public void testCompressionDecodeTailNullPadded() throws Exception { + final byte[] nullPaddedInput = copyWithNullPaddedTail(COMPRESSED_DEFAULT.bytes(), 32); + + final RuntimeException thrownException = assertThrows(RuntimeException.class, () -> codecNone.decode(nullPaddedInput)); + assertThat(thrownException.getMessage(), containsString("Exception while decoding")); + final Throwable rootCause = extractRootCause(thrownException); + assertThat(rootCause.getMessage(), anyOf(containsString("Unknown frame descriptor"), containsString("Data corruption detected"))); + } + + byte[] copyWithNullPaddedTail(final byte[] input, final int tailSize) { + return Arrays.copyOf(input, Math.addExact(input.length, tailSize)); + } + + Throwable extractRootCause(final Throwable throwable) { + Throwable current; + Throwable cause = throwable; + do { + current = cause; + cause = current.getCause(); + } while (cause != null && cause != current); + return current; + } + + void assertDecodesRaw(final CompressionCodec codec) { + assertThat(codec.decode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); + } + + void assertDecodesDeflateAnyLevel(final CompressionCodec codec) { + // zstd levels range from -7 to 22. + for (int level = -7; level < 22; level++) { + final byte[] compressed = compress(RAW_BYTES.bytes(), level); + assertThat(String.format("zstd level %s (%s bytes)", level, compressed.length), codec.decode(compressed), is(equalTo(RAW_BYTES.bytes()))); + } + } + + void assertDecodesOutputOfAllKnownCompressionCodecs(final CompressionCodec codec) { + assertThat(codec.decode(codecDisabled.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecNone.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecSpeed.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecBalanced.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + assertThat(codec.decode(codecSize.encode(RAW_BYTES.bytes())), is(equalTo(RAW_BYTES.bytes()))); + } + + void assertEncodesSmallerRoundTrip(final CompressionCodec codec) { + final byte[] input = RAW_BYTES.bytes(); + + final byte[] encoded = codec.encode(input); + assertThat("encoded is smaller", encoded.length, is(lessThan(input.length))); + assertThat("shaped like zstd", AbstractZstdAwareCompressionCodec.isZstd(encoded), is(true)); + assertThat("round trip decode", codec.decode(encoded), is(equalTo(input))); + } + + public static byte[] compress(byte[] input, int level) { + return Zstd.compress(input, level); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java b/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java new file mode 100644 index 00000000000..276e4d97dba --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/ImmutableByteArrayBarrier.java @@ -0,0 +1,20 @@ +package org.logstash.ackedqueue; + +import java.util.Arrays; + +/** + * An {@link ImmutableByteArrayBarrier} provides an immutability shield around a {@code byte[]}. + * It stores an inaccessible copy of the provided {@code byte[]}, and makes copies of that copy + * available via {@link ImmutableByteArrayBarrier#bytes}. + * @param bytes the byte array + */ +public record ImmutableByteArrayBarrier(byte[] bytes) { + public ImmutableByteArrayBarrier(byte[] bytes) { + this.bytes = Arrays.copyOf(bytes, bytes.length); + } + + @Override + public byte[] bytes() { + return Arrays.copyOf(bytes, bytes.length); + } +} diff --git a/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md new file mode 100644 index 00000000000..6daecabb144 --- /dev/null +++ b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.md @@ -0,0 +1,141 @@ +# Summary + +The logstash data directory contains a queue for pipeline `main` containing: + + - ACK'd events (from a page that is not fully-ack'd) + - raw CBOR-encoded events + - zstd-compressed events with different compression goals + +# Pages +~~~ +1 258 821AACAC page.0 CBOR(stringref) +2 343 3BE717E8 page.0 CBOR(stringref) +3 332 3439807A page.0 CBOR(stringref) +4 258 C04209D4 page.0 CBOR(stringref) +5 343 3DFB08E8 page.0 CBOR(stringref) +6 332 44B0315D page.0 CBOR(stringref) +7 258 90D25985 page.0 CBOR(stringref) +8 343 DAFD5712 page.0 CBOR(stringref) +9 332 AB6A81DF page.0 CBOR(stringref) +10 258 157EA7A6 page.0 CBOR(stringref) +11 258 02C0F7A2 page.0 CBOR(stringref) +12 343 0005E8A8 page.0 CBOR(stringref) +13 332 C2DA39EA page.1 CBOR(stringref) +14 258 377D623C page.1 CBOR(stringref) +15 343 9F76657C page.1 CBOR(stringref) +16 332 50B51A98 page.1 CBOR(stringref) +17 258 827848CC page.1 CBOR(stringref) +18 343 8325D121 page.1 CBOR(stringref) +19 332 E1A1378B page.1 CBOR(stringref) +20 258 1BBDAA1A page.1 CBOR(stringref) +21 254 19C85DF6 page.1 ZSTD(258) +22 317 AD5DC7CC page.1 ZSTD(343) +23 325 BB8CE48C page.1 ZSTD(332) +24 254 27D38856 page.1 ZSTD(258) +25 317 67A7D2F3 page.2 ZSTD(343) +26 325 888AF9B2 page.2 ZSTD(332) +27 254 CAA2FDE3 page.2 ZSTD(258) +28 317 2985771A page.2 ZSTD(343) +29 325 89197F51 page.2 ZSTD(332) +30 254 A9E292EE page.2 ZSTD(258) +31 258 243FC2C1 page.2 CBOR(stringref) +32 219 2E2E0BDF page.2 ZSTD(258) +33 261 5ED17F40 page.2 ZSTD(343) +34 280 86BA1E80 page.2 ZSTD(332) +35 218 6A7B8C41 page.2 ZSTD(258) +36 262 08E69C4C page.2 ZSTD(343) +37 277 CD32DEBD page.2 ZSTD(332) +38 218 43101D61 page.2 ZSTD(258) +39 261 A22033DE page.3 ZSTD(343) +40 279 8F1FE0FA page.3 ZSTD(332) +41 218 FF56D05C page.3 ZSTD(258) +42 258 7077981D page.3 CBOR(stringref) +43 343 7748A127 page.3 CBOR(stringref) +44 332 B4A0C82C page.3 CBOR(stringref) +45 258 96FB0308 page.3 CBOR(stringref) +46 343 40B77975 page.3 CBOR(stringref) +47 332 D5571FDC page.3 CBOR(stringref) +48 258 BF3FC517 page.3 CBOR(stringref) +49 343 1BC62146 page.3 CBOR(stringref) +50 332 418FD829 page.3 CBOR(stringref) +51 258 DB40747E page.3 CBOR(stringref) +52 224 7629AF30 page.4 ZSTD(258) +53 264 D450FC21 page.4 ZSTD(343) +54 284 43F91F18 page.4 ZSTD(332) +55 224 C61DB7BA page.4 ZSTD(258) +56 264 F9547DBC page.4 ZSTD(343) +57 281 3DBB71E5 page.4 ZSTD(332) +58 225 8ACDB484 page.4 ZSTD(258) +59 264 8256E2D2 page.4 ZSTD(343) +60 281 D76156A2 page.4 ZSTD(332) +61 225 EDC6147B page.4 ZSTD(258) +62 258 D3AB1EF4 page.4 CBOR(stringref) +63 220 4851D677 page.4 ZSTD(258) +64 225 C8DCE54A page.4 ZSTD(258) +65 251 3D1E0F5F page.4 ZSTD(258) +66 258 1C5637CB page.4 CBOR(stringref) +67 343 09AE6714 page.5 CBOR(stringref) +68 332 4A97AC77 page.5 CBOR(stringref) +69 254 D1E43C69 page.5 ZSTD(258) +70 317 B6A2361D page.5 ZSTD(343) +71 325 A44CE35F page.5 ZSTD(332) +72 225 B69C7923 page.5 ZSTD(258) +73 265 FEBC2D45 page.5 ZSTD(343) +74 286 5FA5C389 page.5 ZSTD(332) +75 221 C36048C0 page.5 ZSTD(258) +76 262 E988C90B page.5 ZSTD(343) +77 280 6C98308C page.5 ZSTD(332) +~~~ + +# CHECKPOINTS + +~~~ +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.0 +VERSION [ 0001]: 1 +PAGENUM [ 00000000]: 0 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000005]: 5 +MINSEQN [0000000000000001]: 1 +ELEMNTS [ 0000000C]: 12 +CHECKSM [ 4AFA3119] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.1 +VERSION [ 0001]: 1 +PAGENUM [ 00000001]: 1 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [000000000000000D]: 13 +MINSEQN [000000000000000D]: 13 +ELEMNTS [ 0000000C]: 12 +CHECKSM [ 70829F7B] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.2 +VERSION [ 0001]: 1 +PAGENUM [ 00000002]: 2 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000019]: 25 +MINSEQN [0000000000000019]: 25 +ELEMNTS [ 0000000E]: 14 +CHECKSM [ 4ABFB50A] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.3 +VERSION [ 0001]: 1 +PAGENUM [ 00000003]: 3 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000027]: 39 +MINSEQN [0000000000000027]: 39 +ELEMNTS [ 0000000D]: 13 +CHECKSM [ 95B393C6] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.4 +VERSION [ 0001]: 1 +PAGENUM [ 00000004]: 4 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000034]: 52 +MINSEQN [0000000000000034]: 52 +ELEMNTS [ 0000000F]: 15 +CHECKSM [ 9B602904] +# CHECKPOINT mixed-compression-queue-data-dir/queue/main/checkpoint.head +VERSION [ 0001]: 1 +PAGENUM [ 00000005]: 5 +1UNAKPG [ 00000000]: 0 +1UNAKSQ [0000000000000043]: 67 +MINSEQN [0000000000000043]: 67 +ELEMNTS [ 0000000B]: 11 +CHECKSM [ B5F33B10] +~~~ \ No newline at end of file diff --git a/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz b/qa/integration/fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..53a0c4f985159033c7832a54e54890343d887189 GIT binary patch literal 9467 zcmZ9xWmwhU^94$m;2{MOIDn*-lr)D%KtZ}oknZk7NP{5KQqt00lG5GX-QAq{Tz}vD ze{=8K*?Z5-o;}Z+XRig0#zDCQwOFB`pBm!QYS_M_sbOOvV=PoNPaDfvl%=z!Gmlim z`r&WQ&ZuCmd3T!pB3Hvoy@}(fA*ZfMjXpnNnW-gdh2=VX^Pw^-o0kvaoaL+1&z>PP z?l*qlz77jmhiz(kHJ6(TmI^(3J@UK^2yhTzvnfTRK*i^`Y=Ib#F2+W_KpiGRN^ z#@l?1AJ|jjIzNuVo`n%VAag*As)M3JM=Xi|f&yLulMl>>Wa9dhS`cnS+d)3dLrK8) zZH0ap@*Z9q2o0tUBM;&JC1Og>(Z|{K*8~zy6!x4@SW6Nc=KR!&%x@?tz=Xes33y0` z9|$;ZKHfiGewyx{^!FEgDQH}%LfifG?6jxz;^OmA(2jEoJ0&7)gGu!UllO^I8f!JJ zD+96p6clh9huuC0iRf2H!S@)QShZNE*nDC}(cyv$aD_2tN#&x@qN*@#vB(5Q6sf*9 zCnCnQwad@5=U$%}&;vDwot=;O+%jo@w0C`NZ*uC_LKE7M`q_$JD&Le`y2IQK))DYK zyiYdZ9q2U~;&;Fup+O!5QG0t8i(p2HM=D4=od$6Kyy7~!WCACRF( z13(xNGQ>xKGiEU$kULq41l$RQ&aj_m0Z$|tM;B!V*v9g?1#oCFK~q*5Gr+66gLNcm z?&-b~IEOHlLR#nuV*zLE|9KAT1h$*N z2igG2?kfbzAOp|8km$7KCLz!fB?|VVG0G_5^8Zb%Z3l{Lg-l^PFhhbP@e*)pj{pMSn^Hhh zA3)uK{vz+F2E2IEfnB@{W!l&=on|!xkl^NQ1cQ7DB|(nrtb}MjG~sn5np)FVSl)vW z#l{1u@o)6m-smPjVGop*6lFXMQ2D#9ig5i`XGku$1ma$Y>1!)kf)EJ3RYAceodNc- z+Ao1fFG^H-Cc+3l9)kQ|NT8}8<#t?4tY6Xu<8s{rrb4SW@xUbzxC7gFVaq9odLX2>2AuTpWpzk=|sC*Uv@9iXx8ZV8BpVMwD@KGKiVlilI04h$*BLO5# z6WKA13^{oM6(fL4i9JB5nI}Wk&wr&A;f&Kxg@H7_f5`r81Vjb^1}H~Btd-LqKzICK zV_ZUrab*yYik$lgaFz`H#DEnddlSU)v_J9$VgNt^w!p8u@Fi$P;Qr5kxL6jrEr)=s zdv`n}xb$Bb*pwl~K;1Z2!UGqk%2^DM5Lt|DMw0?k3^ zdw`lGRj9=41c^6fw?H@~{vez8cs(0=Y1|7n{a25iYiKi?(F6WVbDFg`G#`fEc>2&% zlW&~wMe^pun$suzxrB~w^lggDzF(TQY>4_B&@#adoUi>`(O-UmhVjCv&-6~jF?_m) z1sE?%8+AP(9Tq$|feieCg;<0flC8p#h0o)iFhq&+2~Tt?^d`V&0ubRf%qL7paY4aD zEf6ol+sTb%iHfLQ$UfXV-P#?xxVcg7t++Gw5mx!aP4UTcD_f1V+jnp_{BUn;@8kVf z8eGjGoLU`rOKmXZLH15%&5a8SCQELNHJx;J-2b_IQ3li^bmzMe_m}C5B`}O1H>zTS zaTFYBju9h4T~_)N{NgG5+wHL**73RGy(Oe82Zm{i@UL3E>XVZ81eayUk)uB)(VSmj z=lc5aq@MJzO+fJN1d=YmR4(cJqS+5&NW?nhOb>@lcs`Xz(Z3L-84TjU zh&^N38Qrs2$0iQ40qMnq5v@63_M^*VkY#5~#Pg`{Yn-dc1h)@wGpgTXIFrUpy_T@p z@0igXx`0Y+fP!HgF&%1hb>wnhfYry4moLSN9lrgEAuMJHyyJhpa)1yleAx|b4NTw{ zL%E&efv(qWSh;Y}_o+1CSI#k{8J!Q{&E<}Oj|aj~WgfGY|JR^y2xjdIdn7gniy&Sk zhQB&3jUnn=c#t3vvhehJ1aMn~q7h{RU1V`+Zb<*@C^nFF zrHMmtz^`BIL~v7SUB!qztdWQZF1|!YVhNs0a>}7X44%0zJ|(zRE)#LY4&bZ$b60r@ z(^ce7o0vc70|qpvxOWf40lUzkVA zXDS(N=(D6?+__l>@j|#3=Av$HEW2aKtqXcumBOr(Sb>oncFm$|4((c6XFbtr zq`hg`=)>26nPY}&3U#&^A(lY`9uePcx>L`vVEs2~!4dl6BMk2v>lsTwE1`s^Z}Y(G2z0su0R9IFjs={st&o;7W)sH%_5_Ib z|Fs$d-DEfINHBxMItI-u;QtWPeG5d?NFWY7=5K+2qwKlYTjO~Yis^09B>{HER2Uc+ z^S-khILGrwLvO`$0kYrI=fI%yB0+{+e-|@?jL?ICbweOi zLh|;1XbwbRg~K4UN7#f9KoxTEKQs(JzX!gY11M7id}wLGPe3E*4qrU(bO4kN>-MAMH`c^Cqyw2GIQ^e-CWGXfFj6e{B@g21%i@ z&_?`s>EjOi24uuhhsQG14S;a|^1TA>oLygZCzab^awkmY_?3W^klpfH!zW3X&39}^ z@3Qeqa!=CM26Vn}{$*RL==;o}_vQ6OHL}vb66c;`N60-+msK(DFvZPl4APRK`Q3$0 z+NZzF8!Yayd`jaPt=b50e0Ig}o0Xg?z!Ce5%#OcU9Kl^IW{8}QE&_YTad$PIC9WA~kXuIb4W#5i4hcFjAoP+%L1Gk-CG2|jR*&qch%HeS&dB+9pqTNPJSQ;>R^Q1}?0K6Tsm^GMQbR*E5I)pcRkspf*VCDWswMG8?l zGWE;GyX6p}N`3-L+c&}6?mTbAnDxCv8I>n~E7ird)vhWewOSPpN;~!+c>mraR)p_3 zn?{#=X7MZYR;S%NkJU0|H=d~}gi}?$)ITetnQZ)w)A96Lb3||4rgYJmj$hV0#epFy zAT61S2pJV`-U#mr+3M4SUmq~NI(LgSUpBi9o5Xq=w!~a(s1^I^M^0+xPdXAma<{DB zk$+PY*O!$mg4vN~+D~||l+0c%z}FpyA#QW8KsRF_v(%(x!q-RJTI@8-Bw`eEW+0h-n?1ZCqDiCS-$_?i9NJ_@Tl{=u_7a0C?E|_* zQ~dJ1@AVCP^=zKieB*3`pZ0k=8gDL7ac(x1Ru@ciRwz8d;*A9D6qdrZO*i`LB*m;G z8A3)6e2%TLPo?L;Y7VDu7W?62NVcbSI#13|mh-(!k|$wCToK0tBRs0bGyT$w$2y+R zb{8A7lW$y3cJ+ronswF3hxS-Mf)S3l_%gA~)k>G3sP9VGc+7ueHdIybC3!I+I;tw1 z=_R0P{M_8LL5?K3mkf z0Q&#oh6vD&)B|^O!wpblc2XB@xNBQ=u3uPO>7D*bYAZIZE4g&~TyJU?6N~B5ns8B2 zRAS06oIq}DPm<8M3%rf$CGNOe2x2snXDxRJ zk2sdA=nQbA6Og`R{Pp!D^$kkQoT|e*0|V>M2WA^(B&+sO7>~44PVJ?#gx79F3jDWk zV;Q+-5-@AjIQJ+y1nb?ffvKicTiJd_F-{TA`3L!J1M>_s`0tGdAsQv!vVo6|^KSW6 zPH(lzI8xSh6Xg;)d9;bwC>t>d(;Ve$KlruGIl4}VefxBx8`|5b2+Qc!XEV z{yyQP0-PH5eVgo=f z1_-%ooQBv`am?&1BG==6e}?OIevLa=>Df_@?NMp+Zu41u($lb% zpvt4?m)nkrT@i|0Vku$)S&;8bwljFGN#?0_>sI6sIqcD?jNljSVzyPc12XQJ2VY>X zDH%Wf<+~qig~O(gqA3a3#%qrProC@`*qs|w^l~vaRV-arY#IEvkZV#+yIheZdeVH3e%Vv$u6Bo1$^LP-SgP!(r3iefl7h?Yb;@ zD(`-Baky3=3x2a6nr=a|3=(5G7>u~;jy!pXKT0Uy- z46tE((9@vJPfH0JWq zSyO8)Sk5?EO`hswexQc(P4i5-VYwPtmrKJv9z`tcxt2#*J_uJCYM@aphg(Dy?qr9Y zr!0kA1on&CvMVfaQIb$y7syRKmSb_sGO{`NVJEZP1LpgWVg=X(St?2?RzKdRyFB7)-Gk<+PgxRSmFTIV{4h zp>_4&Hio}s&%aDc%@#`9YHXx6+dSX*;L<6`La5!GjYb!|V?OC$Fz|0rWb#YQVbmV` zqhMC}p73>tR>Qk+Z^rJ_n}d?Wv!?r3c*(F&g~07!!D~AAkt9|1xY#T4 zC|uc`>z#bpgFHKEBhLu7WU=Nx;N++25Ey(AkL6b>P-q;4eo4e`Q0M+`*W2p$6ZthM z9cH=A%(c>bElM9aya@WT6@kgCXcbG?AN+EfG4&9g@WZt}K^H`R@`eg1jKX7#OJ4Pp z)kkG3hW~Na%sDMZvxOqClAUTU=`M2J)`KTSJ4#TQ6sd)?I)hmQG0b@M?a@c0QLMH! z)Vi5N<+aMPU)=BFtBV_OaNd`lH{?LCq_s%gT$R~+s}lC{1S6$54obehGB-ESd~6sa zF6+qPI579={ia9c;r;i4R&jqb4>~w$QvRN3b^yb%){Xot? zTcwFQtl@c2N@9Vy;T(y^Vvb^Udt{ar&duAo*DeDrds!6lB$$|}-frxG;wStFDj=}UgN^|b|+_R^E=6iO- zm^VCoD?XC7e?s@_OJL$oh4i4lz~YdPmE7AQ4bK;gx{iJxQ8i||lK#mL&yt;joyJ-O zzSNeW>!#pS8WObDdHB9!kUf+rN=`|-3f#ON71e&1U3c%N%@hn0AoWa%(X6@oh4D^X zk-4qNyB=my>h)U`WVY69+$L0eQoMP2KQl=*P1Oj)OmC~_5$=8KH#BTHI3NCbja0$i zk@v0VC%QEn$!Q?W4x4!M6hgIpj}?WMak7O|f}SIs6`Z$$ibhihbAZ%du{VR>zP`@M z!*W22L`$v1(R@rLoNHxP{VK}_EZv16X(oim!ks&C2DG0_)DdL_q&j62@d{DY?v5O@ zQYD8XklH1Z?KUk61}Y=*yiW$ex5UR)>-0!%j=VKn`GXB|QS{uxFf>eSeh&4S6UTtL zg6MEj-?}A$*Y;_;WTe9^PPCHeN}yeL(&ZbSmi8vVgLLXn5dES*$-%V0+JLDdYE1pU zUbuc&`lHg5UWWP{##@FS3yaVp*X}7e*6h-(3QE)|!!IVHn8}AZ4&Q`_v=hUnvZH=H z6z;v}zKN&A^>yC(qikB;_a(RPuKU#2~89DaS8sk;LEHce4U3!s@VI9 zCv$N#vFMAX#qlB5F(}kv#wHEUU=Ox_r$LNrQCjwuLHVPTrO!WqLu-2C+yw5GQsmn{ ze9El}r)oXhw_$shks+K}czXJ4GU-Uk%+O;*R!3#)|2o#%toa7mN94aBR2-B3f z{8nV~3Uxg{)uIa(r3Ihk^Wrm(LTr$9JPzdWa?tHa^Us^ zX%OZR3+B~zbZyNyay)fdqJR6_`s4GvHz&fh@3J~h(!ySigo)2g@oi3A|HrLB5(((D zTPA5|i}+C12caHmQgZj>v{r7Sc017xNO$A2X>DC>Nrb&HV}(s#Rx3YH$x?rLWv zY7(C+((4D=u>Mr2;Ht~CD)W~{R_T1db(}w$=ZUnwhHBUXh;T0%O`ocj%G`ReVkGkFTbuk%kcLJO~ zB9e``Vjwfv$6XeT8_YfaQ}++c8>s zoCC3JPj)ox%DpJ1pODf1SISe>z~fVbqCiHaNCh$rT((sYyP;1!N^_|1o%KmMcO4y! zwhx%~UV% zDuui|Lh#P98W>rDEdIm`TC7OruB7o&fN;ZI1t^vo?!pvV!G1buB`@T@GznSIR2kR^ z(7C23sP$v=i>0Pw5^}p#wLZr+#ptsk_H+G+0jbr>u=DL2IyXTu{Sf&KuluMbolDc(_#gu|pJq?*z#x+c^P{Bl;jamf{v#bp z9dMnkNDeR!Mu_M9pKooqJ2CO?Ly094%SY~Sz|-vF3RtT?+Embz+pfeN%DfwU@SG~i z$KE*Nt1XSd#9gwnzM*Eap}pK)Y-U2wKfH6j=>awdiERRRyVgj%59t%n+1Fh$f2ZE? zyU>5YHLWg+(;l=ZrsfJJ3Qpa2T@~z)*5%jqypm3F(5*Ey`KgLa$&lOfixI`PLQ~~W(Z*%=D-uZ^ zg$zj*Y+76hF{%z`8s*oYAG&m*R|F{i?>FV<(Q3+ zUK!T{IC`IcNpnFQ%-oI!RsS@!mXcUt?sZ&_x{>;O-_=VF*U;PY?>97T_btBj{Cp2v zsS}&`3q8w}PAuLe>OJ^MZa+i=i<|n8yijsTUO__%N);-jscgK6x$1knR{A^)+9_a* z&rg7=!%GmHTJ+h!Ldm$Ey*pGpPx_aw;nPzc0Tbq#@z`EBpO(+?r} zjTy+Va_wRrdG+CjO&&90;96w4FcIWM=zP3|++ z$}>pq5H7O#aYUDAJ2(XK&26?ZJ;1w5HsJ+$Wt+w-f0Lo&GDbb&6X%H5`lsf_RsH^t zO40I4M(#r??|w=|zYfQ|pdrR4_j3}K?gXPNkEuG;iZfK+xWHo1-lpUBMDekwqR@TF zO7n?ha&j{)2rJokEQGSm1O#LRm}QLqQ3c0dP>5o6nolQ6Qxufd^G!&=r>=Gz=pQER ze+ls5Lv~3y^_$jtc&gHG+BIjah%^C%Snqa}O1aO)r>EVWElHlyA}bX$*6<(QHtnF~ z_R42|>sw!Z;mrp&x|#OS^VS60=`-Xf%h`~|t*9Do@p^Ip-5V44?j^SRN7yu&ic6%+ z1;-D%wTPEpPu+V59@3>Y(o*=AJ~QywD|9LcF}`iElJTKH3*zUbXC`Z+q|8>Hpy#7$ z2D9ccGpf(*#p}JSY24O`rgX$G$3&8Rpj=&aJNw;Ez;l>-mH6PNd`a}QS*aJd&q@2) z3QL~l6Ta&|!2|WX5xN2rn|4Fw8p6N_bW`U%W)sCq?m)3tK(vE^+hO*5lnFRU`@#?* zK#Un)M6?in8EO7c6*>7o!J`bA`vTx%{5$$T!NWX^-C)GD0B0%8& zo%G+tW*{Cg?3xp2>huS5TAV$K?x+9%6oCzwhcn;Q5e4ZNoN{`gEo|*zi4l&Z_gI9^ z!{KQ?dkqcd_14R%hFPZ~;=7&s%6<5nK9)GoBx#wyO8ugvHzzIpDrVV$N>VUjoN!k_ zVnJ61L8dm0V)yE2E;6oqrhytSp)l|=2o)MfE!X}8zqR9 zm#F!9auvh!*K1#CvIvmyP7Di^>%EG;(@L!lfw8w=Sk!vS#*{2HT+WC$S+M*cOTjQx zNu(n1ZKFU|5K(QsdvTqVejRg_crSysCZ_6I#G9#=ZROD6`~f z?J9WSmPRDjWo>6>$ugMu{)zQfRCnZDSEI0dPOtm?BvXg$(VO@u^yp({FNdsmX4@*P z6@OaUHJ&kjSrQP6fxx&bep0FKfh}h@YOSO9?eHU1dq2d_9I{u~KBO5McypQ<9JmdW*1|`_YEq~p~F`<_I2((hwrhfEPA@A`N_FbgpoZ4Y-0p`~@ zH65$%-e>e#QsB_DZpPpa=`;TAptn9IkEwwD>e1W&I;ye(tfbFMu}X*BxFsjHU2X-b z;0vQ^+!ZxuYU;ZW+F%{GuKncN`THhLV}605y`J~hL&@?%;ZU!~jyFXasxI_>W=fAk zLTl$?%B;9o6 zgt2d|sg{LASxmV5XRfb{l+x-*Ip%7+Oi?2t72tqOG_ESY>z#V1I$ddnvP2zTIO^Ls zsVvU@VL00sdSjVlaa;XCG0z(PSNo|&IVKBzY{NSOMqSb@_Wyh<9iDHv{%SXo{^dB= ztPpUT{}>vM1P>i&1^)V3_F|iH$bD!5v-mI*v)J=wfJ*FE3wCbg%SjeRCTa}1lM6Zv zUxod6(Sf8TSxCGwxNXn=IWd-xT^CaF&~OM95#zMIJUfa&{5&dK zUsW5gpIdwnpVl*e+T}pJT}z>n9A%!9Eho_+XZcPu9WB%2_W8^~l#I*tWqdpB?Y8}e zD(Z)+4`ZFS!yy TXuVZEP!!LG1Pc^VRFwY*{GJvU literal 0 HcmV?d00001 diff --git a/qa/integration/fixtures/pq_drain_spec.yml b/qa/integration/fixtures/pq_drain_spec.yml new file mode 100644 index 00000000000..9b135a29e70 --- /dev/null +++ b/qa/integration/fixtures/pq_drain_spec.yml @@ -0,0 +1,3 @@ +--- +services: + - logstash \ No newline at end of file diff --git a/qa/integration/specs/pq_drain_spec.rb b/qa/integration/specs/pq_drain_spec.rb new file mode 100644 index 00000000000..b524d7547b8 --- /dev/null +++ b/qa/integration/specs/pq_drain_spec.rb @@ -0,0 +1,142 @@ +# Licensed to Elasticsearch B.V. under one or more contributor +# license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright +# ownership. Elasticsearch B.V. licenses this file to you under +# the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +require_relative '../framework/fixture' +require_relative '../framework/settings' +require_relative '../services/logstash_service' +require_relative '../framework/helpers' +require "logstash/devutils/rspec/spec_helper" + +require 'stud/temporary' + +if ENV['FEATURE_FLAG'] == 'persistent_queues' + describe "Test logstash queue draining" do + before(:all) { @fixture = Fixture.new(__FILE__) } + after(:all) { @fixture&.teardown } + + let(:logstash_service) { @fixture.get_service("logstash") } + + shared_examples 'pq drain' do |queue_compression_setting| + let(:settings_flags) { super().merge('queue.drain' => true) } + + around(:each) do |example| + Stud::Temporary.directory('data') do |tempdir| + # expand the fixture tarball into our temp data dir + data_dir_tarball = File.join(__dir__, '../fixtures/data_dirs/mixed-compression-queue-data-dir.tar.gz') + `tar --directory #{Shellwords.escape(tempdir)} --strip-components 1 -xzf "#{Shellwords.escape(data_dir_tarball)}"` + + @data_dir = tempdir + example.call + end + end + + around(:each) do |example| + Stud::Temporary.file('output') do |tempfile| + @output_file = tempfile.path + example.call + end + end + + let(:pipeline) do + <<~PIPELINE + input { generator { count => 1 type => seed } } + output { file { path => "#{@output_file}" codec => json_lines } } + PIPELINE + end + + it "reads the contents of the PQ and drains" do + + unacked_queued_elements = Pathname.new(@data_dir).glob('queue/main/checkpoint*').map { |cpf| decode_checkpoint(cpf) } + .map { |cp| (cp.elements - (cp.first_unacked_seq - cp.min_sequence)) }.reduce(&:+) + + invoke_args = %W( + --log.level=debug + --path.settings=#{File.dirname(logstash_service.application_settings_file)} + --path.data=#{@data_dir} + --pipeline.workers=2 + --pipeline.batch.size=8 + --config.string=#{pipeline} + ) + invoke_args << "-Squeue.compression=#{queue_compression_setting}" unless queue_compression_setting.nil? + + status = logstash_service.run(*invoke_args) + + aggregate_failures('process output') do + expect(status.exit_code).to be_zero + expect(status.stderr_and_stdout).to include("queue.type: persisted") + expect(status.stderr_and_stdout).to include("queue.drain: true") + expect(status.stderr_and_stdout).to include("queue.compression: #{queue_compression_setting}") unless queue_compression_setting.nil? + end + + aggregate_failures('processing result') do + # count the events, make sure they're all the right shape. + expect(::File::size(@output_file)).to_not be_zero + + written_events = ::File::read(@output_file).lines.map { |line| JSON.load(line) } + expect(written_events.size).to eq(unacked_queued_elements + 1) + timestamps = written_events.map {|event| event['@timestamp'] } + expect(timestamps.uniq.size).to eq(written_events.size) + end + + aggregate_failures('resulting queue state') do + # glob the data dir and make sure things have been cleaned up. + # we should only have a head checkpoint and a single fully-acked page. + checkpoints = Pathname.new(@data_dir).glob('queue/main/checkpoint*') + expect(checkpoints.size).to eq(1) + expect(checkpoints.first.basename.to_s).to eq('checkpoint.head') + checkpoint = decode_checkpoint(checkpoints.first) + expect(checkpoint.first_unacked_page).to eq(checkpoint.page_number) + expect(checkpoint.first_unacked_seq).to eq(checkpoint.min_sequence + checkpoint.elements) + + pages = Pathname.new(@data_dir).glob('queue/main/page*') + expect(pages.size).to eq(1) + end + end + end + + context "`queue.compression` setting" do + %w(none speed balanced size).each do |explicit_compression_setting| + context "explicit `#{explicit_compression_setting}`" do + include_examples 'pq drain', explicit_compression_setting + end + end + context "default setting" do + include_examples 'pq drain', nil + end + end + end + + def decode_checkpoint(path) + bytes = path.read(encoding: 'BINARY').bytes + + bstoi = -> (bs) { bs.reduce(0) {|m,b| (m<<8)+b } } + + version = bstoi[bytes.slice(0,2)] + pagenum = bstoi[bytes.slice(2,4)] + first_unacked_page = bstoi[bytes.slice(6,4)] + first_unacked_seq = bstoi[bytes.slice(10,8)] + min_sequence = bstoi[bytes.slice(18,8)] + elements = bstoi[bytes.slice(26,4)] + + OpenStruct.new(version: version, + page_number: pagenum, + first_unacked_page: first_unacked_page, + first_unacked_seq: first_unacked_seq, + min_sequence: min_sequence, + elements: elements) + end +end \ No newline at end of file diff --git a/tools/dependencies-report/src/main/resources/licenseMapping.csv b/tools/dependencies-report/src/main/resources/licenseMapping.csv index 128eef59960..14fb7c8c28a 100644 --- a/tools/dependencies-report/src/main/resources/licenseMapping.csv +++ b/tools/dependencies-report/src/main/resources/licenseMapping.csv @@ -33,6 +33,7 @@ dependency,dependencyUrl,licenseOverride,copyright,sourceURL "com.fasterxml.jackson.core:jackson-databind:",https://github.com/FasterXML/jackson-databind,Apache-2.0 "com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:",https://github.com/FasterXML/jackson-dataformats-binary,Apache-2.0 "com.fasterxml.jackson.module:jackson-module-afterburner:",https://github.com/FasterXML/jackson-modules-base,Apache-2.0 +"com.github.luben:zstd-jni:1.5.7-4",https://github.com/luben/zstd-jni,BSD-2-Clause "com.google.googlejavaformat:google-java-format:",https://github.com/google/google-java-format,Apache-2.0 "com.google.guava:guava:",https://github.com/google/guava,Apache-2.0 "com.google.j2objc:j2objc-annotations:",https://github.com/google/j2objc/,Apache-2.0 diff --git a/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt b/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt new file mode 100644 index 00000000000..4accd5fd41e --- /dev/null +++ b/tools/dependencies-report/src/main/resources/notices/com.github.luben!zstd-jni-NOTICE.txt @@ -0,0 +1,26 @@ +source: https://github.com/luben/zstd-jni/blob/v1.5.7-4/LICENSE + +Copyright (c) 2015-present, Luben Karavelov/ All rights reserved. + +BSD-2-Clause License https://opensource.org/license/bsd-2-clause + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. \ No newline at end of file From 9e8076574cf1ba0128a2900156874a566339276a Mon Sep 17 00:00:00 2001 From: Rye Biesemeyer Date: Mon, 29 Sep 2025 06:54:42 -0700 Subject: [PATCH 25/44] metrics: add support for user-defined metrics (#18218) * metrics: add support for user-defined metrics * Update logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java Co-authored-by: Andrea Selva --------- Co-authored-by: Andrea Selva --- .../lib/logstash/instrument/collector.rb | 6 ++ .../logstash/api/NamespacedMetric.java | 10 +++ .../co/elastic/logstash/api/UserMetric.java | 64 +++++++++++++++++ .../metrics/AbstractNamespacedMetricExt.java | 7 ++ .../metrics/AbstractSimpleMetricExt.java | 7 ++ .../instrument/metrics/MetricExt.java | 6 ++ .../instrument/metrics/MetricType.java | 5 ++ .../metrics/NamespacedMetricExt.java | 5 ++ .../instrument/metrics/NullMetricExt.java | 5 ++ .../metrics/NullNamespacedMetricExt.java | 5 ++ .../instrument/metrics/UserMetric.java | 42 ++++++++++++ .../plugins/NamespacedMetricImpl.java | 9 +++ .../instrument/metrics/MetricTypeTest.java | 1 + .../metrics/MockNamespacedMetric.java | 7 +- .../plugins/NamespacedMetricImplTest.java | 68 +++++++++++++++++++ 15 files changed, 246 insertions(+), 1 deletion(-) create mode 100644 logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java create mode 100644 logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java diff --git a/logstash-core/lib/logstash/instrument/collector.rb b/logstash-core/lib/logstash/instrument/collector.rb index 1a467f3cce3..a1a7fc3c071 100644 --- a/logstash-core/lib/logstash/instrument/collector.rb +++ b/logstash-core/lib/logstash/instrument/collector.rb @@ -74,6 +74,12 @@ def get(namespaces_path, key, type) end end + ## + # @return [Metric]: the metric that exists after registration + def register(namespaces_path, key, &metric_supplier) + @metric_store.fetch_or_store(namespaces_path, key, &metric_supplier) + end + # test injection, see MetricExtFactory def initialize_metric(type, namespaces_path, key) MetricType.create(type, namespaces_path, key) diff --git a/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java b/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java index ef2574cad36..2a5fe5b085d 100644 --- a/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java +++ b/logstash-core/src/main/java/co/elastic/logstash/api/NamespacedMetric.java @@ -51,6 +51,16 @@ public interface NamespacedMetric extends Metric { */ TimerMetric timer(String metric); + /** + * Creates or retrieves a {@link UserMetric} with the provided {@code metric} name, + * using the supplied {@code userMetricFactory}. + * @param metric the name of the metric + * @param userMetricFactory a factory for creating the metric + * @return the resulting metric at the address, whether retrieved or created + * @param the type of metric to create + */ + > USER_METRIC register(String metric, UserMetric.Factory userMetricFactory); + /** * Increment the {@code metric} metric by 1. * diff --git a/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java b/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java new file mode 100644 index 00000000000..fa8a633faab --- /dev/null +++ b/logstash-core/src/main/java/co/elastic/logstash/api/UserMetric.java @@ -0,0 +1,64 @@ +package co.elastic.logstash.api; + +import java.util.function.Function; + +/** + * A custom metric. + * @param must be jackson-serializable. + * + * NOTE: this interface is experimental and considered internal. + * Its shape may change from one Logstash release to the next. + */ +public interface UserMetric { + VALUE_TYPE getValue(); + + /** + * A {@link UserMetric.Factory} is the primary way to register a custom user-metric + * along-side a null implementation for performance when metrics are disabled. + * + * @param a sub-interface of {@link UserMetric} + */ + interface Factory> { + Class getType(); + USER_METRIC create(String name); + USER_METRIC nullImplementation(); + } + + /** + * A {@link UserMetric.Provider} is an intermediate helper type meant to be statically available by any + * user-provided {@link UserMetric} interface, encapsulating its null implementation and providing + * a way to simply get a {@link UserMetric.Factory} for a given non-null implementation. + * + * @param an interface that extends {@link UserMetric}. + */ + class Provider> { + private final Class type; + private final USER_METRIC nullImplementation; + + public Provider(final Class type, final USER_METRIC nullImplementation) { + assert type.isInterface() : String.format("type must be an interface, got %s", type); + + this.type = type; + this.nullImplementation = nullImplementation; + } + + public Factory getFactory(final Function supplier) { + return new Factory() { + @Override + public USER_METRIC create(final String name) { + return supplier.apply(name); + } + + @Override + public Class getType() { + return type; + } + + @Override + public USER_METRIC nullImplementation() { + return nullImplementation; + } + }; + } + } +} diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java index 1a24b3cc03e..65f68633138 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java @@ -53,6 +53,11 @@ public IRubyObject timer(final ThreadContext context, final IRubyObject key) { return getTimer(context, key); } + @JRubyMethod + public IRubyObject register(final ThreadContext context, final IRubyObject key, final Block metricSupplier) { + return doRegister(context, key, metricSupplier); + } + @JRubyMethod(required = 1, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); @@ -104,5 +109,7 @@ protected abstract IRubyObject doReportTime(ThreadContext context, protected abstract IRubyObject doDecrement(ThreadContext context, IRubyObject[] args); + protected abstract IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier); + public abstract AbstractMetricExt getMetric(); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java index 758d9309eab..8f7f89f5d6a 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java @@ -74,6 +74,11 @@ public IRubyObject time(final ThreadContext context, return doTime(context, namespace, key, block); } + @JRubyMethod(name = "register") + public IRubyObject register(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final Block metricSupplier) { + return doRegister(context, namespace, key, metricSupplier); + } + protected abstract IRubyObject doDecrement(ThreadContext context, IRubyObject[] args); protected abstract IRubyObject doIncrement(ThreadContext context, IRubyObject[] args); @@ -88,4 +93,6 @@ protected abstract IRubyObject doReportTime(ThreadContext context, IRubyObject n protected abstract IRubyObject doTime(ThreadContext context, IRubyObject namespace, IRubyObject key, Block block); + + protected abstract IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block metricSupplier); } diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java index f842500e19d..4b1ef12e1dd 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricExt.java @@ -153,6 +153,12 @@ protected IRubyObject getTimer(final ThreadContext context, ); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block supplier) { + MetricExt.validateKey(context, null, key); + return collector.callMethod(context, "register", new IRubyObject[]{normalizeNamespace(namespace), key}, supplier); + } + @Override protected IRubyObject doReportTime(final ThreadContext context, final IRubyObject namespace, final IRubyObject key, final IRubyObject duration) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java index 0ca15cba310..de8708f1b65 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/MetricType.java @@ -77,6 +77,11 @@ public enum MetricType { * A flow-rate {@link FlowMetric}, instantiated with one or more backing {@link Metric}{@code }. */ FLOW_RATE("flow/rate"), + + /** + * A user metric + */ + USER("user"), ; private final String type; diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java index 388cfa7de7b..38952fc8542 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NamespacedMetricExt.java @@ -53,6 +53,11 @@ public NamespacedMetricExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return metric.register(context, namespaceName, key, metricSupplier); + } + @JRubyMethod(visibility = Visibility.PRIVATE) public NamespacedMetricExt initialize(final ThreadContext context, final IRubyObject metric, final IRubyObject namespaceName) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java index c004cc20b88..3a3c283377e 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullMetricExt.java @@ -111,6 +111,11 @@ protected IRubyObject doTime(final ThreadContext context, final IRubyObject name return block.call(context); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject namespace, IRubyObject key, Block metricSupplier) { + return context.nil; + } + @Override protected AbstractNamespacedMetricExt createNamespaced(final ThreadContext context, final IRubyObject name) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java index 152541b2271..112b89ac5e0 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java @@ -112,6 +112,11 @@ protected IRubyObject doReportTime(final ThreadContext context, final IRubyObjec return context.nil; } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return context.nil; + } + @Override @SuppressWarnings("rawtypes") protected RubyArray getNamespaceName(final ThreadContext context) { diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java new file mode 100644 index 00000000000..4c015efb571 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/UserMetric.java @@ -0,0 +1,42 @@ +package org.logstash.instrument.metrics; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jruby.RubySymbol; +import org.jruby.runtime.Block; +import org.jruby.runtime.JavaInternalBlockBody; +import org.jruby.runtime.Signature; +import org.jruby.runtime.ThreadContext; +import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; + +public class UserMetric { + private UserMetric() {} + + private static Logger LOGGER = LogManager.getLogger(UserMetric.class); + + public static > USER_METRIC fromRubyBase( + final AbstractNamespacedMetricExt metric, + final RubySymbol key, + final co.elastic.logstash.api.UserMetric.Factory metricFactory + ) { + final ThreadContext context = RubyUtil.RUBY.getCurrentContext(); + + final Block metricSupplier = new Block(new JavaInternalBlockBody(context.runtime, Signature.NO_ARGUMENTS) { + @Override + public IRubyObject yield(ThreadContext threadContext, IRubyObject[] iRubyObjects) { + return RubyUtil.toRubyObject(metricFactory.create(key.asJavaString())); + } + }); + + final IRubyObject result = metric.register(context, key, metricSupplier); + final Class type = metricFactory.getType(); + if (!type.isAssignableFrom(result.getJavaClass())) { + LOGGER.warn("UserMetric type mismatch for %s (expected: %s, received: %s); " + + "a null implementation will be substituted", key.asJavaString(), type, result.getJavaClass()); + return metricFactory.nullImplementation(); + } + + return result.toJava(type); + } +} diff --git a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java index bb61ac12a1c..0a76ba59e3f 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java @@ -23,6 +23,7 @@ import co.elastic.logstash.api.CounterMetric; import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; +import co.elastic.logstash.api.UserMetric; import org.jruby.RubyArray; import org.jruby.RubyObject; import org.jruby.RubySymbol; @@ -34,6 +35,7 @@ import java.util.ArrayList; import java.util.List; +import java.util.Objects; import java.util.function.Supplier; import java.util.stream.Stream; @@ -67,6 +69,13 @@ public co.elastic.logstash.api.TimerMetric timer(final String metric) { return TimerMetric.fromRubyBase(metrics, threadContext.getRuntime().newString(metric).intern()); } + @Override + public > USER_METRIC register(String metric, UserMetric.Factory userMetricFactory) { + USER_METRIC userMetric = org.logstash.instrument.metrics.UserMetric.fromRubyBase(metrics, threadContext.runtime.newSymbol(metric), userMetricFactory); + + return Objects.requireNonNullElseGet(userMetric, userMetricFactory::nullImplementation); + } + @Override public NamespacedMetric namespace(final String... key) { final IRubyObject[] rubyfiedKeys = Stream.of(key) diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java index 21c6a7faf83..3c7a562b086 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MetricTypeTest.java @@ -52,6 +52,7 @@ public void ensurePassivity(){ nameMap.put(MetricType.GAUGE_RUBYHASH, "gauge/rubyhash"); nameMap.put(MetricType.GAUGE_RUBYTIMESTAMP, "gauge/rubytimestamp"); nameMap.put(MetricType.FLOW_RATE, "flow/rate"); + nameMap.put(MetricType.USER, "user"); //ensure we are testing all of the enumerations assertThat(EnumSet.allOf(MetricType.class).size()).isEqualTo(nameMap.size()); diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java index 2f784fa9119..bf93411d977 100644 --- a/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/MockNamespacedMetric.java @@ -25,7 +25,7 @@ public class MockNamespacedMetric extends AbstractNamespacedMetricExt { private static final long serialVersionUID = -6507123659910450215L; - private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); + private transient final ConcurrentMap metrics = new ConcurrentHashMap<>(); public static MockNamespacedMetric create() { return new MockNamespacedMetric(RubyUtil.RUBY, RubyUtil.NAMESPACED_METRIC_CLASS); @@ -86,6 +86,11 @@ public AbstractMetricExt getMetric() { return NullMetricExt.create(); } + @Override + protected IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier) { + return RubyUtil.toRubyObject(metrics.computeIfAbsent(key.asJavaString(), (k) -> metricSupplier.call(context))); + } + @Override protected AbstractNamespacedMetricExt createNamespaced(ThreadContext context, IRubyObject name) { return this; diff --git a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java index f705089f2ab..3f0890274fc 100644 --- a/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java +++ b/logstash-core/src/test/java/org/logstash/plugins/NamespacedMetricImplTest.java @@ -22,12 +22,17 @@ import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; +import co.elastic.logstash.api.UserMetric; import org.assertj.core.data.Percentage; import org.jruby.RubyHash; import org.junit.Ignore; import org.junit.Test; import java.util.Arrays; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.ConcurrentHashMap; +import java.util.stream.Stream; import static org.assertj.core.api.Assertions.assertThat; @@ -158,4 +163,67 @@ public void testRoot() { final NamespacedMetric namespaced = root.namespace("someothernamespace"); assertThat(namespaced.namespaceName()).containsExactly("someothernamespace"); } + + @Test + public void testRegister() { + final NamespacedMetric metrics = this.getInstance().namespace("testRegister"); + + CustomMetric leftCustomMetric = metrics.register("left", CorrelatingCustomMetric.FACTORY); + + // re-registering the same metric should get the existing instance. + assertThat(metrics.register("left", CorrelatingCustomMetric.FACTORY)).isSameAs(leftCustomMetric); + + // registering a new metric should be different instance + CustomMetric rightCustomMetric = metrics.register("right", CorrelatingCustomMetric.FACTORY); + assertThat(rightCustomMetric).isNotSameAs(leftCustomMetric); + + // this tests our test-only CustomMetric impl more than anything, but it validates + // that the instances we update are connected to their values. + leftCustomMetric.record("this"); + leftCustomMetric.record("that"); + rightCustomMetric.record("that"); + leftCustomMetric.record("this"); + rightCustomMetric.record("another"); + rightCustomMetric.record("that"); + rightCustomMetric.record("another"); + + assertThat(leftCustomMetric.getValue()).contains("this=2", "that=1"); + assertThat(rightCustomMetric.getValue()).contains("that=2", "another=2"); + } + + private interface CustomMetric extends UserMetric { + void record(final String value); + + UserMetric.Provider PROVIDER = new UserMetric.Provider(CustomMetric.class, new CustomMetric() { + @Override + public void record(String value) { + // no-op + } + + @Override + public String getValue() { + return ""; + } + }); + } + + private static class CorrelatingCustomMetric implements CustomMetric { + private final ConcurrentHashMap mapping = new ConcurrentHashMap<>(); + + static UserMetric.Factory FACTORY = CustomMetric.PROVIDER.getFactory((name) -> new CorrelatingCustomMetric()); + + @Override + public void record(String value) { + mapping.compute(value, (k, v) -> v == null ? 1 : v + 1); + } + + @Override + public String getValue() { + return Map.copyOf(mapping).entrySet() + .stream() + .sorted(Map.Entry.comparingByKey()) + .map((e) -> (e.getKey() + '=' + e.getValue().toString())) + .reduce((a, b) -> a + ';' + b).orElse("EMPTY"); + } + } } From 0d4ff1c52192ebb2ff8e02aee7f05c9546be6755 Mon Sep 17 00:00:00 2001 From: Rye Biesemeyer Date: Mon, 29 Sep 2025 06:55:09 -0700 Subject: [PATCH 26/44] metric: improve accuracy of timer metric under contention (#18219) * metric: improve accuracy of timer metric excludes contention for recording timing from the timing by locking in the completion time before attempting to record it * remove unused imports --- .../timer/ConcurrentLiveTimerMetric.java | 41 ++++++++++--------- 1 file changed, 22 insertions(+), 19 deletions(-) diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java index 4adbef05a0e..9b681944b59 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/ConcurrentLiveTimerMetric.java @@ -43,10 +43,13 @@ protected ConcurrentLiveTimerMetric(final String name) { @Override public T time(ExceptionalSupplier exceptionalSupplier) throws E { try { - trackedMillisState.getAndUpdate(TrackedMillisState::withIncrementedConcurrency); + trackedMillisState.getAndUpdate(existing -> existing.withIncrementedConcurrency(nanoTimeSupplier.getAsLong())); return exceptionalSupplier.get(); } finally { - trackedMillisState.getAndUpdate(TrackedMillisState::withDecrementedConcurrency); + // lock in the actual time completed, and resolve state separately + // so that contention for recording state is not included in measurement. + final long endTime = nanoTimeSupplier.getAsLong(); + trackedMillisState.getAndUpdate(existing -> existing.withDecrementedConcurrency(endTime)); } } @@ -65,13 +68,13 @@ private long getUntrackedMillis() { } private long getTrackedMillis() { - return this.trackedMillisState.getAcquire().getValue(); + return this.trackedMillisState.getAcquire().getValue(nanoTimeSupplier.getAsLong()); } interface TrackedMillisState { - TrackedMillisState withIncrementedConcurrency(); - TrackedMillisState withDecrementedConcurrency(); - long getValue(); + TrackedMillisState withIncrementedConcurrency(long asOfNanoTime); + TrackedMillisState withDecrementedConcurrency(long asOfNanoTime); + long getValue(long asOfNanoTime); } private class StaticTrackedMillisState implements TrackedMillisState { @@ -89,18 +92,18 @@ public StaticTrackedMillisState() { } @Override - public TrackedMillisState withIncrementedConcurrency() { - return new DynamicTrackedMillisState(nanoTimeSupplier.getAsLong(), this.cumulativeMillis, this.excessNanos, 1); + public TrackedMillisState withIncrementedConcurrency(final long asOfNanoTime) { + return new DynamicTrackedMillisState(asOfNanoTime, this.cumulativeMillis, this.excessNanos, 1); } @Override - public TrackedMillisState withDecrementedConcurrency() { + public TrackedMillisState withDecrementedConcurrency(final long asOfNanoTime) { throw new IllegalStateException("TimerMetrics cannot track negative concurrency"); } @Override - public long getValue() { + public long getValue(final long asOfNanoTime) { return cumulativeMillis; } } @@ -122,26 +125,26 @@ private class DynamicTrackedMillisState implements TrackedMillisState { } @Override - public TrackedMillisState withIncrementedConcurrency() { - return withAdjustedConcurrency(Vector.INCREMENT); + public TrackedMillisState withIncrementedConcurrency(final long asOfNanoTime) { + return withAdjustedConcurrency(asOfNanoTime, Vector.INCREMENT); } @Override - public TrackedMillisState withDecrementedConcurrency() { - return withAdjustedConcurrency(Vector.DECREMENT); + public TrackedMillisState withDecrementedConcurrency(final long asOfNanoTime) { + return withAdjustedConcurrency(asOfNanoTime, Vector.DECREMENT); } @Override - public long getValue() { - final long nanoAdjustment = getNanoAdjustment(nanoTimeSupplier.getAsLong()); + public long getValue(final long asOfNanoTime) { + final long nanoAdjustment = getNanoAdjustment(asOfNanoTime); final long milliAdjustment = wholeMillisFromNanos(nanoAdjustment); return Math.addExact(this.millisAtCheckpoint, milliAdjustment); } - private TrackedMillisState withAdjustedConcurrency(final Vector concurrencyAdjustmentVector) { + private TrackedMillisState withAdjustedConcurrency(final long asOfNanoTime, final Vector concurrencyAdjustmentVector) { final int newConcurrency = Math.addExact(this.concurrencySinceCheckpoint, concurrencyAdjustmentVector.value()); - final long newCheckpointNanoTime = nanoTimeSupplier.getAsLong(); + final long newCheckpointNanoTime = asOfNanoTime; final long totalNanoAdjustment = getNanoAdjustment(newCheckpointNanoTime); @@ -165,7 +168,7 @@ private long getNanoAdjustment(final long checkpointNanoTime) { /** * This private enum is a type-safety guard for - * {@link DynamicTrackedMillisState#withAdjustedConcurrency(Vector)}. + * {@link DynamicTrackedMillisState#withAdjustedConcurrency(long, Vector)}. */ private enum Vector { INCREMENT{ int value() { return +1; } }, From 4ed06586634c425d899951ed26eef9c918bfb96d Mon Sep 17 00:00:00 2001 From: Rye Biesemeyer Date: Mon, 29 Sep 2025 14:54:22 -0700 Subject: [PATCH 27/44] Pq compression user metrics (#18227) * pq-compression: wire through custom user metrics for ratio/spend * add test for queue compression metrics * pq metrics: IORatioMetric edge-case logging, use int at interface By using `int` type in `IORatioMetric#incrementBy(int,int)`, we simplify the failure scenarios while still allowing the desired behaviour, since this is always called in practice with `byte[]#length`. We ensure that attempts to decrement the value are ignored, and result in a log message, and that overflows reduce precision and are also logged. Together, these ensure that long overflows won't ever result in pipeline crashes. --- docs/static/spec/openapi/logstash-api.yaml | 53 +++++++ .../AbstractZstdAwareCompressionCodec.java | 17 ++- .../ackedqueue/AtomicIORatioMetric.java | 95 +++++++++++++ .../CalculatedRelativeSpendMetric.java | 58 ++++++++ .../logstash/ackedqueue/CompressionCodec.java | 49 ++++--- .../logstash/ackedqueue/IORatioMetric.java | 49 +++++++ .../java/org/logstash/ackedqueue/Queue.java | 12 +- .../logstash/ackedqueue/QueueFactoryExt.java | 37 +++-- .../ackedqueue/RelativeSpendMetric.java | 35 +++++ .../org/logstash/ackedqueue/Settings.java | 4 +- .../org/logstash/ackedqueue/SettingsImpl.java | 16 +-- .../ackedqueue/ZstdAwareCompressionCodec.java | 7 +- .../ZstdEnabledCompressionCodec.java | 17 ++- .../ackedqueue/ext/JRubyAckedQueueExt.java | 10 +- .../ext/JRubyWrappedAckedQueueExt.java | 40 +++++- .../execution/AbstractPipelineExt.java | 3 +- .../instrument/metrics/AbstractMetricExt.java | 2 + .../metrics/AbstractNamespacedMetricExt.java | 7 + .../metrics/AbstractSimpleMetricExt.java | 7 + .../metrics/NullNamespacedMetricExt.java | 7 + .../metrics/timer/TimerMetricFactory.java | 4 + .../plugins/NamespacedMetricImpl.java | 23 ++++ .../ackedqueue/AtomicIORatioMetricTest.java | 129 ++++++++++++++++++ .../CalculatedRelativeSpendMetricTest.java | 47 +++++++ .../ackedqueue/CompressionCodecTest.java | 40 +++--- .../instrument/metrics/VisibilityUtil.java | 13 ++ qa/integration/specs/monitoring_api_spec.rb | 8 ++ 27 files changed, 716 insertions(+), 73 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java create mode 100644 logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java create mode 100644 logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java create mode 100644 logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java create mode 100644 logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java diff --git a/docs/static/spec/openapi/logstash-api.yaml b/docs/static/spec/openapi/logstash-api.yaml index b979006b7dd..c5ac40b7374 100644 --- a/docs/static/spec/openapi/logstash-api.yaml +++ b/docs/static/spec/openapi/logstash-api.yaml @@ -2340,6 +2340,59 @@ components: max_queue_size_in_bytes: type: integer format: int64 + compression: + type: object + properties: + encode: + type: object + properties: + ratio: + type: object + description: the ratio of event size in bytes to its representation on disk + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + spend: + type: object + description: the fraction of wall-clock time spent encoding events + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + decode: + type: object + properties: + ratio: + type: object + description: the ratio of event representation on disk to event size + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" + spend: + type: object + description: the fraction of wall-clock time spent decoding events + properties: + lifetime: + oneOf: + - type: number + - enum: + - "Infinity" + - "NaN" + - "-Infinity" - type: object description: "The metrics of memory queue." required: diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java index 1cb54c98a77..9cf159d4ce4 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/AbstractZstdAwareCompressionCodec.java @@ -1,5 +1,7 @@ package org.logstash.ackedqueue; +import co.elastic.logstash.api.Metric; +import co.elastic.logstash.api.NamespacedMetric; import com.github.luben.zstd.Zstd; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -13,13 +15,26 @@ abstract class AbstractZstdAwareCompressionCodec implements CompressionCodec { // log from the concrete class protected final Logger logger = LogManager.getLogger(this.getClass()); + private final IORatioMetric decodeRatioMetric; + private final RelativeSpendMetric decodeTimerMetric; + + public AbstractZstdAwareCompressionCodec(Metric queueMetric) { + final NamespacedMetric decodeNamespace = queueMetric.namespace("compression", "decode"); + decodeRatioMetric = decodeNamespace.namespace("ratio") + .register("lifetime", AtomicIORatioMetric.FACTORY); + decodeTimerMetric = decodeNamespace.namespace("spend") + .register("lifetime", CalculatedRelativeSpendMetric.FACTORY); + } + @Override public byte[] decode(byte[] data) { if (!isZstd(data)) { + decodeRatioMetric.incrementBy(data.length, data.length); return data; } try { - final byte[] decoded = Zstd.decompress(data); + final byte[] decoded = decodeTimerMetric.time(() -> Zstd.decompress(data)); + decodeRatioMetric.incrementBy(data.length, decoded.length); logger.trace("decoded {} -> {}", data.length, decoded.length); return decoded; } catch (Exception e) { diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java new file mode 100644 index 00000000000..fc3426a515a --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/AtomicIORatioMetric.java @@ -0,0 +1,95 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.UserMetric; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.logstash.instrument.metrics.AbstractMetric; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; +import java.util.concurrent.atomic.AtomicReference; + +/** + * It uses {@code long} under the hood, and is capable of handling sustained 1GiB/sec + * for ~272 years before overflowing. + */ +class AtomicIORatioMetric extends AbstractMetric implements IORatioMetric { + + public static UserMetric.Factory FACTORY = IORatioMetric.PROVIDER.getFactory(AtomicIORatioMetric::new); + + private static final MathContext LIMITED_PRECISION = new MathContext(4, RoundingMode.HALF_UP); + private static final ImmutableRatio ZERO = new ImmutableRatio(0L, 0L); + private static final Logger LOGGER = LogManager.getLogger(AtomicIORatioMetric.class); + + private final AtomicReference atomicReference = new AtomicReference<>(ZERO); + private final Logger logger; + + AtomicIORatioMetric(final String name) { + this(name, LOGGER); + } + + AtomicIORatioMetric(final String name, final Logger logger) { + super(name); + this.logger = logger; + } + + @Override + public Value getLifetime() { + return atomicReference.get(); + } + + @Override + public void incrementBy(int bytesIn, int bytesOut) { + if (bytesIn < 0 || bytesOut < 0) { + logger.warn("cannot decrement IORatioMetric {}", this.getName()); + return; + } + this.atomicReference.getAndUpdate((existing) -> doIncrement(existing, bytesIn, bytesOut)); + } + + // test injection + void setTo(long bytesIn, long bytesOut) { + this.atomicReference.set(new ImmutableRatio(bytesIn, bytesOut)); + } + + @Override + public Double getValue() { + final Value snapshot = getLifetime(); + + final BigDecimal bytesIn = BigDecimal.valueOf(snapshot.bytesIn()); + final BigDecimal bytesOut = BigDecimal.valueOf(snapshot.bytesOut()); + + if (bytesIn.signum() == 0) { + return switch(bytesOut.signum()) { + case -1 -> Double.NEGATIVE_INFINITY; + case 1 -> Double.POSITIVE_INFINITY; + default -> Double.NaN; + }; + } + + return bytesOut.divide(bytesIn, LIMITED_PRECISION).doubleValue(); + } + + public void reset() { + this.atomicReference.set(ZERO); + } + + private ImmutableRatio doIncrement(final ImmutableRatio existing, final int bytesIn, final int bytesOut) { + + final long combinedBytesIn = existing.bytesIn() + bytesIn; + final long combinedBytesOut = existing.bytesOut() + bytesOut; + + if (combinedBytesIn < 0 || combinedBytesOut < 0) { + logger.warn("long overflow; precision will be reduced"); + final long reducedBytesIn = Math.addExact(Math.floorDiv(existing.bytesIn(), 2), bytesIn); + final long reducedBytesOut = Math.addExact(Math.floorDiv(existing.bytesOut(), 2), bytesOut); + + return new ImmutableRatio(reducedBytesIn, reducedBytesOut); + } + + return new ImmutableRatio(combinedBytesIn, combinedBytesOut); + } + + public record ImmutableRatio(long bytesIn, long bytesOut) implements Value { } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java new file mode 100644 index 00000000000..c70adcba7a3 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetric.java @@ -0,0 +1,58 @@ +package org.logstash.ackedqueue; + +import org.logstash.instrument.metrics.AbstractMetric; +import org.logstash.instrument.metrics.UptimeMetric; +import org.logstash.instrument.metrics.timer.TimerMetric; +import org.logstash.instrument.metrics.timer.TimerMetricFactory; + +import java.math.BigDecimal; +import java.math.MathContext; +import java.math.RoundingMode; + +class CalculatedRelativeSpendMetric extends AbstractMetric implements RelativeSpendMetric { + private static final MathContext LIMITED_PRECISION = new MathContext(4, RoundingMode.HALF_UP); + + private final TimerMetric spendMetric; + private final UptimeMetric uptimeMetric; + + public static Factory FACTORY = RelativeSpendMetric.PROVIDER.getFactory(CalculatedRelativeSpendMetric::new); + + public CalculatedRelativeSpendMetric(final String name) { + this(name, TimerMetricFactory.getInstance().create(name + ":spend"), new UptimeMetric(name + ":uptime")); + } + + CalculatedRelativeSpendMetric(String name, TimerMetric spendMetric, UptimeMetric uptimeMetric) { + super(name); + this.spendMetric = spendMetric; + this.uptimeMetric = uptimeMetric; + } + + @Override + public T time(ExceptionalSupplier exceptionalSupplier) throws E { + return this.spendMetric.time(exceptionalSupplier); + } + + @Override + public void reportUntrackedMillis(long untrackedMillis) { + this.spendMetric.reportUntrackedMillis(untrackedMillis); + } + + @Override + public Double getValue() { + BigDecimal spend = BigDecimal.valueOf(spendMetric.getValue()); + BigDecimal uptime = BigDecimal.valueOf(uptimeMetric.getValue()); + + if (uptime.signum() == 0) { + switch (spend.signum()) { + case -1: + return Double.NEGATIVE_INFINITY; + case 0: + return 0.0; + case +1: + return Double.POSITIVE_INFINITY; + } + } + + return spend.divide(uptime, LIMITED_PRECISION).doubleValue(); + } +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java index b1f99cf9980..848de0ce9a4 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/CompressionCodec.java @@ -1,7 +1,10 @@ package org.logstash.ackedqueue; +import co.elastic.logstash.api.Metric; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.logstash.ackedqueue.ZstdEnabledCompressionCodec.Goal; +import org.logstash.plugins.NamespacedMetricImpl; public interface CompressionCodec { Logger LOGGER = LogManager.getLogger(CompressionCodec.class); @@ -26,33 +29,41 @@ public byte[] decode(byte[] data) { } }; - static CompressionCodec fromConfigValue(final String configValue) { - return fromConfigValue(configValue, LOGGER); + @FunctionalInterface + interface Factory { + CompressionCodec create(final Metric metric); + default CompressionCodec create() { + return create(NamespacedMetricImpl.getNullMetric()); + } } - static CompressionCodec fromConfigValue(final String configValue, final Logger logger) { - return switch (configValue) { - case "disabled" -> { + static CompressionCodec.Factory fromConfigValue(final String configValue, final Logger logger) { + return switch(configValue) { + case "disabled" -> (metric) -> { logger.warn("compression support has been disabled"); - yield CompressionCodec.NOOP; - } - case "none" -> { + return CompressionCodec.NOOP; + }; + case "none" -> (metric) -> { logger.info("compression support is enabled (read-only)"); - yield ZstdAwareCompressionCodec.getInstance(); - } - case "speed" -> { + return new ZstdAwareCompressionCodec(metric); + }; + case "speed" -> (metric) -> { logger.info("compression support is enabled (goal: speed)"); - yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.SPEED); - } - case "balanced" -> { + return new ZstdEnabledCompressionCodec(Goal.SPEED, metric); + }; + case "balanced" -> (metric) -> { logger.info("compression support is enabled (goal: balanced)"); - yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.BALANCED); - } - case "size" -> { + return new ZstdEnabledCompressionCodec(Goal.BALANCED, metric); + }; + case "size" -> (metric) -> { logger.info("compression support is enabled (goal: size)"); - yield new ZstdEnabledCompressionCodec(ZstdEnabledCompressionCodec.Goal.SIZE); - } + return new ZstdEnabledCompressionCodec(Goal.SIZE, metric); + }; default -> throw new IllegalArgumentException(String.format("Unsupported compression setting `%s`", configValue)); }; } + + static CompressionCodec.Factory fromConfigValue(final String configValue) { + return fromConfigValue(configValue, LOGGER); + } } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java new file mode 100644 index 00000000000..85c8840bdc1 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/IORatioMetric.java @@ -0,0 +1,49 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.UserMetric; +import org.logstash.instrument.metrics.MetricType; + +/** + * A {@code IORatioMetric} is a custom metric that tracks the ratio of input to output. + */ +interface IORatioMetric extends UserMetric, org.logstash.instrument.metrics.Metric { + Double getValue(); + + Value getLifetime(); + + void incrementBy(int bytesIn, int bytesOut); + + @Override + default MetricType getType() { + return MetricType.USER; + } + + // NOTE: at 100GiB/sec, this value type has capacity for ~272 years. + interface Value { + long bytesIn(); + + long bytesOut(); + } + + Provider PROVIDER = new Provider<>(IORatioMetric.class, new IORatioMetric() { + @Override + public Double getValue() { + return Double.NaN; + } + + @Override + public Value getLifetime() { + return null; + } + + @Override + public void incrementBy(int bytesIn, int bytesOut) { + // no-op + } + + @Override + public String getName() { + return "NULL"; + } + }); +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java index ceace485888..42771a1b148 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Queue.java @@ -37,6 +37,7 @@ import java.util.concurrent.locks.Condition; import java.util.concurrent.locks.ReentrantLock; +import co.elastic.logstash.api.Metric; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.logstash.FileLockFactory; @@ -46,6 +47,7 @@ import org.logstash.ackedqueue.io.MmapPageIOV2; import org.logstash.ackedqueue.io.PageIO; import org.logstash.common.FsUtil; +import org.logstash.plugins.NamespacedMetricImpl; /** * Persistent queue implementation. @@ -96,7 +98,15 @@ public final class Queue implements Closeable { private static final Logger logger = LogManager.getLogger(Queue.class); + private final Metric metric; + + public Queue(Settings settings) { + this(settings, null); + } + + public Queue(Settings settings, Metric metric) { + this.metric = Objects.requireNonNullElseGet(metric, NamespacedMetricImpl::getNullMetric); try { final Path queueDir = Paths.get(settings.getDirPath()); // Files.createDirectories raises a FileAlreadyExistsException @@ -113,7 +123,7 @@ public Queue(Settings settings) { this.maxBytes = settings.getQueueMaxBytes(); this.checkpointIO = new FileCheckpointIO(dirPath, settings.getCheckpointRetry()); this.elementClass = settings.getElementClass(); - this.compressionCodec = settings.getCompressionCodec(); + this.compressionCodec = settings.getCompressionCodecFactory().create(metric); this.tailPages = new ArrayList<>(); this.unreadTailPages = new ArrayList<>(); this.closed = new AtomicBoolean(true); // not yet opened diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java index 6d64bd4d9e4..de692ee8561 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/QueueFactoryExt.java @@ -25,6 +25,7 @@ import java.nio.file.Path; import java.nio.file.Paths; +import co.elastic.logstash.api.NamespacedMetric; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jruby.Ruby; @@ -40,6 +41,8 @@ import org.logstash.common.SettingKeyDefinitions; import org.logstash.execution.AbstractWrappedQueueExt; import org.logstash.ext.JrubyWrappedSynchronousQueueExt; +import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.plugins.NamespacedMetricImpl; import static org.logstash.common.SettingKeyDefinitions.*; @@ -78,9 +81,16 @@ public QueueFactoryExt(final Ruby runtime, final RubyClass metaClass) { super(runtime, metaClass); } + @Deprecated @JRubyMethod(meta = true) public static AbstractWrappedQueueExt create(final ThreadContext context, final IRubyObject recv, - final IRubyObject settings) throws IOException { + final IRubyObject settings) throws IOException { + return create(context, settings, null); + } + + public static AbstractWrappedQueueExt create(final ThreadContext context, + final IRubyObject settings, + final AbstractNamespacedMetricExt metric) throws IOException { final String type = getSetting(context, settings, QUEUE_TYPE_CONTEXT_NAME).asJavaString(); final BatchMetricMode batchMetricMode = decodeBatchMetricMode(context, settings); if (PERSISTED_TYPE.equals(type)) { @@ -93,7 +103,9 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final Files.createDirectories(queuePath); } - return JRubyWrappedAckedQueueExt.create(context, queueSettings, batchMetricMode); + final NamespacedMetric namespacedMetric = getMetric(context, metric); + return JRubyWrappedAckedQueueExt.create(context, queueSettings, namespacedMetric, batchMetricMode); + } else if (MEMORY_TYPE.equals(type)) { final int batchSize = getSetting(context, settings, SettingKeyDefinitions.PIPELINE_BATCH_SIZE) .convertToInteger().getIntValue(); @@ -103,11 +115,11 @@ public static AbstractWrappedQueueExt create(final ThreadContext context, final return JrubyWrappedSynchronousQueueExt.create(context, queueSize, batchMetricMode); } else { throw context.runtime.newRaiseException( - RubyUtil.CONFIGURATION_ERROR_CLASS, - String.format( - "Invalid setting `%s` for `queue.type`, supported types are: 'memory' or 'persisted'", - type - ) + RubyUtil.CONFIGURATION_ERROR_CLASS, + String.format( + "Invalid setting `%s` for `queue.type`, supported types are: 'memory' or 'persisted'", + type + ) ); } } @@ -122,6 +134,13 @@ private static BatchMetricMode decodeBatchMetricMode(ThreadContext context, IRub return BatchMetricMode.valueOf(batchMetricModeStr.toUpperCase()); } + private static NamespacedMetric getMetric(final ThreadContext context, final AbstractNamespacedMetricExt metric) { + if ( metric == null ) { + return NamespacedMetricImpl.getNullMetric(); + } + return new NamespacedMetricImpl(context, metric); + } + private static IRubyObject getSetting(final ThreadContext context, final IRubyObject settings, final String name) { return settings.callMethod(context, "get_value", context.runtime.newString(name)); @@ -142,11 +161,11 @@ private static Settings extractQueueSettings(final IRubyObject settings) { .checkpointMaxAcks(getSetting(context, settings, QUEUE_CHECKPOINT_ACKS).toJava(Integer.class)) .checkpointRetry(getSetting(context, settings, QUEUE_CHECKPOINT_RETRY).isTrue()) .queueMaxBytes(getSetting(context, settings, QUEUE_MAX_BYTES).toJava(Integer.class)) - .compressionCodec(extractConfiguredCodec(settings)) + .compressionCodecFactory(extractConfiguredCodec(settings)) .build(); } - private static CompressionCodec extractConfiguredCodec(final IRubyObject settings) { + private static CompressionCodec.Factory extractConfiguredCodec(final IRubyObject settings) { final ThreadContext context = settings.getRuntime().getCurrentContext(); final String compressionSetting = getSetting(context, settings, QUEUE_COMPRESSION).asJavaString(); return CompressionCodec.fromConfigValue(compressionSetting, LOGGER); diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java b/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java new file mode 100644 index 00000000000..615f5612c60 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/RelativeSpendMetric.java @@ -0,0 +1,35 @@ +package org.logstash.ackedqueue; + +import co.elastic.logstash.api.TimerMetric; +import co.elastic.logstash.api.UserMetric; +import org.logstash.instrument.metrics.MetricType; +import org.logstash.instrument.metrics.timer.NullTimerMetric; + +interface RelativeSpendMetric extends UserMetric, org.logstash.instrument.metrics.Metric, TimerMetric { + + default MetricType getType() { + return MetricType.USER; + } + + Provider PROVIDER = new Provider<>(RelativeSpendMetric.class, new RelativeSpendMetric() { + @Override + public T time(ExceptionalSupplier exceptionalSupplier) throws E { + return NullTimerMetric.getInstance().time(exceptionalSupplier); + } + + @Override + public void reportUntrackedMillis(long untrackedMillis) { + // no-op + } + + @Override + public Double getValue() { + return 0.0; + } + + @Override + public String getName() { + return "NULL"; + } + }); +} diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java index 36d4b60e3b2..2e4646281f9 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/Settings.java @@ -44,7 +44,7 @@ public interface Settings { boolean getCheckpointRetry(); - CompressionCodec getCompressionCodec(); + CompressionCodec.Factory getCompressionCodecFactory(); /** * Validate and return the settings, or throw descriptive {@link QueueRuntimeException} @@ -91,7 +91,7 @@ interface Builder { Builder checkpointRetry(boolean checkpointRetry); - Builder compressionCodec(CompressionCodec compressionCodec); + Builder compressionCodecFactory(CompressionCodec.Factory compressionCodecFactory); Settings build(); } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java index 923217af366..c5a95c3b67d 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/SettingsImpl.java @@ -31,7 +31,7 @@ public class SettingsImpl implements Settings { private final int checkpointMaxAcks; private final int checkpointMaxWrites; private final boolean checkpointRetry; - private final CompressionCodec compressionCodec; + private final CompressionCodec.Factory compressionCodec; public static Builder builder(final Settings settings) { return new BuilderImpl(settings); @@ -50,7 +50,7 @@ private SettingsImpl(final BuilderImpl builder) { this.checkpointMaxAcks = builder.checkpointMaxAcks; this.checkpointMaxWrites = builder.checkpointMaxWrites; this.checkpointRetry = builder.checkpointRetry; - this.compressionCodec = builder.compressionCodec; + this.compressionCodec = builder.compressionCodecFactory; } @Override @@ -94,7 +94,7 @@ public boolean getCheckpointRetry() { } @Override - public CompressionCodec getCompressionCodec() { + public CompressionCodec.Factory getCompressionCodecFactory() { return this.compressionCodec; } @@ -147,7 +147,7 @@ private static final class BuilderImpl implements Builder { private boolean checkpointRetry; - private CompressionCodec compressionCodec; + private CompressionCodec.Factory compressionCodecFactory; private BuilderImpl(final String dirForFiles) { this.dirForFiles = dirForFiles; @@ -157,7 +157,7 @@ private BuilderImpl(final String dirForFiles) { this.maxUnread = DEFAULT_MAX_UNREAD; this.checkpointMaxAcks = DEFAULT_CHECKPOINT_MAX_ACKS; this.checkpointMaxWrites = DEFAULT_CHECKPOINT_MAX_WRITES; - this.compressionCodec = CompressionCodec.NOOP; + this.compressionCodecFactory = (metric) -> CompressionCodec.NOOP; this.checkpointRetry = false; } @@ -170,7 +170,7 @@ private BuilderImpl(final Settings settings) { this.checkpointMaxAcks = settings.getCheckpointMaxAcks(); this.checkpointMaxWrites = settings.getCheckpointMaxWrites(); this.checkpointRetry = settings.getCheckpointRetry(); - this.compressionCodec = settings.getCompressionCodec(); + this.compressionCodecFactory = settings.getCompressionCodecFactory(); } @Override @@ -216,8 +216,8 @@ public Builder checkpointRetry(final boolean checkpointRetry) { } @Override - public Builder compressionCodec(CompressionCodec compressionCodec) { - this.compressionCodec = compressionCodec; + public Builder compressionCodecFactory(CompressionCodec.Factory compressionCodec) { + this.compressionCodecFactory = compressionCodec; return this; } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java index f82b4b75f2e..c4797e41448 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdAwareCompressionCodec.java @@ -1,14 +1,15 @@ package org.logstash.ackedqueue; +import co.elastic.logstash.api.Metric; + /** * A {@link ZstdAwareCompressionCodec} is an {@link CompressionCodec} that can decode deflate-compressed * bytes, but performs no compression when encoding. */ class ZstdAwareCompressionCodec extends AbstractZstdAwareCompressionCodec { - private static final ZstdAwareCompressionCodec INSTANCE = new ZstdAwareCompressionCodec(); - static ZstdAwareCompressionCodec getInstance() { - return INSTANCE; + public ZstdAwareCompressionCodec(Metric queueMetric) { + super(queueMetric); } @Override diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java index fa5a22b3ee6..8a7ee5f246b 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java @@ -1,5 +1,7 @@ package org.logstash.ackedqueue; +import co.elastic.logstash.api.Metric; +import co.elastic.logstash.api.NamespacedMetric; import com.github.luben.zstd.Zstd; /** @@ -24,14 +26,25 @@ public enum Goal { private final int internalLevel; - ZstdEnabledCompressionCodec(final Goal internalLevel) { + private final IORatioMetric encodeRatioMetric; + private final RelativeSpendMetric encodeTimerMetric; + + ZstdEnabledCompressionCodec(final Goal internalLevel, final Metric queueMetric) { + super(queueMetric); this.internalLevel = internalLevel.internalLevel; + + final NamespacedMetric encodeNamespace = queueMetric.namespace("compression", "encode"); + encodeRatioMetric = encodeNamespace.namespace("ratio") + .register("lifetime", AtomicIORatioMetric.FACTORY); + encodeTimerMetric = encodeNamespace.namespace("spend") + .register("lifetime", CalculatedRelativeSpendMetric.FACTORY); } @Override public byte[] encode(byte[] data) { try { - final byte[] encoded = Zstd.compress(data, internalLevel); + final byte[] encoded = encodeTimerMetric.time(() -> Zstd.compress(data, internalLevel)); + encodeRatioMetric.incrementBy(data.length, encoded.length); logger.trace("encoded {} -> {}", data.length, encoded.length); return encoded; } catch (Exception e) { diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java index 770824935ce..d98d49a3362 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyAckedQueueExt.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Objects; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyBoolean; import org.jruby.RubyClass; @@ -42,6 +43,8 @@ import org.logstash.ackedqueue.QueueExceptionMessages; import org.logstash.ackedqueue.Settings; import org.logstash.ackedqueue.SettingsImpl; +import org.logstash.plugins.NamespacedMetricImpl; + /** * JRuby extension to wrap a persistent queue instance. @@ -62,9 +65,14 @@ public Queue getQueue() { return this.queue; } + @Deprecated public static JRubyAckedQueueExt create(final Settings settings) { + return create(settings, NamespacedMetricImpl.getNullMetric()); + } + + public static JRubyAckedQueueExt create(final Settings settings, final Metric metric) { JRubyAckedQueueExt queueExt = new JRubyAckedQueueExt(RubyUtil.RUBY, RubyUtil.ACKED_QUEUE_CLASS); - queueExt.queue = new Queue(settings); + queueExt.queue = new Queue(settings, metric); return queueExt; } diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java index 42d11da8875..ca58436bd4d 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ext/JRubyWrappedAckedQueueExt.java @@ -23,6 +23,7 @@ import java.io.IOException; import java.util.Objects; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyBoolean; import org.jruby.RubyClass; @@ -40,6 +41,8 @@ import org.logstash.ext.JrubyAckedReadClientExt; import org.logstash.ext.JrubyAckedWriteClientExt; import org.logstash.ext.JrubyEventExtLibrary; +import org.logstash.instrument.metrics.AbstractMetricExt; +import org.logstash.plugins.NamespacedMetricImpl; /** * JRuby extension @@ -52,8 +55,9 @@ public final class JRubyWrappedAckedQueueExt extends AbstractWrappedQueueExt { private JRubyAckedQueueExt queue; private QueueFactoryExt.BatchMetricMode batchMetricMode; - @JRubyMethod(required=2) - public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject settings, IRubyObject batchMetricMode) throws IOException { + @JRubyMethod(required=2, optional=1) + public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject[] args) throws IOException { + final IRubyObject settings = args[0]; if (!JavaUtil.isJavaObject(settings)) { // We should never get here, but previously had an initialize method // that took 7 technically-optional ordered parameters. @@ -64,6 +68,7 @@ public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject s settings)); } + final IRubyObject batchMetricMode = args[1]; Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); if (!JavaUtil.isJavaObject(batchMetricMode)) { throw new IllegalArgumentException( @@ -73,9 +78,10 @@ public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject s batchMetricMode)); } + final Metric metric = getApiMetric(args.length > 2 ? args[2] : null); Settings javaSettings = JavaUtil.unwrapJavaObject(settings); - this.queue = JRubyAckedQueueExt.create(javaSettings); + this.queue = JRubyAckedQueueExt.create(javaSettings, metric); this.batchMetricMode = JavaUtil.unwrapJavaObject(batchMetricMode); this.queue.open(); @@ -83,14 +89,23 @@ public JRubyWrappedAckedQueueExt initialize(ThreadContext context, IRubyObject s return this; } - public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { - return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings, batchMetricMode); + public static JRubyWrappedAckedQueueExt create(ThreadContext context, Settings settings, Metric metric, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { + return new JRubyWrappedAckedQueueExt(context.runtime, RubyUtil.WRAPPED_ACKED_QUEUE_CLASS, settings, metric, batchMetricMode); } + @Deprecated public JRubyWrappedAckedQueueExt(Ruby runtime, RubyClass metaClass, Settings settings, QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { + this(runtime, metaClass, settings, NamespacedMetricImpl.getNullMetric(), batchMetricMode); + } + + public JRubyWrappedAckedQueueExt(final Ruby runtime, + final RubyClass metaClass, + final Settings settings, + final Metric metric, + final QueueFactoryExt.BatchMetricMode batchMetricMode) throws IOException { super(runtime, metaClass); this.batchMetricMode = Objects.requireNonNull(batchMetricMode, "batchMetricMode setting must be non-null"); - this.queue = JRubyAckedQueueExt.create(settings); + this.queue = JRubyAckedQueueExt.create(settings, metric); this.queue.open(); } @@ -98,6 +113,19 @@ public JRubyWrappedAckedQueueExt(final Ruby runtime, final RubyClass metaClass) super(runtime, metaClass); } + private static Metric getApiMetric(IRubyObject metric) { + if (Objects.isNull(metric) || metric.isNil()) { + return NamespacedMetricImpl.getNullMetric(); + } + if (metric instanceof AbstractMetricExt rubyExtensionMetric) { + return rubyExtensionMetric.asApiMetric(); + } + if (Metric.class.isAssignableFrom(metric.getJavaClass())) { + return metric.toJava(Metric.class); + } + throw new IllegalArgumentException(String.format("Object <%s> could not be converted to a metric", metric.inspect())); + } + @JRubyMethod(name = "queue") public JRubyAckedQueueExt rubyGetQueue() { return queue; diff --git a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java index 5bf98abff19..b90b77de54f 100644 --- a/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java +++ b/logstash-core/src/main/java/org/logstash/execution/AbstractPipelineExt.java @@ -300,8 +300,9 @@ private AbstractPipelineExt initialize(final ThreadContext context, */ @JRubyMethod(name = "open_queue") public final IRubyObject openQueue(final ThreadContext context) { + final AbstractNamespacedMetricExt queueNamespace = metric.namespace(context, pipelineNamespacedPath(QUEUE_KEY)); try { - queue = QueueFactoryExt.create(context, null, settings); + queue = QueueFactoryExt.create(context, settings, queueNamespace); } catch (final Exception ex) { LOGGER.error("Logstash failed to create queue.", ex); throw new IllegalStateException(ex); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java index a0298faf18c..7b1ada37c13 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractMetricExt.java @@ -52,6 +52,8 @@ public final IRubyObject collector(final ThreadContext context) { return getCollector(context); } + public abstract co.elastic.logstash.api.Metric asApiMetric(); + protected abstract AbstractNamespacedMetricExt createNamespaced( ThreadContext context, IRubyObject name ); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java index 65f68633138..4dfc8b48555 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractNamespacedMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyClass; @@ -28,6 +29,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.plugins.NamespacedMetricImpl; @JRubyClass(name = "AbstractNamespacedMetric") public abstract class AbstractNamespacedMetricExt extends AbstractMetricExt { @@ -109,6 +111,11 @@ protected abstract IRubyObject doReportTime(ThreadContext context, protected abstract IRubyObject doDecrement(ThreadContext context, IRubyObject[] args); + @Override + public Metric asApiMetric() { + return new NamespacedMetricImpl(getRuntime().getCurrentContext(), this); + } + protected abstract IRubyObject doRegister(ThreadContext context, IRubyObject key, Block metricSupplier); public abstract AbstractMetricExt getMetric(); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java index 8f7f89f5d6a..6f3553f72af 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/AbstractSimpleMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyClass; import org.jruby.anno.JRubyClass; @@ -27,6 +28,7 @@ import org.jruby.runtime.Block; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.plugins.RootMetricImpl; @JRubyClass(name = "AbstractSimpleMetric") public abstract class AbstractSimpleMetricExt extends AbstractMetricExt { @@ -37,6 +39,11 @@ public abstract class AbstractSimpleMetricExt extends AbstractMetricExt { super(runtime, metaClass); } + @Override + public Metric asApiMetric() { + return new RootMetricImpl(getRuntime().getCurrentContext(), this); + } + @JRubyMethod(required = 2, optional = 1) public IRubyObject increment(final ThreadContext context, final IRubyObject[] args) { return doIncrement(context, args); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java index 112b89ac5e0..21eb5c0186b 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/NullNamespacedMetricExt.java @@ -20,6 +20,7 @@ package org.logstash.instrument.metrics; +import co.elastic.logstash.api.Metric; import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyClass; @@ -31,6 +32,7 @@ import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; import org.logstash.RubyUtil; +import org.logstash.plugins.NamespacedMetricImpl; @JRubyClass(name = "NamespacedNullMetric", parent = "AbstractNamespacedMetric") public final class NullNamespacedMetricExt extends AbstractNamespacedMetricExt { @@ -69,6 +71,11 @@ public NullNamespacedMetricExt initialize(final ThreadContext context, return this; } + @Override + public Metric asApiMetric() { + return NamespacedMetricImpl.getNullMetric(); + } + @Override protected IRubyObject getCollector(final ThreadContext context) { return metric.collector(context); diff --git a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java index d2cb39be9e2..04af5f5f790 100644 --- a/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java +++ b/logstash-core/src/main/java/org/logstash/instrument/metrics/timer/TimerMetricFactory.java @@ -5,6 +5,10 @@ public class TimerMetricFactory { static final TimerMetricFactory INSTANCE = new TimerMetricFactory(); + public static TimerMetricFactory getInstance() { + return INSTANCE; + } + private TimerMetricFactory() { } diff --git a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java index 0a76ba59e3f..cc57421e08e 100644 --- a/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java +++ b/logstash-core/src/main/java/org/logstash/plugins/NamespacedMetricImpl.java @@ -24,13 +24,17 @@ import co.elastic.logstash.api.Metric; import co.elastic.logstash.api.NamespacedMetric; import co.elastic.logstash.api.UserMetric; +import org.jruby.Ruby; import org.jruby.RubyArray; import org.jruby.RubyObject; import org.jruby.RubySymbol; import org.jruby.runtime.ThreadContext; import org.jruby.runtime.builtin.IRubyObject; +import org.logstash.RubyUtil; import org.logstash.Rubyfier; import org.logstash.instrument.metrics.AbstractNamespacedMetricExt; +import org.logstash.instrument.metrics.NullMetricExt; +import org.logstash.instrument.metrics.NullNamespacedMetricExt; import org.logstash.instrument.metrics.timer.TimerMetric; import java.util.ArrayList; @@ -45,6 +49,25 @@ */ public class NamespacedMetricImpl implements NamespacedMetric { + private static final NamespacedMetric NULL_METRIC; + static { + final Ruby rubyRuntime = RubyUtil.RUBY; + final ThreadContext context = rubyRuntime.getCurrentContext(); + final NullMetricExt nullMetricExt = NullMetricExt.create(); + final AbstractNamespacedMetricExt namespacedMetricExt = NullNamespacedMetricExt.create(nullMetricExt, rubyRuntime.newArray()); + + NULL_METRIC = new NamespacedMetricImpl(context, namespacedMetricExt){ + @Override + public NamespacedMetric namespace(String... key) { + return this; + } + }; + } + + public static NamespacedMetric getNullMetric() { + return NULL_METRIC; + } + private final ThreadContext threadContext; private final AbstractNamespacedMetricExt metrics; diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java new file mode 100644 index 00000000000..2d209d2b1c4 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/AtomicIORatioMetricTest.java @@ -0,0 +1,129 @@ +package org.logstash.ackedqueue; + +import org.apache.logging.log4j.Logger; +import org.junit.Test; +import org.mockito.Mockito; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.stringContainsInOrder; +import static org.mockito.Matchers.argThat; +import static org.mockito.Matchers.eq; + +public class AtomicIORatioMetricTest { + @Test + public void test() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + assertThat(ioRatioMetric.getValue()).isNaN(); + + ioRatioMetric.incrementBy(1024, 768); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.75); + + ioRatioMetric.incrementBy(256, 128); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.7); + + ioRatioMetric.incrementBy(512, 128); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.5714); + + ioRatioMetric.incrementBy(256, 0); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.5); + + ioRatioMetric.incrementBy(0, 1024); + assertThat(ioRatioMetric.getValue()).isEqualTo(1.0); + + ioRatioMetric.reset(); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(0L); + assertThat(value.bytesOut()).isEqualTo(0L); + }); + + int iterations = 100000000; + int bytesInPerIteration = 4000000; + int bytesOutPerIteration = 3000000; + for (int i = 0; i < iterations; i++) { + ioRatioMetric.incrementBy(bytesInPerIteration, bytesOutPerIteration); + } + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(Math.multiplyExact((long)bytesInPerIteration, iterations)); + assertThat(value.bytesOut()).isEqualTo(Math.multiplyExact((long)bytesOutPerIteration, iterations)); + }); + assertThat(ioRatioMetric.getValue()).isEqualTo(0.75); + } + + @Test + public void testZeroBytesInPositiveBytesOut() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + ioRatioMetric.incrementBy(0, 768); + assertThat(ioRatioMetric.getValue()).isEqualTo(Double.POSITIVE_INFINITY); + } + + @Test + public void testNegativeBytesIn() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.incrementBy(-1, 768); + + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("cannot decrement")), eq(ioRatioMetric.getName())); + } + + @Test + public void testNegativeBytesOut() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.incrementBy(768, -1); + + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("cannot decrement")), eq(ioRatioMetric.getName())); + } + + @Test + public void testZeroBytesInZeroBytesOut() { + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name"); + + ioRatioMetric.incrementBy(0, 0); + assertThat(ioRatioMetric.getValue()).isNaN(); + } + + @Test + public void testLongBytesInOverflow() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.setTo(Long.MAX_VALUE, 2L); + + assertThat(ioRatioMetric.getValue()).isEqualTo(2.168E-19); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(Long.MAX_VALUE); + assertThat(value.bytesOut()).isEqualTo(2L); + }); + + //overflow reset + ioRatioMetric.incrementBy(1, 10); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(4611686018427387903L + 1L); + assertThat(value.bytesOut()).isEqualTo(1L + 10L); + }); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("long overflow", "precision", "reduced"))); + } + + @Test + public void testLongBytesOutOverflow() { + final Logger mockLogger = Mockito.mock(Logger.class); + final AtomicIORatioMetric ioRatioMetric = new AtomicIORatioMetric("name", mockLogger); + ioRatioMetric.setTo(2L, Long.MAX_VALUE); + + assertThat(ioRatioMetric.getValue()).isEqualTo(4.612E18); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(2L); + assertThat(value.bytesOut()).isEqualTo(Long.MAX_VALUE); + }); + + //overflow reset/truncate + ioRatioMetric.incrementBy(10, 1); + assertThat(ioRatioMetric.getLifetime()).satisfies(value -> { + assertThat(value.bytesIn()).isEqualTo(1L + 10L); + assertThat(value.bytesOut()).isEqualTo(4611686018427387903L + 1L); + }); + Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("long overflow", "precision", "reduced"))); + } + +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java new file mode 100644 index 00000000000..fe5067e7b39 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/CalculatedRelativeSpendMetricTest.java @@ -0,0 +1,47 @@ +package org.logstash.ackedqueue; + +import org.junit.Test; +import org.logstash.instrument.metrics.ManualAdvanceClock; +import org.logstash.instrument.metrics.UptimeMetric; +import org.logstash.instrument.metrics.timer.TestTimerMetricFactory; +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.time.Duration; +import java.time.Instant; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.logstash.instrument.metrics.VisibilityUtil.createUptimeMetric; + +public class CalculatedRelativeSpendMetricTest { + @Test + public void testCalculateRelativeSpendMetric() { + final ManualAdvanceClock clock = new ManualAdvanceClock(Instant.now()); + final TimerMetric burnMetric = new TestTimerMetricFactory(clock::nanoTime).newTimerMetric("burn"); + final UptimeMetric uptimeMetric = createUptimeMetric("wall", clock::nanoTime); + final CalculatedRelativeSpendMetric relativeSpendMetric = new CalculatedRelativeSpendMetric("spend", burnMetric, uptimeMetric); + + clock.advance(Duration.ofMillis(17)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.0); + + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(17))); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.5); + + clock.advance(Duration.ofMillis(34)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.25); + + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(147))); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.7628); + + // nesting for forced concurrency; interleaving validated upstream in TimerMetric + relativeSpendMetric.time(() -> { + clock.advance(Duration.ofMillis(149)); + relativeSpendMetric.time(() -> clock.advance(Duration.ofMillis(842))); + clock.advance(Duration.ofMillis(17)); + }); + assertThat(relativeSpendMetric.getValue()).isEqualTo(1.647); + + // advance wall clock without any new timings + clock.advance(Duration.ofMillis(6833)); + assertThat(relativeSpendMetric.getValue()).isEqualTo(0.25); + } +} \ No newline at end of file diff --git a/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java index d1a154bd3d3..3035507f957 100644 --- a/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java +++ b/logstash-core/src/test/java/org/logstash/ackedqueue/CompressionCodecTest.java @@ -23,15 +23,15 @@ public class CompressionCodecTest { static final ImmutableByteArrayBarrier COMPRESSED_DEFAULT = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 3)); static final ImmutableByteArrayBarrier COMPRESSED_MAXIMUM = new ImmutableByteArrayBarrier(compress(RAW_BYTES.bytes(), 22)); - private final CompressionCodec codecDisabled = CompressionCodec.fromConfigValue("disabled"); - private final CompressionCodec codecNone = CompressionCodec.fromConfigValue("none"); - private final CompressionCodec codecSpeed = CompressionCodec.fromConfigValue("speed"); - private final CompressionCodec codecBalanced = CompressionCodec.fromConfigValue("balanced"); - private final CompressionCodec codecSize = CompressionCodec.fromConfigValue("size"); + private final CompressionCodec codecDisabled = CompressionCodec.fromConfigValue("disabled").create(); + private final CompressionCodec codecNone = CompressionCodec.fromConfigValue("none").create(); + private final CompressionCodec codecSpeed = CompressionCodec.fromConfigValue("speed").create(); + private final CompressionCodec codecBalanced = CompressionCodec.fromConfigValue("balanced").create(); + private final CompressionCodec codecSize = CompressionCodec.fromConfigValue("size").create(); @Test public void testDisabledCompressionCodecDecodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled").create(); assertDecodesRaw(compressionCodec); // ensure true pass-through when compression is disabled, even if the payload looks like ZSTD @@ -42,7 +42,7 @@ public void testDisabledCompressionCodecDecodes() throws Exception { @Test public void testDisabledCompressionCodecEncodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("disabled").create(); // ensure true pass-through when compression is disabled assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); } @@ -50,13 +50,13 @@ public void testDisabledCompressionCodecEncodes() throws Exception { @Test public void testDisabledCompressionCodecLogging() throws Exception { final Logger mockLogger = Mockito.mock(Logger.class); - CompressionCodec.fromConfigValue("disabled", mockLogger); + CompressionCodec.fromConfigValue("disabled", mockLogger).create(); Mockito.verify(mockLogger).warn(argThat(stringContainsInOrder("compression support", "disabled"))); } @Test public void testNoneCompressionCodecDecodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none").create(); assertDecodesRaw(compressionCodec); assertDecodesDeflateAnyLevel(compressionCodec); assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); @@ -64,20 +64,20 @@ public void testNoneCompressionCodecDecodes() throws Exception { @Test public void testNoneCompressionCodecEncodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("none").create(); assertThat(compressionCodec.encode(RAW_BYTES.bytes()), is(equalTo(RAW_BYTES.bytes()))); } @Test public void testNoneCompressionCodecLogging() throws Exception { final Logger mockLogger = Mockito.mock(Logger.class); - CompressionCodec.fromConfigValue("none", mockLogger); + CompressionCodec.fromConfigValue("none", mockLogger).create(); Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "read-only"))); } @Test public void testSpeedCompressionCodecDecodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed").create(); assertDecodesRaw(compressionCodec); assertDecodesDeflateAnyLevel(compressionCodec); assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); @@ -85,20 +85,20 @@ public void testSpeedCompressionCodecDecodes() throws Exception { @Test public void testSpeedCompressionCodecEncodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("speed").create(); assertEncodesSmallerRoundTrip(compressionCodec); } @Test public void testSpeedCompressionCodecLogging() throws Exception { final Logger mockLogger = Mockito.mock(Logger.class); - CompressionCodec.fromConfigValue("speed", mockLogger); + CompressionCodec.fromConfigValue("speed", mockLogger).create(); Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "speed"))); } @Test public void testBalancedCompressionCodecDecodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced").create(); assertDecodesRaw(compressionCodec); assertDecodesDeflateAnyLevel(compressionCodec); assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); @@ -106,20 +106,20 @@ public void testBalancedCompressionCodecDecodes() throws Exception { @Test public void testBalancedCompressionCodecEncodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("balanced").create(); assertEncodesSmallerRoundTrip(compressionCodec); } @Test public void testBalancedCompressionCodecLogging() throws Exception { final Logger mockLogger = Mockito.mock(Logger.class); - CompressionCodec.fromConfigValue("balanced", mockLogger); + CompressionCodec.fromConfigValue("balanced", mockLogger).create(); Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "balanced"))); } @Test public void testSizeCompressionCodecDecodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size").create(); assertDecodesRaw(compressionCodec); assertDecodesDeflateAnyLevel(compressionCodec); assertDecodesOutputOfAllKnownCompressionCodecs(compressionCodec); @@ -127,14 +127,14 @@ public void testSizeCompressionCodecDecodes() throws Exception { @Test public void testSizeCompressionCodecEncodes() throws Exception { - final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size"); + final CompressionCodec compressionCodec = CompressionCodec.fromConfigValue("size").create(); assertEncodesSmallerRoundTrip(compressionCodec); } @Test public void testSizeCompressionCodecLogging() throws Exception { final Logger mockLogger = Mockito.mock(Logger.class); - CompressionCodec.fromConfigValue("size", mockLogger); + CompressionCodec.fromConfigValue("size", mockLogger).create(); Mockito.verify(mockLogger).info(argThat(stringContainsInOrder("compression support", "enabled", "size"))); } diff --git a/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java b/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java new file mode 100644 index 00000000000..f252fae7766 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/instrument/metrics/VisibilityUtil.java @@ -0,0 +1,13 @@ +package org.logstash.instrument.metrics; + +import org.logstash.instrument.metrics.timer.TimerMetric; + +import java.util.function.LongSupplier; + +public class VisibilityUtil { + private VisibilityUtil() {} + + public static UptimeMetric createUptimeMetric(String name, LongSupplier nanoTimeSupplier) { + return new UptimeMetric(name, nanoTimeSupplier); + } +} diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index 4c3e43a8245..d7da245552d 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -215,6 +215,11 @@ logstash_service.start_with_stdin logstash_service.wait_for_logstash + number_of_events.times { + logstash_service.write_to_stdin("Testing flow metrics") + sleep(1) + } + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do # node_stats can fail if the stats subsystem isn't ready result = logstash_service.monitoring_api.node_stats rescue nil @@ -234,6 +239,9 @@ expect(queue_capacity_stats["page_capacity_in_bytes"]).not_to be_nil expect(queue_capacity_stats["max_queue_size_in_bytes"]).not_to be_nil expect(queue_capacity_stats["max_unread_events"]).not_to be_nil + queue_compression_stats = queue_stats.fetch("compression") + expect(queue_compression_stats.dig('decode', 'ratio', 'lifetime')).to be >= 1 + expect(queue_compression_stats.dig('decode', 'spend', 'lifetime')).not_to be_nil else expect(queue_stats["type"]).to eq("memory") end From 2c09f4c858d47e3683c05a2ea6998319f2ea1438 Mon Sep 17 00:00:00 2001 From: Rye Biesemeyer Date: Mon, 29 Sep 2025 19:27:50 -0700 Subject: [PATCH 28/44] metrics: add gauge with compression goal if enabled (#18230) --- docs/static/spec/openapi/logstash-api.yaml | 5 +++++ .../ackedqueue/ZstdEnabledCompressionCodec.java | 3 +++ qa/integration/specs/monitoring_api_spec.rb | 12 +++++++++++- 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/static/spec/openapi/logstash-api.yaml b/docs/static/spec/openapi/logstash-api.yaml index c5ac40b7374..bd6e499b7cc 100644 --- a/docs/static/spec/openapi/logstash-api.yaml +++ b/docs/static/spec/openapi/logstash-api.yaml @@ -2346,6 +2346,11 @@ components: encode: type: object properties: + goal: + - enum: + - speed + - balanced + - size ratio: type: object description: the ratio of event size in bytes to its representation on disk diff --git a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java index 8a7ee5f246b..510bb0cca6f 100644 --- a/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java +++ b/logstash-core/src/main/java/org/logstash/ackedqueue/ZstdEnabledCompressionCodec.java @@ -4,6 +4,8 @@ import co.elastic.logstash.api.NamespacedMetric; import com.github.luben.zstd.Zstd; +import java.util.Locale; + /** * A {@link ZstdEnabledCompressionCodec} is a {@link CompressionCodec} that can decode deflate-compressed * bytes and performs deflate compression when encoding. @@ -34,6 +36,7 @@ public enum Goal { this.internalLevel = internalLevel.internalLevel; final NamespacedMetric encodeNamespace = queueMetric.namespace("compression", "encode"); + encodeNamespace.gauge("goal", internalLevel.name().toLowerCase(Locale.ROOT)); encodeRatioMetric = encodeNamespace.namespace("ratio") .register("lifetime", AtomicIORatioMetric.FACTORY); encodeTimerMetric = encodeNamespace.namespace("spend") diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index d7da245552d..4bcead29634 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -208,7 +208,12 @@ shared_examples "pipeline metrics" do # let(:pipeline_id) { defined?(super()) or fail NotImplementedError } let(:settings_overrides) do - super().merge({'pipeline.id' => pipeline_id}) + super().dup.tap do |overrides| + overrides['pipeline.id'] = pipeline_id + if logstash_service.settings.feature_flag == "persistent_queues" + overrides['queue.compression'] = %w(none speed balanced size).sample + end + end end it "can retrieve queue stats" do @@ -242,6 +247,11 @@ queue_compression_stats = queue_stats.fetch("compression") expect(queue_compression_stats.dig('decode', 'ratio', 'lifetime')).to be >= 1 expect(queue_compression_stats.dig('decode', 'spend', 'lifetime')).not_to be_nil + if settings_overrides['queue.compression'] != 'none' + expect(queue_compression_stats.dig('encode', 'goal')).to eq(settings_overrides['queue.compression']) + expect(queue_compression_stats.dig('encode', 'ratio', 'lifetime')).to be <= 1 + expect(queue_compression_stats.dig('encode', 'spend', 'lifetime')).not_to be_nil + end else expect(queue_stats["type"]).to eq("memory") end From ba7420ccb22318b677113e90249a861306195e2a Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Tue, 30 Sep 2025 14:49:21 +0200 Subject: [PATCH 29/44] Documentation for batch's event metrics (current and average) (#18017) Update the list of logstash settings with the new pipeline.batch.metrics.sampling_mode to control the batch size ones. Co-authored-by: Rob Bavey --- docs/reference/logstash-settings-file.md | 1 + docs/reference/tuning-logstash.md | 1 + docs/static/spec/openapi/logstash-api.yaml | 17 +++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/docs/reference/logstash-settings-file.md b/docs/reference/logstash-settings-file.md index 419e1e3f5ae..197e7e55052 100644 --- a/docs/reference/logstash-settings-file.md +++ b/docs/reference/logstash-settings-file.md @@ -48,6 +48,7 @@ The `logstash.yml` file includes these settings. | `pipeline.workers` | The number of workers that will, in parallel, execute the filter and outputstages of the pipeline. This setting uses the[`java.lang.Runtime.getRuntime.availableProcessors`](https://docs.oracle.com/javase/7/docs/api/java/lang/Runtime.md#availableProcessors())value as a default if not overridden by `pipeline.workers` in `pipelines.yml` or`pipeline.workers` from `logstash.yml`. If you have modified this setting andsee that events are backing up, or that the CPU is not saturated, considerincreasing this number to better utilize machine processing power. | Number of the host’s CPU cores | | `pipeline.batch.size` | The maximum number of events an individual worker thread will collect from inputs before attempting to execute its filters and outputs. Larger batch sizes are generally more efficient, but come at the cost of increased memory overhead. You may need to increase JVM heap space in the `jvm.options` config file. See [Logstash Configuration Files](/reference/config-setting-files.md) for more info. | `125` | | `pipeline.batch.delay` | When creating pipeline event batches, how long in milliseconds to wait for each event before dispatching an undersized batch to pipeline workers. | `50` | +| `pipeline.batch.metrics.sampling_mode` {applies_to}`stack: preview 9.2.0`| Controls frequency of collection of batch size metrics. These metrics measure the actual number of events and byte size of batches processed through a pipeline. This can be helpful to tune `pipeline.batch.size` to reflect the actual batch sizes processed.

Note: This feature is in **technical preview** and may change in the future.

Current options are:

* `disabled`: disabling the collection.
* `minimal`: calculate based on a subset of batches.(default)
* `full`: calculate based on every processed batch.
| `minimal` | | `pipeline.unsafe_shutdown` | When set to `true`, forces Logstash to exit during shutdown even if there are still inflight events in memory. By default, Logstash will refuse to quit until all received events have been pushed to the outputs. Enabling this option can lead to data loss during shutdown. | `false` | | `pipeline.plugin_classloaders` | (Beta) Load Java plugins in independent classloaders to isolate their dependencies. | `false` | | `pipeline.ordered` | Set the pipeline event ordering. Valid options are:

* `auto`. Automatically enables ordering if the `pipeline.workers` setting is `1`, and disables otherwise.
* `true`. Enforces ordering on the pipeline and prevents Logstash from starting if there are multiple workers.
* `false`. Disables the processing required to preserve order. Ordering will not be guaranteed, but you save the processing cost of preserving order.
| `auto` | diff --git a/docs/reference/tuning-logstash.md b/docs/reference/tuning-logstash.md index d7958a026e5..04165505440 100644 --- a/docs/reference/tuning-logstash.md +++ b/docs/reference/tuning-logstash.md @@ -50,6 +50,7 @@ Make sure you’ve read the [Performance troubleshooting](/reference/performance If you plan to modify the default pipeline settings, take into account the following suggestions: * The total number of inflight events is determined by the product of the `pipeline.workers` and `pipeline.batch.size` settings. This product is referred to as the *inflight count*. Keep the value of the inflight count in mind as you adjust the `pipeline.workers` and `pipeline.batch.size` settings. Pipelines that intermittently receive large events at irregular intervals require sufficient memory to handle these spikes. Set the JVM heap space accordingly in the `jvm.options` config file (See [Logstash Configuration Files](/reference/config-setting-files.md) for more info). +* {applies_to}`stack: preview 9.2.0` Consider enabling the metering of batch sizes using the setting `pipeline.batch.metrics.sampling_mode` to help you understand the actual batch sizes being processed by your pipeline. This setting can be useful tuning the `pipeline.batch.size` setting. For more details see [logstash.yml](/reference/logstash-settings-file.md). * Measure each change to make sure it increases, rather than decreases, performance. * Ensure that you leave enough memory available to cope with a sudden increase in event size. For example, an application that generates exceptions that are represented as large blobs of text. * The number of workers may be set higher than the number of CPU cores since outputs often spend idle time in I/O wait conditions. diff --git a/docs/static/spec/openapi/logstash-api.yaml b/docs/static/spec/openapi/logstash-api.yaml index bd6e499b7cc..cbf133cc403 100644 --- a/docs/static/spec/openapi/logstash-api.yaml +++ b/docs/static/spec/openapi/logstash-api.yaml @@ -810,6 +810,7 @@ paths: - stats for each configured filter or output stage - info about config reload successes and failures (when [config reload](https://www.elastic.co/guide/en/logstash/current/reloading-config.html) is enabled) - info about the persistent queue (when [persistent queues](https://www.elastic.co/guide/en/logstash/current/persistent-queues.html) are enabled) + - metrics related to processed batch sizes. Includes the size in bytes and the number of events of batches processed in this pipeline. (when setting [pipeline.batch.metrics.sampling_mode](https://www.elastic.co/docs/reference/logstash/logstash-settings-file.html) is not `disabled`). content: application/json: @@ -821,6 +822,15 @@ paths: example: pipelines: beats-es: + batch: + event_count: + current: 78 + average: + lifetime: 115 + byte_size: + current: 32767 + average: + lifetime: 14820 events: duration_in_millis: 365495 in: 216610 @@ -1095,6 +1105,13 @@ paths: value: pipelines: heartbeat-ruby-stdout: + batch: + event_count: + average: + lifetime: 115 + byte_size: + average: + lifetime: 14820 events: queue_push_duration_in_millis: 159 in: 45 From 9f0df4b58872cb68ad594ab84d5afb1906926100 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Tue, 30 Sep 2025 16:48:18 -0400 Subject: [PATCH 30/44] Fix heading (#18237) --- docs/reference/connecting-to-serverless.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/connecting-to-serverless.md b/docs/reference/connecting-to-serverless.md index 8e381b63b45..aeccb03cccd 100644 --- a/docs/reference/connecting-to-serverless.md +++ b/docs/reference/connecting-to-serverless.md @@ -22,7 +22,7 @@ Set the value to port :443 instead. :::: -## Communication between {{ls}} {{es-serverless}} [connecting-to-elasticsearch-serverless] +## Communication between {{ls}} and {{es-serverless}} [connecting-to-elasticsearch-serverless] [{{es-serverless}}](docs-content://solutions/search/serverless-elasticsearch-get-started.md) simplifies safe, secure communication between {{ls}} and {{es}}. When you configure the Elasticsearch output plugin to use [`cloud_id`](logstash-docs-md://lsr/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-cloud_id) and an [`api_key`](logstash-docs-md://lsr/plugins-outputs-elasticsearch.md#plugins-outputs-elasticsearch-api_key), no additional SSL configuration is needed. From 75b50e99cebc5589147851a00d052aaf4cab12bc Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Wed, 1 Oct 2025 16:40:41 -0700 Subject: [PATCH 31/44] Remove redundant testing and circular dependency from docker acceptance testing (#18181) * Use locally build artifact to build container from public dockerfile Previously we would build an image (which would not actually be used), build dockerfiles, modify dockerfiles to curl from `https://snapshots.elastic.co/downloads/logstash'` then build the image used for testing based on the modified dockerfile. This resulted in testing the last published image to `snapshots`. This presents two problems 1. The test is running against the last published image (not the tip of the branch being tested) 2. this carries a dependency on both a DRA and unified stack release having been run. Therefor acceptance tests will fail in between the time we bump logstash version and a successful run of unified release. This commit modifies the dockerfile to use the artifact prepared in the first step instead of curling the last published one. This solves both issues as the tests run against the code from the tip fo the branch being tested and there is no dependency on an artifact existing as a result of a unified release pipeline. * Remove redundant docker steps from workflows Previously for docker acceptance tests three steps were performed: 1. Build container images (based on local artifacts) 2. Build "public" dockerfiles 3. Build container based on (a modified) file from step 2. The ONLY difference between the dockerfile that ultimately is used to define an image between 1 and 2 is WHERE the logstash source is downloaded from. In acceptance testing we WANT to use the source at the current checkout of logstash (not a remote). Using remote causes a dependency issue when changes are not published. Publishing is tied to unified release and gated on tests so naturally this is a bad fit for that dependency. This commit removes the redundancy by ONLY generating images for testing (step 1 from above). This also firms up our use of LOCAL_ARTIFACTS. Namely, the ONLY time we want that set to `false` is when we build a "public" dockerfile. We explicitly set that in the corresponding DRA script now. Similarly we explicitly set it to `true` when testing. * Remove unused function and argument This commit removes the unused function for building from dockerfiles. It also removes an unused argument for the make task for build_docker. --- .buildkite/scripts/dra/build_docker.sh | 3 ++ ci/docker_acceptance_tests.sh | 5 +- docker/Makefile | 38 ++------------ rakelib/artifacts.rake | 70 +------------------------- 4 files changed, 14 insertions(+), 102 deletions(-) diff --git a/.buildkite/scripts/dra/build_docker.sh b/.buildkite/scripts/dra/build_docker.sh index 0a5ee1998ec..3799b0a8251 100755 --- a/.buildkite/scripts/dra/build_docker.sh +++ b/.buildkite/scripts/dra/build_docker.sh @@ -24,6 +24,9 @@ esac rake artifact:docker || error "artifact:docker build failed." rake artifact:docker_oss || error "artifact:docker_oss build failed." rake artifact:docker_wolfi || error "artifact:docker_wolfi build failed." + +# Generating public dockerfiles is the primary use case for NOT using local artifacts +export LOCAL_ARTIFACTS=false rake artifact:dockerfiles || error "artifact:dockerfiles build failed." STACK_VERSION="$(./$(dirname "$0")/../common/qualified-version.sh)" diff --git a/ci/docker_acceptance_tests.sh b/ci/docker_acceptance_tests.sh index b1e62de2c26..74df2cc1023 100755 --- a/ci/docker_acceptance_tests.sh +++ b/ci/docker_acceptance_tests.sh @@ -8,6 +8,9 @@ set -x export JRUBY_OPTS="-J-Xmx1g" export GRADLE_OPTS="-Xmx4g -Dorg.gradle.console=plain -Dorg.gradle.daemon=false -Dorg.gradle.logging.level=info -Dfile.encoding=UTF-8" +# Use local artifacts for acceptance test Docker builds +export LOCAL_ARTIFACTS=true + if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" fi @@ -48,7 +51,7 @@ if [[ $SELECTED_TEST_SUITE == "oss" ]]; then elif [[ $SELECTED_TEST_SUITE == "full" ]]; then echo "--- Building $SELECTED_TEST_SUITE docker images" cd $LS_HOME - rake artifact:build_docker_full + rake artifact:docker echo "--- Acceptance: Installing dependencies" cd $QA_DIR bundle install diff --git a/docker/Makefile b/docker/Makefile index cdc9915b6bf..c220f57f898 100644 --- a/docker/Makefile +++ b/docker/Makefile @@ -132,19 +132,12 @@ public-dockerfiles_full: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="full" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-full" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-full Dockerfile && \ tar -zcf ../logstash-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_full: public-dockerfiles_full - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_full && cd dockerfile_build_full && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-full:$(VERSION_TAG) . - public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -153,19 +146,12 @@ public-dockerfiles_oss: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="oss" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-oss" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-oss Dockerfile && \ tar -zcf ../logstash-oss-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_oss: public-dockerfiles_oss - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_oss && cd dockerfile_build_oss && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-oss:$(VERSION_TAG) . - public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -174,19 +160,12 @@ public-dockerfiles_wolfi: templates/Dockerfile.erb docker_paths $(COPY_FILES) version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="wolfi" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-wolfi" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-wolfi Dockerfile && \ tar -zcf ../logstash-wolfi-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_wolfi: public-dockerfiles_wolfi - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_wolfi && cd dockerfile_build_wolfi && \ - tar -zxf ../../logstash-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-wolfi:$(VERSION_TAG) . - public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(COPY_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ created_date="${BUILD_DATE}" \ @@ -195,19 +174,12 @@ public-dockerfiles_observability-sre: templates/Dockerfile.erb docker_paths $(CO version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="observability-sre" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/Dockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-observability-sre" && \ cd $(ARTIFACTS_DIR)/docker && \ cp $(ARTIFACTS_DIR)/Dockerfile-observability-sre Dockerfile && \ tar -zcf ../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz Dockerfile bin config env2yaml pipeline -build-from-dockerfiles_observability-sre: public-dockerfiles_observability-sre - cd $(ARTIFACTS_DIR)/docker && \ - mkdir -p dockerfile_build_observability-sre && cd dockerfile_build_observability-sre && \ - tar -zxf ../../logstash-observability-sre-$(VERSION_TAG)-docker-build-context.tar.gz && \ - sed 's/artifacts/snapshots/g' Dockerfile > Dockerfile.tmp && mv Dockerfile.tmp Dockerfile && \ - docker build --progress=plain --network=host -t $(IMAGE_TAG)-dockerfile-observability-sre:$(VERSION_TAG) . - public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/IronbankDockerfile.erb ironbank_docker_paths $(COPY_IRONBANK_FILES) ../vendor/jruby/bin/jruby -S erb -T "-"\ elastic_version="${ELASTIC_VERSION}" \ @@ -219,7 +191,7 @@ public-dockerfiles_ironbank: templates/hardening_manifest.yaml.erb templates/Iro version_tag="${VERSION_TAG}" \ release="${RELEASE}" \ image_flavor="ironbank" \ - local_artifacts="false" \ + local_artifacts="$(or $(LOCAL_ARTIFACTS),false)" \ templates/IronbankDockerfile.erb > "${ARTIFACTS_DIR}/Dockerfile-ironbank" && \ cd $(ARTIFACTS_DIR)/ironbank && \ cp $(ARTIFACTS_DIR)/Dockerfile-ironbank Dockerfile && \ diff --git a/rakelib/artifacts.rake b/rakelib/artifacts.rake index 87a57d262b2..2739e1fc6a7 100644 --- a/rakelib/artifacts.rake +++ b/rakelib/artifacts.rake @@ -169,7 +169,7 @@ namespace "artifact" do desc "Generate rpm, deb, tar and zip artifacts" task "all" => ["prepare", "build"] - task "docker_only" => ["prepare", "build_docker_full", "build_docker_oss", "build_docker_wolfi", "build_docker_observabilitySRE"] + task "docker_only" => ["prepare", "docker", "docker_oss", "docker_wolfi", "docker_observabilitySRE"] desc "Build all (jdk bundled and not) tar.gz and zip of default logstash plugins with all dependencies" task "archives" => ["prepare", "generate_build_metadata"] do @@ -397,52 +397,24 @@ namespace "artifact" do build_dockerfile('oss') end - namespace "dockerfile_oss" do - desc "Build Oss Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_oss"] do - build_docker_from_dockerfiles('oss') - end - end - desc "Generate Dockerfile for observability-sre images" task "dockerfile_observabilitySRE" => ["prepare-observabilitySRE", "generate_build_metadata"] do puts("[dockerfiles] Building observability-sre Dockerfile") build_dockerfile('observability-sre') end - namespace "dockerfile_observabilitySRE" do - desc "Build ObservabilitySrE Docker image from Dockerfile context files" - task "docker" => ["archives_docker_observabilitySRE", "dockerfile_observabilitySRE"] do - build_docker_from_dockerfiles('observability-sre') - end - end - desc "Generate Dockerfile for full images" task "dockerfile_full" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building full Dockerfiles") build_dockerfile('full') end - namespace "dockerfile_full" do - desc "Build Full Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_full"] do - build_docker_from_dockerfiles('full') - end - end - desc "Generate Dockerfile for wolfi images" task "dockerfile_wolfi" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building wolfi Dockerfiles") build_dockerfile('wolfi') end - namespace "dockerfile_wolfi" do - desc "Build Wolfi Docker image from Dockerfile context files" - task "docker" => ["archives_docker", "dockerfile_wolfi"] do - build_docker_from_dockerfiles('wolfi') - end - end - desc "Generate build context for ironbank" task "dockerfile_ironbank" => ["prepare", "generate_build_metadata"] do puts("[dockerfiles] Building ironbank Dockerfiles") @@ -469,30 +441,6 @@ namespace "artifact" do Rake::Task["artifact:archives_oss"].invoke end - task "build_docker_full" => [:generate_build_metadata] do - Rake::Task["artifact:docker"].invoke - Rake::Task["artifact:dockerfile_full"].invoke - Rake::Task["artifact:dockerfile_full:docker"].invoke - end - - task "build_docker_oss" => [:generate_build_metadata] do - Rake::Task["artifact:docker_oss"].invoke - Rake::Task["artifact:dockerfile_oss"].invoke - Rake::Task["artifact:dockerfile_oss:docker"].invoke - end - - task "build_docker_observabilitySRE" => [:generate_build_metadata] do - Rake::Task["artifact:docker_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE"].invoke - Rake::Task["artifact:dockerfile_observabilitySRE:docker"].invoke - end - - task "build_docker_wolfi" => [:generate_build_metadata] do - Rake::Task["artifact:docker_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi"].invoke - Rake::Task["artifact:dockerfile_wolfi:docker"].invoke - end - task "generate_build_metadata" do require 'time' require 'tempfile' @@ -927,27 +875,13 @@ namespace "artifact" do "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), "RELEASE" => ENV["RELEASE"], "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS + "BUILD_DATE" => BUILD_DATE } Dir.chdir("docker") do |dir| safe_system(env, "make build-from-local-#{flavor}-artifacts") end end - def build_docker_from_dockerfiles(flavor) - env = { - "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), - "RELEASE" => ENV["RELEASE"], - "VERSION_QUALIFIER" => VERSION_QUALIFIER, - "BUILD_DATE" => BUILD_DATE, - "LOCAL_ARTIFACTS" => LOCAL_ARTIFACTS - } - Dir.chdir("docker") do |dir| - safe_system(env, "make build-from-dockerfiles_#{flavor}") - end - end - def build_dockerfile(flavor) env = { "ARTIFACTS_DIR" => ::File.join(Dir.pwd, "build"), From 128ac9cbfafc2218d6b905d9c6ce0df98f472d46 Mon Sep 17 00:00:00 2001 From: Andrea Selva Date: Thu, 2 Oct 2025 10:49:20 +0200 Subject: [PATCH 32/44] Rewrite Password setting class in Java (second attempt) (#18231) Secoind attempt to translates Password setting class into plain Java, which exposes a logger used by the Ruby ValidatedPassword subclass. Fixes a bug happened in previous #18183. Co-authored: @donoghuc --- logstash-core/lib/logstash/environment.rb | 4 +- logstash-core/lib/logstash/settings.rb | 22 +---- .../spec/logstash/settings/password_spec.rb | 4 +- logstash-core/spec/logstash/settings_spec.rb | 28 +++---- .../logstash/settings/PasswordSetting.java | 53 ++++++++++++ .../settings/PasswordSettingTest.java | 82 +++++++++++++++++++ 6 files changed, 155 insertions(+), 38 deletions(-) create mode 100644 logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java create mode 100644 logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java diff --git a/logstash-core/lib/logstash/environment.rb b/logstash-core/lib/logstash/environment.rb index a04b3b9b0b4..4eb8de501d7 100644 --- a/logstash-core/lib/logstash/environment.rb +++ b/logstash-core/lib/logstash/environment.rb @@ -75,7 +75,7 @@ def self.as_java_range(r) Setting::StringSetting.new("api.environment", "production"), Setting::StringSetting.new("api.auth.type", "none", true, %w(none basic)), Setting::StringSetting.new("api.auth.basic.username", nil, false).nullable, - Setting::Password.new("api.auth.basic.password", nil, false).nullable, + Setting::PasswordSetting.new("api.auth.basic.password", nil, false).nullable, Setting::StringSetting.new("api.auth.basic.password_policy.mode", "WARN", true, %w[WARN ERROR]), Setting::NumericSetting.new("api.auth.basic.password_policy.length.minimum", 8), Setting::StringSetting.new("api.auth.basic.password_policy.include.upper", "REQUIRED", true, %w[REQUIRED OPTIONAL]), @@ -84,7 +84,7 @@ def self.as_java_range(r) Setting::StringSetting.new("api.auth.basic.password_policy.include.symbol", "OPTIONAL", true, %w[REQUIRED OPTIONAL]), Setting::BooleanSetting.new("api.ssl.enabled", false), Setting::ExistingFilePath.new("api.ssl.keystore.path", nil, false).nullable, - Setting::Password.new("api.ssl.keystore.password", nil, false).nullable, + Setting::PasswordSetting.new("api.ssl.keystore.password", nil, false).nullable, Setting::StringArray.new("api.ssl.supported_protocols", nil, true, %w[TLSv1 TLSv1.1 TLSv1.2 TLSv1.3]), Setting::StringSetting.new("pipeline.batch.metrics.sampling_mode", "minimal", true, ["disabled", "minimal", "full"]), Setting::StringSetting.new("queue.type", "memory", true, ["persisted", "memory"]), diff --git a/logstash-core/lib/logstash/settings.rb b/logstash-core/lib/logstash/settings.rb index 81d208cb151..39fe4fa9406 100644 --- a/logstash-core/lib/logstash/settings.rb +++ b/logstash-core/lib/logstash/settings.rb @@ -439,27 +439,9 @@ def validate(value) java_import org.logstash.settings.NullableStringSetting - class Password < Coercible - def initialize(name, default = nil, strict = true) - super(name, LogStash::Util::Password, default, strict) - end - - def coerce(value) - return value if value.kind_of?(LogStash::Util::Password) - - if value && !value.kind_of?(::String) - raise(ArgumentError, "Setting `#{name}` could not coerce non-string value to password") - end - - LogStash::Util::Password.new(value) - end - - def validate(value) - super(value) - end - end + java_import org.logstash.settings.PasswordSetting - class ValidatedPassword < Setting::Password + class ValidatedPassword < Setting::PasswordSetting def initialize(name, value, password_policies) @password_policies = password_policies super(name, value, true) diff --git a/logstash-core/spec/logstash/settings/password_spec.rb b/logstash-core/spec/logstash/settings/password_spec.rb index ca90bc4f9a8..d20819c44c8 100644 --- a/logstash-core/spec/logstash/settings/password_spec.rb +++ b/logstash-core/spec/logstash/settings/password_spec.rb @@ -18,7 +18,7 @@ require "spec_helper" require "logstash/settings" -describe LogStash::Setting::Password do +describe LogStash::Setting::PasswordSetting do let(:setting_name) { "secure" } subject(:password_setting) { described_class.new(setting_name, nil, true) } @@ -55,7 +55,7 @@ context 'with an invalid non-string value' do let(:setting_value) { 867_5309 } it 'rejects the invalid value' do - expect { password_setting.set(setting_value) }.to raise_error(ArgumentError, "Setting `#{setting_name}` could not coerce non-string value to password") + expect { password_setting.set(setting_value) }.to raise_error(IllegalArgumentException, "Setting `#{setting_name}` could not coerce non-string value to password") expect(password_setting).to_not be_set end end diff --git a/logstash-core/spec/logstash/settings_spec.rb b/logstash-core/spec/logstash/settings_spec.rb index bc1b704c4cb..03e3e4ecb29 100644 --- a/logstash-core/spec/logstash/settings_spec.rb +++ b/logstash-core/spec/logstash/settings_spec.rb @@ -300,22 +300,22 @@ end end - context "containing a mode WARN policy" do - before :each do - # Needs to mock the logger method at LogStash::Settings instead of LogStash::Setting::ValidatedPassword - # else the LOGGABLE_PROXY hide the mock itself. - allow(LogStash::Settings).to receive(:logger).at_least(:once).and_return(mock_logger) - allow(mock_logger).to receive(:warn) - end - let(:mock_logger) { double("logger") } - let(:password_policies) { super().merge({ "mode": "WARN" }) } + describe "mode WARN" do + let(:password_policies) { super().merge("mode": "WARN") } + + context "when the password does not conform to the policy" do + let(:password) { LogStash::Util::Password.new("NoNumbers!") } + let(:mock_logger) { double("logger") } + + before :each do + allow_any_instance_of(LogStash::Setting::ValidatedPassword).to receive(:logger).and_return(mock_logger) + end + + it "logs a warning on validation failure" do + expect(mock_logger).to receive(:warn).with(a_string_including("Password must contain at least one digit between 0 and 9.")) - it "logs a warning on validation failure" do - password = LogStash::Util::Password.new("Password!") - expect { LogStash::Setting::ValidatedPassword.new("test.validated.password", password, password_policies) - }.not_to raise_error - expect(mock_logger).to have_received(:warn).with(a_string_including("Password must contain at least one digit between 0 and 9.")) + end end end end diff --git a/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java b/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java new file mode 100644 index 00000000000..d0329f663b7 --- /dev/null +++ b/logstash-core/src/main/java/org/logstash/settings/PasswordSetting.java @@ -0,0 +1,53 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.logstash.settings; + +import co.elastic.logstash.api.Password; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + + +public class PasswordSetting extends Coercible { + + private static final Logger LOG = LogManager.getLogger(); + + public PasswordSetting(String name, Object defaultValue) { + this(name, defaultValue, true); + } + + public PasswordSetting(String name, Object defaultValue, boolean strict) { + super(name, defaultValue, strict, noValidator()); + } + + @Override + public Password coerce(Object obj) { + if (obj instanceof Password) { + return (Password) obj; + } + if (obj != null && !(obj instanceof String)) { + throw new IllegalArgumentException("Setting `" + getName() + "` could not coerce non-string value to password"); + } + return new Password((String) obj); + } + + public Logger getLogger() { + return LOG; + } +} diff --git a/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java b/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java new file mode 100644 index 00000000000..5f75777f939 --- /dev/null +++ b/logstash-core/src/test/java/org/logstash/settings/PasswordSettingTest.java @@ -0,0 +1,82 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.logstash.settings; + +import org.junit.Before; +import org.junit.Test; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.Assert.*; + +public class PasswordSettingTest { + + private final String SETTING_NAME = "setting_name"; + private PasswordSetting sut; + + @Before + public void setUp() { + sut = new PasswordSetting(SETTING_NAME, null, true); + } + + @Test + public void givenUnsetPasswordSetting_thenIsConsideredAsValid() { + assertNotThrown(() -> sut.validateValue()); + assertThat(sut.value(), is(instanceOf(co.elastic.logstash.api.Password.class))); + assertNull(((co.elastic.logstash.api.Password) sut.value()).getValue()); + } + + @Test + public void givenUnsetPasswordSetting_whenIsSetIsInvoked_thenReturnFalse() { + assertFalse(sut.isSet()); + } + + @Test + public void givenSetPasswordSetting_thenIsValid() { + sut.set("s3cUr3p4$$w0rd"); + + assertNotThrown(() -> sut.validateValue()); + assertThat(sut.value(), is(instanceOf(co.elastic.logstash.api.Password.class))); + assertEquals("s3cUr3p4$$w0rd", ((co.elastic.logstash.api.Password) sut.value()).getValue()); + } + + @Test + public void givenSetPasswordSetting_whenIsSetIsInvoked_thenReturnTrue() { + sut.set("s3cUr3p4$$w0rd"); + + assertTrue(sut.isSet()); + } + + @Test + public void givenSetPasswordSettingWithInvalidNonStringValue_thenRejectsTheInvalidValue() { + Exception e = assertThrows(IllegalArgumentException.class, () -> sut.set(867_5309)); + assertThat(e.getMessage(), is("Setting `" + SETTING_NAME + "` could not coerce non-string value to password")); + } + + private void assertNotThrown(Runnable test) { + try { + test.run(); + } catch (Exception e) { + fail("Exception should not be thrown"); + } + } + +} \ No newline at end of file From 4ed46006fed5394bca32b029fe30f11d48ac1626 Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Thu, 2 Oct 2025 08:10:41 -0700 Subject: [PATCH 33/44] Ensure docs gen inserts at correct place in file (#18250) The regex for finding the latest release (as a fallback when current is not in the file) had a bug. This caused the generated docs to be inserted at the wrong place in the file (the end of the file) instead of the top. This commit fixes the logic such that we find the last release when the current is not found. --- tools/release/generate_release_notes_md.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/release/generate_release_notes_md.rb b/tools/release/generate_release_notes_md.rb index 36baeb3f3ae..cedf0e6ec63 100755 --- a/tools/release/generate_release_notes_md.rb +++ b/tools/release/generate_release_notes_md.rb @@ -42,7 +42,7 @@ coming_tag_index = release_notes.find_index {|line| line.match(/^## #{current_release} \[logstash-#{current_release}-release-notes\]$/) } coming_tag_index += 1 if coming_tag_index -release_notes_entry_index = coming_tag_index || release_notes.find_index {|line| line.match(/\[logstash-\d+-release-notes\]$/) } +release_notes_entry_index = coming_tag_index || release_notes.find_index {|line| line.match(/^## .*\[logstash-.*-release-notes\]$/) } unless coming_tag_index report << "## #{current_release} [logstash-#{current_release}-release-notes]\n\n" From 2940069e07608cc5cd3f01552ef60005f8a499d2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 2 Oct 2025 14:21:46 -0700 Subject: [PATCH 34/44] Bump logstash version 9.3.0 (#18241) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * chore: Update logstash version in versions.yml Made with ❤️️ by updatecli * chore: Update logstash-core version in versions.yml Made with ❤️️ by updatecli --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- versions.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/versions.yml b/versions.yml index bad7755a11d..02422e9021a 100644 --- a/versions.yml +++ b/versions.yml @@ -1,7 +1,7 @@ --- # alpha and beta qualifiers are now added via VERSION_QUALIFIER environment var -logstash: 9.2.0 -logstash-core: 9.2.0 +logstash: 9.3.0 +logstash-core: 9.3.0 logstash-core-plugin-api: 2.1.16 bundled_jdk: From 804dc3630f63f7beb1a8418dad58df98ac2eaa4a Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Mon, 6 Oct 2025 10:25:05 -0700 Subject: [PATCH 35/44] Downgrade gradle to coninute testing on windows server 2016 (#18263) * Downgrade gradle to coninute testing on windows server 2016 * REVERT ME! Temporarily force testing on win 2016 * Revert "REVERT ME! Temporarily force testing on win 2016" This reverts commit 079898ba95a37765f2f3e767b1b0e7c4f32d92b0. * update gradle wrapper --- gradle/wrapper/gradle-wrapper.jar | Bin 43764 -> 43583 bytes gradle/wrapper/gradle-wrapper.properties | 2 +- gradlew | 9 +++++---- gradlew.bat | 4 ++-- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1b33c55baabb587c669f562ae36f953de2481846..a4b76b9530d66f5e68d973ea569d8e19de379189 100644 GIT binary patch delta 34959 zcmXVXV_cp8|NoX2PS(k*W!qS`-LmbLJ4+|?WLwM2u9I7~ZDVoOf1mH~f9JY&-MAjt z`-P|8ck?jab1=1bXiB$=zsD5hnV6?h<(cRweoy{VW1ZvJ+Q0eDG%P!=IL;u;_!0R8 zY@V`Lq(|3+PgSy4L?41rg@;pwckO!Z`tgH`{3k?*I$@!&A3l5#`2e{VAckO|OMx01 zs~QdASV-N}R?pQ=O{yqlrqz|1yp(^RK)Bhkwq`;Yh)md4RrtR%sNbw?F7+wVN@9oT5^KvyxHCChVwDz29-_(~6`YI}kOI zb^sOR2x~T#ZdIJ>Rf@`fWMMck8Z~Fk7!ymA-q=^Hp5eZ$X)}%69EWv#a)HMQBo+#f z36F86&q=PH!h1hfL>Ol{cXt`zy7GFq%Eq79O{IA-u!cH*(wj1wN}D2M4WT6o(qxrW zEB}r}@-+r4&wIr;xO0(AI@=cYWb?m21~K;0A^-T{gEQnxfCN&@N(#Zq#RXZY87O0m z;t0Wp7M~;I&<5qU1T+?pjfUye_TixR_f>$?rT1}+*6u;9Gn0cXM{`4grB6(W zyBDpHwv$&%UIzt(jZMh^e3jZ{I@kE301olpI{yj0+;ZWogmFjno1+v zMW;sMFf7sR(_fhVjl~QhEC!kN?S1GnQ8&fuPw9z{5eDbyAAsT&CyjpUf=RK)X*YhW zwf>HLeXJxlm0mFjo>lB@ni;CUkg)*JRligsG*5>@wN*UJvbS&X^}x zn@^UJmJ90QY)d4OLkji-vg;l*>VWz+eRS?0G0Bg!HhZc?2Wz}S3kMg^_@+65nA?uo zkBwh=aDQVGH8XVK>zh0u{gJbev&iTnS1h3p(pF$?`aC^rhJj2lK`5&HHV#_?kJb zGMSi_SJ(*5xg|k>>Dvgt0#5hN#b8)>x5&pj4Wy_c7=p-XQ=>p*vRykohWoq+vj1uk znu?X~2=n2?uaB_*+Lr;+&434q#3lhbD9@_k1Te#nwy}MM^TTHt=B7p23Hvw*C##@< z$6AnfJ+Ri~X^`J(;3$v;d?J5C5U~zQwBA9#k|t1Y#>7ZrY#I@2J`|kfQ=Sxhc*rH| z{varkusu6HJ$Ca6x^v$ZA6sX;#AVi73(ebp61*3)LCF6yToc0LMMm{D%k+S_eJ<3CTZgjVEpgE=i5mX z0o|kFlPT7$0gM?NfN_Wk=T=zCXFhtz_fJrXuKFQ#uaUzUCWj%}$pz$g05t#ar{-1o z#ZYh6o&A&s>>NA5>#m&gf?X>M)bj>Q7YY}AR8nPC<0CJ`QolY!M*@PhNF4%4$5nFf z4{VxA-;8{~$A&>%Yo@~y4|O}IqYemSgP7Sy?d}}+e`ng%{?_hDUhCm`I`hP=rda|n zVWx~(i&}Q|fj^k+l$Y30zv6ME&AX7HTjy~frLaX)QgCMmQq3_qKEcRyY7nk_fa}Z$ ztrwMjNeJ|A@3=y7o^6LMBj@LkTyHm7pK(Vxq%M=uXr;M7{wWsrG~I1ki5OQ6#92Ih%Quj|8Z|qUzyy6 zUf%s*-I*73e%AX}cTI5r+ZsgVR1jr6I*hnu%*rSWqzs(T0KD7A4U}76 z)lH{eBF=pRy0q*o<*iM4@ojv65`y{#TKm=!5+7PwC>z)to^he4BI9`z60IYcFC8XC zZ<65C;OV<=0*{u4*i@nn?J4m6_p_jauY-;RSof^%yxer|uPQvyzOCP1x_-}6H;)~6 zkQH$^6A(lu&B^q)5vwSypjGu5P`Y#UdzM%Uhuh>vlisoS7c?a}|1hah-vo_i`e5;! z93hb``au;ow+t;(wB3-=ww(pgb`ZrEODvFvfEiQvXaSX6+A0ooWdEx3u-oBf9V((3iwRO z7r|AqsNjl$(oTUVvOf^E%G%WX=xJnm>@^c!%RBGy7j<>%w26$G5`?s89=$6leu-z; zm&YocPl2@2EDw6AVuSU&r>cR{&34@7`cLYzqnX)TU_5wibwZ+NC5dMyxz3f!>0(Y zJDdZUg*VS5udu>$bd~P>Zq^r)bO{ndzlaMiO5{7vEWb3Jf#FOpb7ZDmmnP?5x?`TX z@_zlHn)+{T;BtNeJ1Kdp2+u!?dDx4`{9omcB_-%HYs2n5W-t74WV76()dbBN+P)HN zEpCJy82#5rQM+vTjIbX*7<~F)AB_%L*_LL*fW-7b@ATWT1AoUpajnr9aJ19 zmY}jSdf+bZ;V~9%$rJ-wJ3!DTQ3``rU@M~E-kH$kdWfBiS8QL&(56OM&g*O73qNi( zRjq8{%`~n?-iv!fKL>JDO7S4!aujA}t+u6;A0sxCv_hy~Y2Pbe53I*A1qHMYgSCj0z6O zJ!z}o>nI#-@4ZvRP|M!GqkTNYb7Y)$DPWBF3NCjNU-395FoDOuM6T+OSEwNQn3C`D z-I}Tw$^1)2!XX+o@sZp^B4*!UJ=|lZi63u~M4Q%rQE`2}*SW$b)?||O1ay`#&Xjc! z0RB3AaS%X&szV$SLIsGT@24^$5Z8p%ECKsnE92`h{xp^i(i3o%;W{mjAQmWf(6O8A zf7uXY$J^4o{w}0hV)1am8s1awoz0g%hOx4-7 zx8o@8k%dNJ(lA#*fC+}@0ENA#RLfdZB|fY9dXBb;(hk%{m~8J)QQ7CO5zQ4|)Jo4g z67cMld~VvYe6F!2OjfYz?+gy}S~<7gU@;?FfiET@6~z&q*ec+5vd;KI!tU4``&reW zL3}KkDT;2%n{ph5*uxMj0bNmy2YRohzP+3!P=Z6JA*Crjvb+#p4RTQ=sJAbk@>dP^ zV+h!#Ct4IB`es)P;U!P5lzZCHBH#Q(kD*pgWrlx&qj1p`4KY(+c*Kf7$j5nW^lOB#@PafVap`&1;j9^+4;EDO%G9G4gK zBzrL7D#M1;*$YefD2I-+LH{qgzvY8#|K=-X`LN578mTYqDhU}$>9W&VOs z*wW$@o?Vfqr4R0v4Yo_zlb?HKOFS zU@WY7^A8Y{P)qU9gAz52zB8JHL`Ef!)aK7P)8dct2GxC*y2eQV4gSRoLzW*ovb>hR zb0w+7w?v6Q5x1@S@t%$TP0Wiu2czDS*s8^HFl3HOkm{zwCL7#4wWP6AyUGp_WB8t8 zon>`pPm(j}2I7<SUzI=fltEbSR`iSoE1*F3pH4`ax^yEo<-pi;Os;iXcNrWfCGP^Jmp935cN;!T8bve@Qljm z>3ySDAULgN1!F~X7`sAjokd_;kBL99gBC2yjO+ zEqO##8mjsq`|9xpkae&q&F=J#A}#1%b%i3jK-lptc_O$uVki1KJ?Y=ulf*D$sa)HC z=vNki?1aP~%#31<#s+6US0>wX5}nI zhec(KhqxFhhq%8hS?5p|OZ02EJsNPTf!r5KKQB>C#3||j4cr3JZ%iiKUXDCHr!!{g z=xPxc@U28V8&DpX-UCYz*k~2e)q?lRg<{o%1r;+U)q^{v&abJ9&nc6a32ft(Yk}`j ztiQP@yEKf@Nu3F;yo9O})Roh9P08j7@%ftn7U1y;`mard4+5 zB62wpg$Py_YvQ!PE2HpuC}3el-F3g{*&a z3q{eLy6Xz|F+aMrn8R8IW2NZu{tgsyc(>*TdV79@?V$jG(O+Iz2rnDBc|1cK8gR$Y zthvVTI;(eYhOdjapHe=9KI`|2i;{VIfvnR6`qof=4a=(BTZkev78+6GJW**Z!|yvS zes)T%U573C~Hm`&XJzE=2t7tFIZM`!^r^&z;W?dOj-N+a10^>wV(l~2naa?s; zTxU{z;Go|Ve!vUjUrZ$B#mWH)NSdxi;dWa-@w)-$wBOpo`DEG<;C#W||W}&@z>C`*j9V|`ai)z*2PG`TZt6T{a zj!#m3`Vz5R9wJkNMsJ1`fSCS2mHnizWDT!G0Ukp$%*_^X1=k=%mmO$^_0_d|kc8ek4_DZwomL(>GGtfEB)Wy&cfZ@9-T|hAq&fx;XR$$_yl6iogcR{u zm9g)axS6=_IL4=wQXf|EkzO68$Ms4*JXAt8gFxLCibt^C#C|I|v|U{%A;+NaBX-Yn z`HAmP*x5Ux@@Wkpxest$F~K8v0wlb9$3gHoPU(RMt+!BfjH?`8>KMK|!{28+fAk%6 zWdfyaD;Dr~`aJHn0}HIf^Y9*keGvm6!t?o%;je)wm`Dm$fN?YtdPI7S=Y23+15L{J zr;n3MYg`<50nW^`BM$&M(+PQ7@p7Lvn(kE`cmoNS7UkQmfvXQBs_unhdfM){k`Ho! zHL0#a6}Uzs=(bu;jnBAu>}%LzU3+{sDa6~)q_|pW1~*Is5J(~!lWvX(NpK_$=3Rbn zej|)%uR0imC;D5qF7p}kdg(-e{8#o!D_}?Fa<&{!5#8^b(dQl40ES%O_S(k8Z$?Hs z;~ee=^2*5S#A*gzEJgBkXyn*|;BBH97OOmvaZ>&U&RfU0P(?jgLPyFzybR2)7wG`d zkkwi) zJ^sn7D-;I;%VS+>JLjS6a2bmmL^z^IZTokqBEWpG=9{ zZ@<^lIYqt3hPZgAFLVv6uGt}XhW&^JN!ZUQ|IO5fq;G|b|H@nr{(q!`hDI8ss7%C$ zL2}q02v(8fb2+LAD>BvnEL8L(UXN0um^QCuG@s}4!hCn@Pqn>MNXS;$oza~}dDz>J zx3WkVLJ22a;m4TGOz)iZO;Era%n#Tl)2s7~3%B<{6mR!X`g^oa>z#8i)szD%MBe?uxDud2It3SKV>?7XSimsnk#5p|TaeZ7of*wH>E{djABdP7#qXq- z7iLK+F>>2{EYrg>)K^JAP;>L@gIShuGpaElqp)%cGY2UGfX1E;7jaP6|2dI@cYG%4 zr`K1dRDGg3CuY~h+s&b2*C>xNR_n>ftWSwQDO(V&fXn=Iz`58^tosmz)h73w%~rVOFitWa9sSsrnbp|iY8z20EdnnHIxEX6||k-KWaxqmyo?2Yd?Cu$q4)Qn8~hf0=Lw#TAuOs(*CwL085Qn9qZxg=)ntN*hVHrYCF3cuI2CJk7zS2a%yTNifAL{2M>vhQxo?2 zfu8%hd1$q{Sf0+SPq8pOTIzC&9%Ju9Rc1U9&yjGazlHEDaxY|nnS7rATYCW_NA&U? zN!7-zF#DXu0}k4pjN05yu#>x8o#Jx7|Fk=%OR((ti%UVKWQNH>+JhH#ziW1hD=rk* zD#1j?WuGxd-8VqG@n_Lqj^i=VBOg@GLePo0oHX9P*e7qBzIs1lzyp;}L3tP1 zl5;OiHG&-flQ;rYznH%~hz>fuJ!n*H#O)3NM3`3Z9H|VFfS-_xHRCuLjoIS9wT!F0 zJ-kV3w>7EguDzoBPxW>Rra0#+Y?;Woi7qJ1kpxTad?O?^=1cG@GeNtRZRi8_l-1CS z`(#oF<;VYR(l(gHIYH$y2=rj5m3QL{HQgbW9O!TU*jGj!bFazIL?MYnJEvELf}=I5 zTA6EhkHVTa0U#laMQ6!wT;4Tm4_gN$lp?l~w37UJeMInp}P>2%3b^Pv_E1wcwh zI$`G-I~h!*k^k!)POFjjRQMq+MiE@Woq$h3Dt8A%*8xj1q#x?x%D+o3`s*)JOj2oD7-R4Z*QKknE3S9x z8yA8NsVl&>T`a;qPP9b7l{gF&2x9t5iVUdV-yOC12zJnqe5#5wx0so2I)@8xb$uPG zNmv=X)TjpHG(H!$6Xp>)*S}r538R99Y{Pofv}pAFlUK;xi{E43^->z1srWR=J$8N! z4jRu;EAiLG9R$5#{gR){5?o^W^!t140^f=vCVSs@vK7#`-fv`P*WV|>nX610pK08< z>r#{r)fR?2pNG}8o)?uvX#UJI)YM5CG@0E8s1lEV`rom|kBmf={%h!o|26a=lNJbX z6gkBS7e{-p$-Vubn$(l_IbwS02j;+6h2Q5F7P?Du2N!r;Ql$M>S7Frf*r3M`!bvWU zbTgl2p}E<*fv?`N8=B71Dk03J=K@EEQ^|GY*NoHaB~(}_ zx`Su{onY@5(Owc#f`!=H`+_#I<0#PTT9kxp4Ig;Y4*Zi>!ehJ3AiGpwSGd<{Q7Ddh z8jZ(NQ*Nsz5Mu_F_~rtIK$YnxRsOcP-XzNZ)r|)zZYfkLFE8jK)LV-oH{?#)EM%gW zV^O7T z0Kmc1`!7m_~ zJl!{Cb80G#fuJa1K3>!bT@5&ww_VSVYIh_R#~;If$43z`T4-@R=a1Px7r@*tdBOTw zj-VzI{klG5NP!tNEo#~KLk(n`6CMgiinc1-i79z$SlM+eaorY!WDll+m6%i+5_6Mc zf#5j#MYBbY)Z#rd21gtgo3y@c(zQVYaIYKI%y2oVzbPWm;IE#Cw$8O$fV}v}S%QDA zkwxW{fa#Goh1O|+=CF3h3DWNw+L^ly?BNQ7DY~Eca}5nt^>p#3cc9s3iDub0nh`Wy z?oH|dW8-HG@d5E@U>NWPjnhTjr7C${Iwj#;F2G@++N=Y2tjV;z57RNgE|kXQC)1h- zx8ODU>kk};J8KiSUx5jSsA_XPou1OH8=R~q9{`r>VnHkU6A=!zNOH8IGJoO!+bQys zDS2-H(7+Jfe+&zf#;OSV=83I|^M;0`Kv*#4%%O7x>@BgGMU*@ajUvY>cYw^`*jm@+ z{LZ2lr{OTMoQXn2XUsK-l72oysi9vgV4Sux^1GsW6zTV;?p#J06EvSVyUq5$f4kq< z{Chq5Z?I%ZW}6&uL+f&0uCW#^LyL!Ac2*QRII5TDGfZ43YpXyS^9%6HBqqog$Sal3 zJjI$J+@}ja9Xp)Bnbk+pi=*ZAHN}8q@g$$g<6_4?ej&Rw)I%w(%jgGlS5dTHN`9(^<}Hg zD$PbZX+X>;$v4NjGJxMDvVBiIam$cP-;h0YqQ{YgxYn-g&!}lHgaG3^B=>Z!D*7tp zu19e;r`u*+@4h41Da&NZv$qy-i6#DdI)EVvmKO*PvIKz-9E5R*k#|`$zJza8QJ)Q{ zf~Vl+I=8oaq)K!lL7Et5ycH;m&LKIvC|z4FH5bo|>#Kg5z+Jy*8Ifai}5A#%@)TgPRaC4f>Qk&} z4WciN&V(T~u^xBgH=iP(#nd;_@L&`7FUF>Qm-;hOljv(!74f&if;fz2Mg=b%^8$^C zna!2I&iCz&9I5ckX-5mVoAwz~)_&b#&k$e+pp=U2q-OjkS@yZ8ly1$2Vh?}yF0={P zPd3O@g{0L=eT-Dm9?imeUP(!As&DJ_D=5lwQ=3)XWXg)12CoB=-g-HX9RSXgL;yo0 z?$7z8Sy9w?DvA^u`Fnl7r_J&_jJ7claq*2l9E~#iJIWAPXuAHfmF3-4YjFYhOXkNJ zVz8BS_4KCUe68n{cPOTTuD<#H&?*|ayPR2-eJ2U0j$#P!>fhd(LXM>b_0^Gm27$;s ze#JTrkdpb*ws{iJ1jprw#ta&Lz6OjSJhJgmwIaVo!K}znCdX>y!=@@V_=VLZlF&@t z!{_emFt$Xar#gSZi_S5Sn#7tBp`eSwPf73&Dsh52J3bXLqWA`QLoVjU35Q3S4%|Zl zR2x4wGu^K--%q2y=+yDfT*Ktnh#24Sm86n`1p@vJRT|!$B3zs6OWxGN9<}T-XX>1; zxAt4#T(-D3XwskNhJZ6Gvd?3raBu$`W+c(+$2E{_E_;yghgs~U1&XO6$%47BLJF4O zXKZLVTr6kc$Ee0WUBU0cw+uAe!djN=dvD*scic%t)0Jp*1& zhjKqEK+U~w93c<~m_Oh;HX{|zgz=>@(45=Ynh{k#3xlfg!k z>hsq90wPe(!NljYbnuL6s`Z!wQSL8|(A*@M8K>`nPJ<9Hb^ zB6o?#^9zP>3hp0>JAite*3N?Rm>nJ1Lpq4)eqSe8KM_f(0DB?k8DNN6(3 zU#>-{0}3~vYJ7iIwC?Zbh@aJ8kfIvY%RveZltThMN73#Ew}jOwVw+|vU5u-wMoo9C zO(tv#&5`DOhlzunPV?M~qlM|K74x4cBC_AC?2GNw_-Uv&QtPOj(7L4NtVh$`J%xci zioGVvj5s|GY886)(}g`4WS3_%%PrF(O|s-n&-SdfbssL`!Gi7Hrz_r$IO@*$1fYbQ zgdp6?(IUaNPaH7}0%U|9X8HFonsJRrVwfmf*o1;k0+PwV^i%f7U{LAayu`!x*FmhN za(#a^@Idw9)jN)K!=sFC(G)ZNaYY169*IJ_ouY9>W8tC>S&MEp$+7 zy)NFumpuE>=7T@`j}8pa)MGpJaZoG(Ex3AzzH>gUU^eyWp*N2Fx+9*4k~BU;lQ1PG zj4)_JlelzJ==t*7=n2(}B4^^bqqcKFcJ7yVzbH_CWK?{eXdpKm);4|o{aM=M&`E$=_~PVi2>>L zKTN_x&qA)@ak=v=0Hl5H6~?LOfO@1+fu5(sB|VWID)w?%{m+n#7bLaszEJ#;$HMdt z9qP0gk)hIYvE1!jseA^FGTyK=i4eTPjTL$R;6FywMBZBPlh2ar9!8wlj1sinLF-1g zR5}hLq>pb1|AC-WcF!38e*kFv|9n<$etuB=xE%B=PUs}iVFl>m;BiWUqRIxYh7}L&2w@{SS-t(zUp`wLWAyO=PEE=Ekvn@YS*K@($=i zBkTMaH<&cAk${idNy0KZ8xh}u;eAl*tstdM8DYnM5N;bDa`AB+(8>DqX+mj17R2xBp45UES|H*#GHb_%Nc{xWs7l{0pqmiBIPe@r=X%Y-h<-Ceo;4I>isrw1Hd zZd*VjT`H9gxbf{b3krEKNAaV$k>SzK(gzv}>;byq##WEhzTN^@B4+VJvW>y|U}}AQ z4^Bdz9%QKBWCy+h$I?L@ffl{fLLL41Tx|M+NjjRf(`KjHG4^y=x3l z!!-{*v7_^6MiJOC@C$WV=hz9J^Y^lK9#tzs6}-

Gn4F+B~IivciU9^t0j-Mgao3 zSDF_?f~c=V=QJRSDTG0SibzjML$_?2eqZ;J*7Sv$*0SQ|ck$fX&LMyXFj}UH(!X;; zB_rKmM-taavzEk&gLSiCiBQajx$z%gBZY2MWvC{Hu6xguR`}SPCYt=dRq%rvBj{Fm zC((mn$ribN^qcyB1%X3(k|%E_DUER~AaFfd`ka)HnDr+6$D@YQOxx6KM*(1%3K(cN)g#u>Nj zSe+9sTUSkMGjfMgDtJR@vD1d)`pbSW-0<1e-=u}RsMD+k{l0hwcY_*KZ6iTiEY zvhB)Rb+_>O`_G{!9hoB`cHmH^`y16;w=svR7eT_-3lxcF;^GA1TX?&*pZ^>PO=rAR zf>Bg{MSwttyH_=OVpF`QmjK>AoqcfNU(>W7vLGI)=JN~Wip|HV<;xk6!nw-e%NfZ| zzTG*4uw&~&^A}>E>0cIw_Jv-|Eb%GzDo(dt3%-#DqGwPwTVxB|6EnQ;jGl@ua``AFlDZP;dPLtPI}=%iz-tv8 z0Wsw+|0e=GQ7YrS|6^cT|7SaRiKzV3V^_ao_ zLY3Jnp<0O6yE&KIx6-5V@Xf^n02@G2n5}2Z;SiD4L{RAFnq$Q#yt1)MDoHmEC6mX1 zS^rhw8mZJk9tiETa5*ryrCn&Ev?`7mQWz*vQE!SAF{D@b7IGpKrj^_PC2Cpj!8E{W zvFzy&O4Z-Exr$Z*YH4e|imE`&n<$L-_Bju=Axiik+hBtA4XNDik(G_;6^mQ3bT)Y% z6x=a+LKFZbjyb;`MRk~Dbxyc&L; z8*}!9&j0wewMM#O`c#7HJ|+Gh5%3~W10b6sdmCg3G_v+@H>n*c5H`f+7%{TeSrzt89GYJqm>j-!*dReeu&KHubhzjSy_c~BJcbaFtZWAB}~KP3%*u{zHi zVSUi2H8EsuSb3l7_T1hP!$xTtb{3|ZZNAJ{&Ko;#>^^43b7`eE;`87q81Jp;dZfC< z$BD`h-*j=%uTpG8Me6dF zrH%)Bw-a0}S41ILo*k2zn6P@?USXtC>pX*tzce7A^JD7^^p7K5kh-HO&2haDTL%2^ zSWQb2B6}e*;x?eKq?CdG7F=wHVY)Lb(kQu1R#1Fx|3?>_%cjNM-xJlAg9kr`!>&;E zTYmHhqHh&qbfO`~w3V;BM(q(_Q-5^!esaBI&QbZ^%N-ZDYft#FTS;%{ zKzlSwZIS%zDi#%DMK>`_vmE^krJL5@PmpT2m26Q`O)VRAL>){MN45|7GTk=q^zLpF zjS(Os=`#On$XI#$A5ewac9Ma}mDxSu^5{#jHC+24a2GbfBJ&Zn8W= zm=l7VE0g^z$3ikyU#ysh8b-PH(&-yZL$JV-of-ZM@~N^#DbQ3Ltlq*5@>WzSNxrRK zYl2VS8r;TT`wLfD_O0dhX9vR#S8rMOuUCRkWZE#OjRi$l*#C7}mgGzZBD%Z=p3z|CaVM$$pyW5-pJJDCToY zO3R5)P(Gnd>6wh9Z$Sr@cMXmClU(h-@5kmiBTNTU-|5vq&Fs!ah|o47kW?SO8uWv> zW$=Ud@@|*9p@Rb=!wl;%>k)kH7fPtcD=gd}^IxN^=Cg>zq^jij!f=1PlT|9jh3K9g zF~Z)B;kb^a0hLmJvON8Ho)foq-oC)&E)b|a^|b}6n!8&AIaousO^VnYzYfuijuEo5 z7IcUMbYD=vec4eZX7;p31NB+T9BOMJp9ZI9$dH1kJsJpEtf@}tL4)_*PxgdOge9_EaR!?wWtBx%*f$IGoR>f3Qf2aT0%+fq=1xVEqRl;UaA2Ncs4B1M1#foI2bj4 znX}t7;-FCLK&;>ZGP}{GxK67$Kz&pO%%J>DBMP_zZsLOmdpDUDp&f8=L>(Kcj+S^jA5dco4-7XN z)h;m#54CEy9)Ch-E7gHP@a@TXl=_%&|iUlIrQzn=LqONBu9FCn`3f8aqvRu=RrJ_RH1^Uf=t z%Ir*({+wEeC??C+u!hCi<5m`RsRO6ti7YaEtY0|U)-QfNsdN{=83K_}m$0Z=ElWyt znvo5=%f<;|hNnL-r#v5ab&S2*yK>~a7m(My$cfd*tff?=?7-j3^|&9H7G*W`)m8M7 zzd0+b)c@`bQN1-^dC$_04tK0{mU5tx_zo;&TWou8F(H_J?O+Y)VLXzmU^> zvL!5+1H?opj`?lAktaOu%N#k4;X;UX5LuO`4UCVO$t+kZBYu`1&6IV@J>0}x1ecuH zlD9U=_lk1TIRMm6DeY2;BJJEE%b0z;UdvH_a3%o)Z^wM&<$zhQpv90@0c+t?W`9kolKUklpX5M&Qw06u=>GPCr5Imvh*% zfI`tI-eneDRQo?m*zD1i;!B>*z4Xioa_-S=cbv-k_#Wg=)b$0@{SK>Mr!_T?H`S-?j;3$4)ITn$`g;J$^TppD)^pRz#^l?XgZ2CW z3g5G^iF*GZYQ}{B|H-fqh=_>)E~=3y3Zg=i75G5E)*a>R9bn~cNW{h5&P(vQ6!WHv zw1-89smtY~JnCQS(=9zM)6>UAi%G-r^LA9_HF0Vp3%JF2P%+E&^afy61yxnAyU;Z{ z$~H5X6?sMoUuOT_tU7i5i%5HI{^@#Hx@zhtP55>r_<3LwusK*SC#%i+gn&iRg z_8UN=rLVp*gT(K~{0X0f_=?~bBbfB`=XrTFn3U!)9n*@Uj$-mr^9PNi<22UJKAK&D z|1@Ck3(Ub;>68;)gIn_Zu{uoVRMhAkIqgBS(v2b2{gf?0xd(1sJfY`56mVy>~^w!wmX_kjW8#?_Nk{}zB9ULo>4fO(vnWfC+pG4>%*KZ?JuCdXu%aZ}q7pC%E50@U9+KQZL5 z!*I`SOtNf$Y$CsRsNaf~yyw^>#X_mCiF&*gr=cBb zoPu7PwX(+Wvl~i(XH|)jj@Cu+rzpJMn4kVvCJ~ReCf08viF$q9;CYnv-96k{G?pf_ zQglN`JiS#vok)~^Z2>41#7LPFgd_xrqNO%DQI|!Qs|nWt`co#BwY$&Wm^6#~)`_1k zpwiR~&z#mtSDuYm(=NoLv$%Y}bTjog$RJ8$j1(s})=}su0b?o8i28-|xu58ipFBml z2`4qZ$BbY5>(i2%wmh!+C}$97?X3LgTQ_{(SaFZvq9YCn@BNz z&h#;4h?5#`&_0()uJ;_rR(Q^eY*=&vu)#EeMeaN1puPv5+iQFg1EC(`_99_5v<1r4D ztc(+-eVWf_np;q$M*H49#{R)eIWCI%R&6F34;h9eNG(XNO5ao2MI8;j}y% zZeA>zX{#$;muhtY{_|;bkk~!U~Ih z2QUO}hk~o?sn;#|Mt$0}4=+BRa703n6>fBm(cesk8Cmugg_wi|BWj}V-VuU9jNH+o zgNYGSKPm>qR&nI(2Gu*})AOBfXf0J~CC50C!3KXu6-qZAG!VMZbmnqL6HWG>o$^sjoSLbQxra@WyKV$+_Qe}t7d)c`bpJG++ zw|9D3>XUH^Wplo~MN%WK18n3HeXoe*jKwVRK!=RMtIr1v z;Py~7;eZl&=^UyumN&CecrGBEat}4?mtZ>@`wPjVK@Z)FZ;05^9kztq;qmbxQIJ4kXTk)) zaVfD^K2x7SB6E!Zz@0p|Fkge*0(0?ogmTX8d=?n{2x)}K2$`bjDmcLg3#wU)i)by? zW^G8rRQKBwjke5zHScinRlE|wo0XyhBc9R52IsKWf4-@=l!yO&+l=K`-7Ib9U~hPy z!cH>H)e6$;m&w^0d`axGqDwBgu`B+L4a`xr#5g%b=0?c41`|lx0O9fiIVaFAsO$Ol zayhm4C9X%hzUf&ctylV$%ntuA$(yo*X`gaVX0$|x{#!YK^cvLmNWPZaTd3&xP7ny% zkn}2AdJkpAgmsh}Q$tY3(2RtO;%R*~8r#ZbSbMR4LaL9Sb6O&Ce(GlO${jtl&`n|D z9;zUQPXCHqTm&t^lk9RlZiiquSY_og^?kgVruz%myd95Fr!V z-$OIXSt?(pxN-M{NjA)j1KKIp(&c2RVjd_}7+CbQfw zTRjg}A0~}Ht_?-@wD0bI-;LQwT?mKywmDZ7*j4>4pR6@UVU3mb?-cbQt~aIG&RBjl zs-4UNtOH3+dAF%U=={qB@qijh4J6K?Et zPLlfPlv<+i>ty5rh;Q>iGFoaq4LyBIZl3L{KGUmqPL~ZCosOl;7w2SxcE}pvK;5|6 zly3JjUsvk|d7L3bFs&;q@_|p?vdU_UzhrS$Fw-_NoEdoIT#-0hKC37!>-i6FaO(es zY97)m4YO<|eqGMrYejC&-IFmc{=P7>qFWX;)}q!&e9-F59o>V+`X>J}%Te0$|A>0W z;7*>m4>udzwr$(C?TzhZqi<~6wv&x*+qP}v?C<}aI_Jeq*K|$4>AGurZe5=U>-0IX z>&2?v81(_Tn1tITYDSF@^Enhl9>e1$iAnX!+&YJVi>1uYEWsZ?o*Vyg+K~%XCxQP(WrdtEpc3sgbpTM_ zI7i6|pDr z{=xGh4O=PrB}pkX@o@A(%GfdU!c<$p#T*mLo^*7@bd4rIJ5eS&&A9VB$EhabJ1^TG z+dke8lOG5I(xMYZ`Xw8+olY0y6M)M0rcr%9tZHa=G0zICN@DQ>0rVASCK4=3OeMSv zD!v+POT0`UZEnP~1ro1?HPLqJ)xx0#Pg^yBJz@S6gmFN~cGvl(#fz4oTs7_Pi^+i_ zZP7<#ukx>i%V;uJJ~WwUW7pgq=>yuT+A5w(J5$1no67e(;mIO5>@`(U0{}+kg)B_8 zs=bfBbmZ{U`xjMpkAcEcEeF7^#ka}2zDU-sBt6yQqw&2p<+6Hb(Hi56S!+bU9AJJv*{ep2vD zG;PVwX@NC)+=6@I6J=nW6_99&4R00FKpUPepXoBVN*|V*C{e7X+Q({6O_^@SlI(9Y z8kRO3WDG5u=vmTjZ4DW89H&vNa;i%H@`{%(|J%tVs;1gDadzF0Jy%}C68|k?Zr!B9 z*lBN4{#6p#SQS-q#Ck&x#xhAOu4mK=Jxf+5E$h8l3-F4mQY^qaS5;Z* z-ddglOueLtXJhJ!%yJGk^-iZ_+qLJ zpTZn+6kq81D@^m(v$VFFI1Q!dtczYBt1xSn9~Q=@h%tsf*hCm%fwfx2u(u=-4|qf=I8WR*%`lsQ ziP!-b?(d_`TdA=^<$@(2c77&FowB0vhswM)fS>lYvjK7B_$<0SiQNzL6T?D721Y*( z9nG=@aWvmJMd%j$Jxp3-L4x99-X-9aGkW}yiPAo*9{^6b1>tDg4zIPFiTqVK$xq1rv1*kaE|~T5-jH#8{g31#^7M_uSsmQvNjyk; zbo|yP0w|uD1)wGrSavi=<;=H>IejRQlac$HMkU2rbq1{8UntI;oJ}*o(bXy{JC*l&^W{Y^}<%Nj1Tk z$(9f2a`BoyZZqxWF=hhmc3ldg+8&Ep%fVCSjopduonggw7@?XulP^JPo+_le`o@z)ofi9U%I z=~YZ3?Jok#3NeQ)U&qUqvoyuEMA?b&Ki=s%;_MTDX+8^>z@TOxb3qw~biG4!)XuQp z=>cVLGcp<{Piu-TqWLFz^P0>R1go1M41xFSn~y%8LZ{~t{iz!z$|ne5qkw!VwuI<6 z*6Bsnap!L>JA;B$u$J09!L&_iGdX<&v1jeDcEWM4&2q97^g9gK1%+zl7nY)PUU9<~ z!B??-0oFH5TEpfNW#V1m;(6-=mlUxm699O$g=ZrFZpn(6h%3n#!U7eFnC1BJzLFB) z-)SER^cpQ~AF(`0^?pNYWsz6(suJg4)Ke+|iTo4!8P8ND$ML1a%4|QMYe@SDDH#d& z)P6SOk~%xdQ?i^t{N0)(baSgQ(Fp*daGXR>=Vt-*#@)>A1Sfz0!iqKtjlY4}1i0v0 zyz)Z|vB+_QIX99Q+NFppI1+3`=qUen8NVELr!SOS8Vq1;{<}WKOhe7HMurM4mg~j5 z%|wM0)r4^=uC{9_OTf*An{G}>6hw}C=H|&8MY~l@u zmW-R8h;dJxjKNqEdGf85(5BrR>lY2A= z-_%9;IglQfHBuO%U)bt|g%1h-OMbL9H{TdFgM^rdBTt~gJ%{*c<;b$D13(ac>}*nJ zo@&y3%13-hUh^Oa$9U1ImdNfGO4bPX$I!c!6e;sRC>z{knTf~G5{#4J7y(vbrq-qWk%J5#0Iv((P!QKa6f#3?;#q$+(teR!nw%kOp&_W`3L^Xw}Dw&e2#l zc{fk56;UyHDpT@XdB?u!*)EdIMT8X1&e>VO;M_QH&MXI5|3xTbET#NTfyi14#+0+t zDS(NC?jbc{yIDjm-=9g^4*f1c;0!ytb~iQ;DSTKoa4ow@d-x3HI`EYcAe(li zjajb0cM*@u*kiU{)jd9yTNeRZLL+Y1&q`L>gx^Jj_B%sh2+%Z1d6xNVmTw5Fw!kd@ z+uT`4r(0=PXUZCNn9$VPo=aj+p${a|eqjB{Mf+k&$GEGV(lWHl#1xy1%5E)1KD$bK z0Z1Tsk4LpTn+b-iy}25uN>wvTfN+B~4r!aC19d7}&hDFchbqZ0;e7I0BK}RNujj9n zY8As>D%ez?Fkng~c1L3e^}<%h%!NhB5ZFmv4qmi`am*+A28lE6Pu4ekBJ8DW?YR4c zPeG`sZYLihHq~K3`oYvnQL$26Ojwnj1AOypgX_ca^06&6f`T8bedVhWj1y>F>d-sg zr9@SeL^T`CHIwyKW*F#~AZd==$aA_zOLRP>>S_&HK0s{HcEDpNQm9u|IZ{W%#*w4} zmN;)dX5OA?I{M$KLje0TCiQd&|g9E!YKD5 z)_8>@<$&L)EoO;WhhvUYgEDDJ8PPVpR_u`RN${}`PnjHc-4^~CwIh;mLF+#KK>Wc> zE|Wkj(OZ@zIa8-8rUq=a=x-F%J+$ozWaVUV@yS!{UWJ)}=^jM1_f&XffEjCb6H?Es zrqQ!sdrLtEHq=DIu@B|%&N$@{wC|>I`>>2EXn@+22x7PaM4p3V5XhXp8gSH8{)yq+VsXB@4DmPLA`4Qc`r2Z>3E&lVsUbpRejKO8Xc|ayAI6YT)d!q zrfQj!sa@T&5KPMxDUd4bZwub#5<;yenI>0~Zx=@R*M{S6d|Z3TAEsEW-w#undSQP7 z0ryg{By3CNOC^`$t=P&xCf<~vRz1}|>Oh+v>rBMi?&+;xKSGs;7Ie~^T>J4C9Ke&G zL&{aTYZk-|Pa*unK});DaF?Y=y73~NA0(lMPUz1G>G;8n^cmm2S>twrpU6ynN~J1! zHD!AXWk^D?nq)%#A^&d%DwIkh3Ku$<4{$Bnqe{R^e!E zD6qaK4g^V5kCJH~Ot$Im{2T}8sS28Gk(>QFg9I7A-=nDns|{X8NjAD%l(zhXxPR+i zsaKZiVQjKRN#@N{`Cm?#slb!NghtaUv~`T@mvslIbq5TcS-15muB2Hb$Zs``b(Pmm z>-keg*068f|SD zm-1~aS@!4?{PuWQ(%MlB?$oG~Y0UBQX_Nz{MC3%JvnoK+x5+GR`cIfTOE7r3_Xi|f z(1x{Bqg$A^m57WLbkEAc&hWkBABmV|cqNS(`o`}NaSI8Lm6{l$b%3paaK-^r1yrc* zQM|lY+je@P=AS7fX6VXPV>UYV77X|5G z5Zow(9=j+q0*H%#H}fpu-HF%`(GEbvHmWK({pqfv^b!p^KiWxjYXL)gZO^yLvY!1#{eH$?|l`7XcETF-V>)m#$Y-KUauf z^b+<*r?&Mks6o?n2JrEvgk?j+9|~S~2U~dq^}6M%or)_T?%jaFi!#+q3>YaIG?m3X z;{>&cQSHf29MCWgsDR$xyTZCe^~uYQ{iM+(@1tKCpyDxFoeVGQeW)9uT349)IDK!3 zsmbQfykCr7P5@r7$@N8b6KjN-vAfM%rz7|bveQ2v`Y|)B{2rfRwNw!r&1%%b*lWIy z+l$A~f%;yYgfY6h_(-1nXB!C4(VAsEqS^YKh9a{{_uW8t$M^?gPsm-J}^#E z_uO7hC+?sb1Iw^TeS$QC`8qwrX85eSYLIFX93I>dS^)6QIMdwX$;6F>2_T&M6o;jL zp&W3|Bd8rLlV}iSVY9G7Lo?V2_E`JVM(`rw^}DX9)wk0Q5GJ%esB@}u@C>dZ-byh| zBFz*MoXGGiF}DG?h!UZ#FN`;~1bd*pAWflMa5AtD-+Ut8Ymf#=b`potx5YLf&A%ZwGv$|Si7 z(0)Re$(F;{=Dhtq1%wCl0ijfk+T4jd3}^2Z$Q?L=1_lkM&nIax-Yo%VqZk6#Et%n& z0S9_V?yja0r@wi$m!-JJM2G=aQ@nYectR_Ln*dN6gmAR8L^dIf-bxR>0A)c$?#Ug@ zVlrY8#6Wp4wiP3OZ1@T=EBaaz(jrxuLG%?*J+=c#K7CorpL5*eKWVYiw<>#a7zv(N zO^RpkPM=xn!2?&s^7NCTu~a+aiGwc^_4Rnyqj!-l3-f+;6mkOx5@ynO(YF&u{yH5a z0{{W^{1E}V-LFeZcLzkH=SpZ_y1l&>1S=X`+@!Ai#KmNT?5ox%_;tp9`=F^;&%fxn zpX4I|M!d6`y%-8hequbo4%INVKruc+o|NwhsZB0<&TBCe}v2@CyI^$jlCsTrwmBFnzIMofx8PeKa1Av-Nj zlLtw2SI?rq_1(xc%<3sF%)ZrYIf>Xe7@jPt9BWoU%bg~g+6=1f;eW00nOrbo#*(mjYHCr_?8!#my~|i(0+2j{Uo+J%%rvg+%X5* z4!HCVyg~`t!LBG+X&89L&@QkGXe};GQ^moDsqI%U>#?IVQc53nUukdN%ij?m+%#Fv z*$`n_GFdWHC(!1z-ZhRjEV&n1wt#7VUXkgkW9Q5V;)k`XOO{*>9)xi@4}6zxlm4Ck zPC4Eq^0qB+yLg@{^VCgieuns3B!x#NzSr6q_VlhP>I4gzH4BI}DTx^r5(>Dyhc;-w znWU^i-9$N49%O1eIWyBV{K>wROpYjgCc5b?os*f=l~V;o)CB3G-E7LA7Rg3;!)~m@8(whM7Es zwF%4mEd^gMI<<|N60&DB)!+6-+8@EFbvGs4UP0$q5NEO<7?$NeaVcvz#eXkrXV;$H zPjNrI8gWTpphtwY&md>1N7T|$T^i@CM$EWZ;`6{q__Yr(^B!<>OPXT5%ICC%;4jl=T77^3T z0A$3`@j>`8*wH>vT`en;tj&YA60zbZw2F#^jE;rfTJ}-rcajHddN|Q>g}o$TX~osy`RPP=q0j_f1g@QgXPlY@q1Jh?-r4bB@~25Cj@AmJph{QR^Ya<4r(z*{F~ z=-nsVQY2K`sKEl*CR=AMEDIZD88T(wtjZ_((xf$>SIA*D#|jjfGw84wta;Nk03w~g zI(#i!OQDMse#AO065D@_gm?pQx@{rBjMat|bA$6MfVPq;S5zT5IKK&|LFZXuA zqj(kJK8jP}^ZYm?74hlPtf)m?w!rUP42d;f3Xx1K3raV-*P;*>hmzjAkyfcbEfZVM zJuLMoUQ0*&6p_BS@>f9!k`6HtNO_~}(0Jkg|_f8#- z!m%Jn^dX^G#qp$LnY0H)6WbFMeDL2eCjALoKs@6Ai81!~l3d5bNgZQ?f zTgufN#)|A&im|)K13cIGc?~(RCQ+E^pAR%xa6I`LxD$=mcOf z@v4=zb!i^TVJ(CsX?zlhk2fs((qe>+8Y#o60peO430M?7HT|g( zcVfD7@Ob>SyV%mu6}7g*=p&J}hJTo9hFn2o9Jy}QCXfAbC}WgpkeMXs7QNle)Z`PI zaU4~Uz`idIpQPmpq$?{N(5Wj_y%UX!5{=9|{BFV$P&Z}ciIVj<`zLyWb*T2wf|8o* zOk|-Qs_aJayia$?0k_jr6b#)1ONJ!Z;{~4NDyZJ6id*&SjT|kFCPH^!Q8MlaAE-*_ zNR!vqG}YZ6i}M3h>ENPmCHxC(#1( z7}2c0*RmVw1@+)M+n8t~gQT#+Yg3>|OA<9`Ynl5)ftY4g0EGA!t?E*;j*jRcB>mr~ z4f=etCrR1X;V_euWY<6p_AK%IoHB+bS8vl&LZ-5Q*QvzmfHq zZ>>MgWVvSa-wRV7cJ8O%vi&R+@2I&X=r`1P1;x8lhOpY4Z58^@Wm+--yBQ{&>GOL- zIJm(euOw?WYjBR|f~ue4(%k0i{lp`gI1~mF;g{;-0_gdf@ z*Q?M9wQ1ZdZwvrK|IY39={n^R^(zI|p=Px@ff|e_NEBug4N0vK!L9-J_DIiI7e5Pr z^Sce&Prjs*$mOY7Rf3V+?poBWP^ki{PIa+)OK%4)E`rV zxx7V^Qy14sZ;Dc2jD|ccyt5(5Zp~;Rg7N_IwB&EZ1jv&GoxT!1H7k>pY>Aa{$&oHg z`ykhr&GpvCL?|Xb;O}(ErzQAl=DZgICR);;Y=xkO<~chKzvaND<3}Wy~d>W0L>Q| z2-}wM73&w!hC@XZojB#$EnGzb4HAp3FWovUq|4f%x4KLKUg6YfVpokO|+JO^JSzIZEji>8`uBI~^1wYq9L`S;8*pu)y zTN!cO5)p_vO7vsEgglr#ee5WTiRh}7f0zLYNA)eB;_ z63%8_pGF-Dnkx@eu`dPn7Z1~vMk@*nIMW6HtpQX86HiyI1H>8W+4Y50C=@;!{F)Za-A9+#^G9aiAu<-#DuLR>+Vm6|21n$W?isfhl9KnurA)AcxJ* zIl$Iy_sl)Ewu1nV)Wiqc6M8RZ-OvG~x&%#S9h{L)QE&q|7$gk|*5h2|^bAvwHm@~P zRY4`*Kw4vB$#(Yqt2+Rd{vNGl*GA$FksiM6%fjfp!BEgA!3EEIq!j+(-cS%{(44@I z+KuDSMAy-fyJ3j}-3vV|_^?zVAkrrzw!3@QF<9e~z*m55Kjm<#D3z(4wCoyq=E3Z+5+o%*c82=9Dn;-mR<5ukCVG}$pfS0a zGXdRdAa-u4>?Cv7*|^+XrkWQGzzvT;h$l5u$vMI>9ouxPD^S{5-qvWAprQ>*&?#SpxdJ-SE&Kk2hn zy8lWI>IKrj;hSj%<-bXl8V%B!q_?jcj{k-hy&J%P3vb%^Qfyv08YOw$Qv~F2IOcFi z%I^ScI`VdU!El-&Werf%8X2asF7Tsk7{xt!qlOL$mCejuXC38O9pJ8y|M>$P50HUy zhcG}uKWP7NB@OTY;fq3kG@GPwLy>1x#YEu`vmQ=(0K)g*ckkeaAkM(C2nZ)rJS}8_IMTxIBXH|>190=4 zD%!`?a-E!T;jSVXMP%ETk{4ij&~`Q)&DZieRx)rLfXGfwvm9#PvZgMyX7+TpsoXa= z4Qq583C|0#1W{@tX6kUwtN40v^oyycsiqPP<(V!5f5bA~B0ZGZ{CU#4q>RznC|I_) z7I8BytRK$$wnfi79s*Phn%|0s_u9`zwWi2#=GE5F_sk({H`bq&(QCDy^X97O7~dVV zjm7hN0FhFY>Zr6d?l;%A(Z~&Ew$4)I4_&92>1%LB&Iz>(85AY z;VB`o-(qZZj2^wUL9TY=pDZ9{|L{Rg0eiHZxKR(>6I;B}xV?kpOG_~18o5kM9>bF; zvl22sk@FP)d1Mu!iPBd8n%hqPUH?B{lf+vBfKDaUjH};FB`hI|=TD}i4-Df(W|+FB zCt09JV@dNOy}=s3AS(U4&Ca^LI#IkDbY6-0Iby5ba=y`Wp2hYzhwTE5+|7W}HwTbp z9OzNwQYpe;mIt%rDX*W89h~mxYK3jmf-7Q*)B9kUP?Evo3sn(X81NyML>*eVx+RUlBPA+sDViBwk z7*Dl;#i5JP1+7=3^WriySJy*Ub#&|n!0jaOtW}%-grYW2t+eT{wz)iu1P?+?*78D4 z?m5`fN!6Uv7J4JU)^8tW`D-N9QO%RdtYTA8+bXhEgPf34?k{g{4Tq?|%C$Kz+U{9j z8RcUt*R}dKX*G74+BGaNebZUV{DCm;@U(5XnJYWyX(1gNvxR#br(Qa6)^hmsfX#aR zk+}yFE?Rp5@=+8!0rVoYMrk4eHt6+-pV!|CZFOXL81z;&nOQ!ct!B%hYyCe z$8CC^HadwLAC?`$JgYtvu%$b7`9Y=%pqA!R6Z96z- zLhL(4qE89OG&)oMjo05P>;5?Mp60` zPWdJ5-2@SE9T{-ytDRE{6sX)|Y1X;+C@K>yY^}14Y!088xh~SPfbJG?M1tBi?E>u?zdU>G{5+S>|$%tGJB zQ*X_vOy)g;@fbPm0a(Zh7zTzw2Ct$FB6Gz7!tmK*tZ2h588F#jY1p`jSJMli*7u-; z3tSU(fscAw1h}5i`&i`+?4UAF;AeV|b}3)i5zA^E*L0X|u;#%xYNx~?#g6jEh~;8t zQ8$5Sx)(-Y-j-9ugVW%b2(t*(k6(`>S>s9^t-podjkrgd0G}k7#${=(J0T7``%9)` zbz@# z89pMA4}>(ymEcPbh@I>#D9Az~sbv{(OXEh+fnx{b z6H8ULM@UCCdJbtvxLPl+w?prh49<(wWQ*(&g-1S%fFdrWy;&bp2wdG!zXt0n@O|(h^&64U7Am>%tK&1tn{(CN?9?pRJVbV0abQse6W* zjaunJ1r9_dkDSXE8y~{blX@E9+XdZr?+Cj9fSv4Dr%sM0X8+%}yVNrc%}Pks zfLfd-a~NL@9Ae&`->H9ihbrSTQK7`l0(9ei<9)-C-ZjdIKdOKOVrZbL^1x5+({hmz z^ka^IzOo7Z5kDX{UB^aJa=ZJ664{}im=U8r5}V}6e33gr#%&kPksN&;R!|y`-hx0+!ub!fTfgoWJ@3*jQ48CTp{?Y z$+bKR>!aBjD7x?Y0>>e`M#1*rfv0;edmByS@dJq0U>!j z12B#0J8%)E#AT3Tv<7hwsa2De$TgZ!6ya*gBbt8{dMpCoYg`{48qN!f$4KFI>9kSj zXqP7qQXV6DfRu{Jr(Mj>;=zUW>U{0sd8$z^(2$UE1b=z(K3T=YUsL(r3UwB%vS_@i zUw15;g`ql@wnozVkC>v|rqdrPO1t2>x^$SM@_>ucDEgntIq=60A2|p%szF-JmH5_! z>2S4sVX}c!H;5b!MnOy^fZYTP60VDhA{ikCTh{$>P4GK|N)1u_VGJ22k_IyXwj7Sj zcn5~M5{rQqE`|I<$3Bj`K#{b$K^z(UVwE$D46wB&kBgN&?rjSskPyQ3X&G^Acx^iv zW6lXF-}{o%ux^olbi{%ZmZM_C=6u(%CKQ={xs{jYqD zM26k$`Qj{UlW5Jt`l&1QP|d=7B{Dx;qd$8JdU$AE5&l(!MUkXC0mFRCM3JnDw?zVe z7`mm7)u~!VZs$|ahb9Y>#(9sjOV zcH~0w!lwVVM3oxLQd(|~MDZCpxbXh7qmbj2l;)N4J+?HVc6Jx7LG<@F&tGUvek#38UUOBInuVP22k}b4Ep?bEu^--cB#Ag|hqHNP79!T*v5&|g?2bQG86x5lB{ff(Rjr7|;rT&I0Ef(#dGARy zq-)N|z^0X-fAevH$bL+ip~x^dH#=T?vKN@HF~)7*3?~kd(`GwzGp*%S?H7db>`8F> zgx!tP`bl5-7lQ@AQ4i^?mNUb^ki+(Qvxg{R!^Ut%ya1_K$Ci-wGtO^W+(5We9^Z|i*}v@%bg{vBl7i??boO`xvQUh$k~C|d$i?y7U=W| z!<=;Y;tf9FpB=nOaU(_U#7Npj4id5?8H4? zsL^r@1_p9?VMR4cVe#mEOOH=f?>dB_m{#vzpM&E&KVbxd<&r?NMbz+F*duzV(?Y8LUgUpO4?&3)QPk z5&HoWONJr}EUHfHzJW4vCdqg&<>PN7f)paE#1!i^P<-8JfbLD7%T`A%By{h7P)CAW zJ1E&XBE96%#4a;dwNYQjcdiR0Nxh?uH~|2q&7C9LQ+QSv8X^PP0>Usz*HSS9C0>to ze1pO&s7BCS{x!VW_Pg@E-%TErJGYbnQ2hXL%RBzBNmFecgMmO#_uULhV~c2I)KHP{ zv{Eui!aMjaX?Mf>WoHp0KtGR^e4E^69*4@*{%8^>HwxUFNcSt7W0h7X$VzQ5JTGQg zLpd?yN%(bgiP_o-cst z@QA_VD0&n&*dj?j63J-vndy~X;lwmo=Q_8PV#w^VZOiYw;}mS|B;|u)e#GS8JRqxP zoWEuBMb#F=PknRG3P* z4GJA~MMpEbM%i4(YahXGEOSo2nB;oM z*5&1O`U}@hdRDps0PqD~2c@$6cz7sxmZ+b)O!Nllqto*I#I^<9nQ}0`3gtZjgFSc` zr<;IuXQCn=vP25FV3h8Z+}TdG6Sel7VCP+9#!U`9SHR~u*QtV&Ir;S6Z^sSGm|s;y z-f{CTn7y-&!B@eo#~6{h(77Nh6dHLyQG)b$p_3Gj)aRs!q6N>lUC*~^HSvWstrW}u z*CU=O3^xF*0&%aIQS)f~p!Vfgr70q9_)Pqs1=T}zL2n7bM8o8g#*F|Q%n>{#zGI3aoM5ptgqb|5#Q0-fuPveFm}*t#6J>nQI?04W zddadPl-27!^`1tRpwAVEqlr1diwI*)RCifevrPbt5Gp@fxs&zT5 zsb*ne&_BG~c(7H^P%7ADWn2!iMjp*h2XH3HT6VU72#$t`4=n-ZMCj(Lx2fTA@Q*v3DH1nr6oj-PQmZ9zCOcnn|~y1H8R1_aO#cRLv8n zA^SQ>qnD0V>X0{ZGw#)({*;uB(U$-bb3>y#gPQ0j{V0TAh2!q01pnET-gA>Z&%Zu& z{QmIumszVzi2m>gDlumvArvK|eWjErehNwr_*YQB+{U0n2iH{TJ z;qL1>Q|tNR;tK>w-Y~Xr!pxa~?@n`+EF(yvE$iV|s+c}C9kp5-ApELWNNyD z|D+=Q7PY%KH^%y&U#ewXB(vfZd=y2g6mLmY^!M=zO*K@jEGVFm+gRBYv6`7`j!j#_ z9w|2DzzCJJ^>~J#5j;E8*py74CK@&dIy0mkEqwTPE}}scXFHs_!v+39v(Q!~u%}FWO}FpFHX>#>99{bVQXu z&Mv05icalrL5O4IcpQ-%8V0q0)*4^oV6E1=wCFNkQG8D|Vcl#K3ekLmEmuno2}tcn+QcBWaoDND z?$>_WkP~3jJBVSpFIV5PxKA;nAt-PpDTxDvS|U0B~sCx$DrPuUWy1s-9;QX4FU@5U37&vhcuXyFpWC$dZ2bo2M?j zANK_Zrju>J;S;e;$Q-lXs>AJ;X+V(MnIVQV<}7RvF2tip0dAnk>SJRl?)-~WoU!77 zQ=Tzv)wwG*H6)RHIJxxBSAnc$34YukwX=MWwb+&MO&{6*3?R8{8xnSKM?Fx^SIqyB zbIrq9*-wfEPB-!(hD)U;417Yhr*_v$3yfCOLjgK9ct=m3wC4po@*K`;f?423NQ%Ha z=HQfTdxjl&#yC@aA?gUOwDc`m_JtKN%GtmX{+jhTzM{j)Zz!HLVWS zT3ud61ZuseM>#VB zB1v^H3>~f3ZuQ1y1W{>t-Z=ZAh`cL8Ph>}_y|h?Wg&}{_PP-`L`oK-Ig}U9hdlkA` zD(w7nYK?aP_vu?cAgjvw$DWY~|Nr`6dn+Ike-c>$`F=-2aTLj*LyZCcadEaCUHG~; z86DPAtoK5nu-&tR!-E*UKmtjQ&F-bed^U;yv{`=a-Q3MyR&EFcei`C7LwUEikDKv_ z{n2hUv{KSVf+2Ghr?p6~s8Uo}UNjM-Va{4f?=S0P)GQHiP&5mMDO6_~Oh#6NWhYTD zHVIY-Br?zR-A}*_d1E(u4)4jZiSX;qv}@p<)$5PHa8uof$- zN#h;PX!Sh`GyKY@#3`XavDTF!tlLp7pOnP|n7ydSTSeRN`9lT0{FsiXdyibTb1c%L zVA^GmC!c-pE7zzK?fNiiRLgGuZTzKsr@X+hJ&sngBnxa3+bfw(?G&G3Q%W|MUt{C{~s zF!W;nx?2MjfY!+%*n5u;$!Pee07wYZ@g^V02=j281Q-OI#l0q(9<@WCr<;o4(a|TM zH_t`S9?g&v-JRw*Z;u>5#?|UTBD=ggqWPrGOk$%Eut6-?OV>%E(R=5l*y|X#64&>rZ z#W3LPCfr7TgzQ0(qgidWUQd+uWMCx7o zEB>|%Jj&TVz$-D|qVAVU4!CF!@J}!yxFe4cX8SF|Y-XBWZzD>se-R!+{t?Wh6=}E7 zVI*Eoa1su_6K2`e8XfsS4OJM|U+&-7VS zIRJ0}JFs%}kcBm|$KkOHXW8Yj-C+KS#mq``V56%9am)P^?MzJPWU+*SyoQeWkRCz< zQ&Lq-Q>VTUJh=@7B#nHSC6HUHAey1!j}y>tP-yPh!o;992`-QHd7AI5t9 zPzm;}i0kMO6~Kl4TT`Y-BTU9Ku;r}*Q1TDl8m%S{+PFzk4&HGip;0#LkTx>X5q%>5 zvea2A%tl(PyC6CoWZ>)xHQQMu6n`UxQHJwS^%+zbld7C*CafaNLfh=(7&7eb)>jvC znLDJo2#ICn^BvWW7|$|a>!k)dOwPL;_Ao<@lzuJMoVs>;vkRhel4yyS2) zNMgz=@z?&pdF|R2kYSCb~_c?Vn#f0va))?V7TyrsA4t^o14=CVLW+YJt zornR!@R}SEh5X@8Mecwsv4(I7&TsC{FBAkUqM~hI4`ElK`EdgmwXTtz>9XPZVjTba zBi?BtsK{w&VnIK?b}XqbS5ujgFthngi(n$Qf0!GV*Ck3#A5=c-XwE4I2shGOBSw|T zij+DsI~26%8A9#jM#!kkG4k(|p=DlNOtp$^w;d!`3Z6v)Np-zYDWC&3J{ zwaUiwtA2L~pTeKQ%+q-puz^>p5WizwIVWT}a7;I6vmOl}V!9x!Q0+N)w0dK<>Zy?Q zIMqMK-zUY;#%$)=v;*}7l%0g)L@qrQ%(KKJ+7(26naCnPXDl!4!)l8vCvdPEi@Jw* z|6Y0vPmvHvkk-$$00p5yRzY+{Zx>_nKI_Xh)l_9kFz3dgjETw(U=}g;=}5EaiyMu4 z_K5!H6(p54QnUJxGgc8!K#+;aOOofhNq5c;z10R2IrtP1H4@T9A)rjBp`BPHrYhlL z+@cieQ3~0svr%Pi6*}fPW-L9x=CjjPl73d0y^9szowR56%tm}k>B)RtEMvOL*=5n6 z-O4NJdBneKC@(Ak6105naj(;SX_5pO7!J@7^!qDe`+jzeJ|J9eMX~dq_a4ty_&9?( zEDkVKBj$N0>Ka>58Y|PQq{Q2j-1e%45yo0bM~*k}vj%t;)h4!(={qG%V1_LSFm}aK zY-tE~MG&?}B;H1))pTEj@~LYqj3<1_=`$4^b24-b8Y}Do-qUr>x|NiG?ruc-9+TCz z;?EP^qy0SZdX`9sh!jt2^KgHyRrl?I`X8rO z8NK~qffuwrcv^i<^-sN;(~rF>En&Wk(?xUpXJ1i$BT!_#xy7-)Kt@ezB>Cmr;5qh^mji@urT}VzT*Om+_r%F`x$OqeakZ|EVfr%`L5IZXlLN1Lx$X$ z+~*?=bbBH!DkWE20Z&N_tCU_B5$>9N<-1b_)B4t9h0o5Fdg(TV#T=ZS;k;e9y5Pt( zcf%BKR`r}pq4b=}Y5!VT0!2?uu5S_u400^GsdDb9m9+E0!adTPK5T5=_*&)oy9xJV zF2%9jIC6B{IhfKk_L`{##PdAGvbj`=i^IWZR_QpWl7Pcg=0JJdXRWYv_wxuM9&rzRW2JGR-w|x_nY#<=SNhGv@xPUGak-)N>My zOneaxybJRv4`{BQkx7I>1a{^b!-nmXAIx>-%-v{b>i|3i&3>}pJSUmS2~`n_z^+yS z5F0W84=jO$-F%Y+=gUmi<5!s6KVLxR@N}V>dBECiGq5qIhN93#0IX18zN$3hPIm?d zV-!XFlLO}a%OLKmW?-;Ek-sboG(;JA1H1~@Hsm`!ZBY~!NrDxAkW>XLMBK-SZsJh| zutEn#h>3_B?HCwPO>9vHDV(GNHjo8$f7;~2gO;L~=q~SL-0fWZ~#j)X&6Bqf(AYY$jk0PJ03wGnXMds4rYbk)o%O?X5s6!3k zfXNPvon#Tm&!fx7m@-U0Xlej*iY)lxbYN7j0b(5#t3F$TR4GoDU7{+BI87QonpRme zOct=Q1)0SHI@Eabh9zRm!uB9RsmW9A4Z;2eABzjLU@_3Yb|{tzO}1YeB?~&EwGSvS z2b9-Gk@s+Bn7q;166{pOsgw*1jwq^ZTtTWtCL1hsmqk9p&jdx)T@RQl&dDjBieNJl zr|tj``9o2y>jP8GF7ag{X4W>)a%KhoKvyva1`M9A)97C%`B`O-U1bAu471WI(n_BRXdc33Qc~vQcM(m z%*7)yFC}Mk;$lTsaNBmW!75Q^;mHs)A-y`Vxw6QmkOqpmsncMpwYY?M85qRpg322J DDw4oP delta 34943 zcmXuKV_+Rz)3%+)Y~1X)v28cDZQE*`9qyPrXx!Mg8{4+s*nWFo&-eXbzt+q-bFO1% zb$T* z+;w-h{ce+s>j$K)apmK~8t5)PdZP3^U%(^I<0#3(!6T+vfBowN0RfQ&0iMAo055!% z04}dC>M#Z2#PO7#|Fj;cQ$sH}E-n7nQM_V}mtmG_)(me#+~0gf?s@gam)iLoR#sr( zrR9fU_ofhp5j-5SLDQP{O+SuE)l8x9_(9@h%eY-t47J-KX-1(`hh#A6_Xs+4(pHhy zuZ1YS9axk`aYwXuq;YN>rYv|U`&U67f=tinhAD$+=o+MWXkx_;qIat_CS1o*=cIxs zIgeoK0TiIa7t`r%%feL8VieY63-Aakfi~qlE`d;ZOn8hFZFX|i^taCw6xbNLb2sOS z?PIeS%PgD)?bPB&LaQDF{PbxHrJQME<^cU5b!Hir(x32zy{YzNzE%sx;w=!C z_(A>eZXkQ1w@ASPXc|CWMNDP1kFQuMO>|1X;SHQS8w<@D;5C@L(3r^8qbbm$nTp%P z&I3Ey+ja9;ZiMbopUNc2txS9$Jf8UGS3*}Y3??(vZYLfm($WlpUGEUgQ52v@AD<~Y z#|B=mpCPt3QR%gX*c^SX>9dEqck79JX+gVPH87~q0-T;ota!lQWdt3C-wY1Ud}!j8 z*2x5$^dsTkXj}%PNKs1YzwK$-gu*lxq<&ko(qrQ_na(82lQ$ z7^0Pgg@Shn!UKTD4R}yGxefP2{8sZ~QZY)cj*SF6AlvE;^5oK=S}FEK(9qHuq|Cm! zx6ILQBsRu(=t1NRTecirX3Iv$-BkLxn^Zk|sV3^MJ1YKJxm>A+nk*r5h=>wW*J|pB zgDS%&VgnF~(sw)beMXXQ8{ncKX;A;_VLcq}Bw1EJj~-AdA=1IGrNHEh+BtIcoV+Te z_sCtBdKv(0wjY{3#hg9nf!*dpV5s7ZvNYEciEp2Rd5P#UudfqXysHiXo`pt27R?Rk zOAWL-dsa+raNw9^2NLZ#Wc^xI=E5Gwz~_<&*jqz0-AVd;EAvnm^&4Ca9bGzM_%(n{>je5hGNjCpZJ%5#Z3&4}f3I1P!6?)d65 z-~d}g{g!&`LkFK9$)f9KB?`oO{a0VXFm1`W{w5bAIC5CsyOV=q-Q7Z8YSmyo;$T?K za96q@djtok=r#TdUkd#%`|QlBywo>ifG69&;k%Ahfic6drRP;K{V8ea_t2qbY48uYWlB3Hf6hnqsCO?kYFhV+{i> zo&AE+)$%ag^)ijm!~gU78tD%tB63b_tbv9gfWzS&$r@i4q|PM+!hS+o+DpKfnnSe{ zewFbI3Jc0?=Vz}3>KmVj$qTWkoUS8@k63XRP2m^e50x-5PU<4X!I#q(zj@EyT9K_E z9P%@Sy6Mq`xD<-E!-<3@MLp2Dq8`x}F?@}V6E#A9v6xm%@x1U3>OoFY{fX5qpxngY z+=2HbnEErBv~!yl%f`Eq2%&K%JTwgN1y@FZ#=ai+TFMFlG?UV{M1#%uCi#Knkb_h| z&ivG$>~NQ4Ou2-gy=8JdRe8`nJDsqYYs?)(LJkJ}NHOj|3gZxVQJWWp>+`H?8$$J5 z*_)+tlyII%x#dId3w(oXo`YEm^-|tFNNj-0rbEuUc2-=pZDk7fxWUlw;|@M9s1 zmK9*C)1Q?F5@NPUJOYOAe`GHnYB%G37_sg3dxAttqLs6Bro)4z ziy8j%C7KKDNL8r#Oj6!IHx|N(?%Zvo31y4;*L1%_KJh$v$6XhFkw*E|fEu9`or?JD_ z13X4g92;TZm0jA0!2R5qPD$W^U z`5XK|Y^27y_Q%D>wWGtF=K00-N0;=svka>o`(;~dOS(eT0gwsP{=Rq+-e2Ajq?D<)zww5V36u6^Ta8YT4cDaw} zfuGnhr_5?)D*1+*q<3tVhg(AsKhR1Di=nsJzt_si+)uac_7zx_pl#t(dh816IM zvToHR%D)$!Zj4Q^$s8A%HLRYa>q9dpbh=*kcF7nkM0RhMIOGq^7Tgn|Fvs)A% zznI7nlbWoA2=rHHbUZ4PJMXf{T$@>W1Tt4lb|Or4L;O!oFj8Op8KEE`^x^*VSJ`9~ z;Pe~{V3x*-2c|jBrvSV8s+*Y3VqFKa@Napr#JAd}4l7;sgn|Q#M!(<|IX1<)z!AC3 zv<5YpN58Fs4NYi|ndYcb=jVO6Ztpwd={@3Yp6orUYe6EG#s{qhX+L^7zMK+@cX1hh?gbp56>jX*_Z|2u9 zb*glt!xK>j!LyLnFtxs&1SLkyiL%xbMqgxywI-U*XV%%qwa5oiufFerY!wn*GgMq` zZ6mFf8MukDPHVaCQk#oyg^dhl*9p@Jc+4Q9+0iv?{}=}+&=>n+q{o z#rEZ<&Ku65y+1eRHwcl3G7bR`e{&~^fGg|0))$uW?B@;_sWSls!ctnjH6ykmM8WJx};hvdXZ>YKLS($5`yBK38HULv}&PKRo9k zdFzj>`CDIUbq8GxeIJ?8=61G-XO?7dYZ;xqtlG?qr`wzbh7YyaD=>eup7bVH`q*N5 z)0&n)!*wW$G<3A&l$vJ^Z-%1^NF$n3iPgqr6Yn_SsAsFQw?9fj z&AvH|_-6zethC3^$mLF7mF$mTKT<_$kbV6jMK0f0UonRN_cY?yM6v&IosO?RN=h z{IqdUJvZd#@5qsr_1xVnaRr`ba-7MyU4<_XjIbr$PmPBYO6rLrxC`|5MN zD8ae4rTxau=7125zw|TQsJpqm`~hLs@w_iUd%eMY6IR9{(?;$f^?`&l?U%JfX%JyV z$IdA`V)5CkvPA0yljj4!Ja&Hjx`zIkg_ceQ;4)vhoyBeW$3D<_LDR~M-DPzQQ?&!L*PUNb^moIz|QXB=S z9^9NnEpF+>_Oh6+Xr55ZLJ7`V=H}@D<70NiNGH{~^QE-U)*Sg@O}M|%{Rcpn z{0nD@D%@8!dE*mndd2g!-q9;)jb=IUED<(Pxh`9B>V3z#f>82~&CVZASC?|;C-VKy zJU35T|3jd(p8F|#n@T~Wh2l1yURI=LC>Uj_!8i7-DE_IaSKIMAx`WMEq8kN%8sAx% zOQs~R1v12(=_ghVxzylsYZum-%8QmjM3-s2V!jY|w#ccP)}OSW?MWhNu@o-t0eTg{ zyy`}x+}GObZC(k>-upb2C6#S*NOfWbKEyReP%gay8MT!pJpsx4jwCu%>7%sY}1L6Vybj_P+;yP`YS92 z^o_G!Gr_NP!ixe7d&82H&achfi83L;le3Fs?u%E*xbeOKkJr7mp=)RXjZF;h*hR<= zP_cs1hjc}0JlHal=enmG&G8wsn%Sm$5Wcgs=Zc}}A%3i6_<4k_`-$k2E5f6QV{a$V zg3VZO36o^w5q`q2ASwJw#?n7pBJyGt3R<`Sd8d|52=h&`|CPq&1Cz&42rRCHNjDZL z$}Y*L+#N;!K2Ov){~fmQM8hVYzj3H@{yS>?q3QhhDHWfNAJ#q@qko|rhlaGG4Qrvh zmHpmg&7YvgRuI|i78-{)|wFx(R^_ z{ag(}Kbbbx=UW42sAu}kg3yB#96dJlOB{+or<(51ylVwpXII7Hrlztq!pefQ?6pQhqSb76y=sQx zOC-swAJaqnL_ok{74u_IHojFk;RSSFfjdLrfqq{syUxA$Ld6D2#TMX(Phf~dvSuuX zmN2xzjwZxWHmbvK2M#OhE#{`urOzs=>%ku}nxymK-dB~smas?Z(YM^>x#K)M@?<&L zeagMnj!XK4=Mid$NvJ+JfSjvc`4rX9mTo^+iFs0q7ntZ{gfU3oSAbK_yzW3WA^`6x zWgPSLXlEVvh!G^fOzZ-O{C_v;V6=;DE+ZqRT4mbCq}xeQ0o z98Cho%25r#!cT_ozTd~FK^@AB3OnrAAEDI4==}#I_v}iw0nhA{y99mFRG*1kxFkZP z+are- z8D|3WoYE>s0<=h)^)0>^up+nPeu}Sv-A($6t3AUedFczOLn;NW5_xM0tMvvrOSZ}) zA2YG1m4GxLAHZ5k>%}pHYtf-caXMGcYmH8ZPLX9VCew0;@Pi-8zkH^#}Cu$%FmKJb=!)Twj!PgBmY0+>VUsyyT}Jy>vMt zo<^5lmPo5Jt-=)z2-F{2{jB{CpW2JDj%~JnP*rq^=(okNQpH=}#{kqMUw{&=e-5;G z!FwJVQTDS7YGL&|=vJ+xhg{dMika2m2A#l@$PazLQ<6$GLC+>4B37`4aW3&MgENJ% z#*tOQsg{>zmcuSgU?peLA}!Rlu&K3LTc@drSBaI?91dK75;_`(V`NHjkMj``jwjJx zcm_!liUxn=^!~0|#{g2#AuX9%;GTBq&k+Jz!~Cc+r?S}y=Q1okG0PRIi3C3wgP8F| zO2jcmnVbGXp*Mu&e#a9Q5a}w7$sITx@)8b}sh(v9#V(H$3GLHF@k!Wh+)kNueq;+r zFtj+^b1TQe?R#Y8{m!7~e6%83hbPKoizd2LIg3yS5=X2HE^l4_|(2q#LB zeNv&njrS$?=zzG?0Min#kY+3A)H1uMfogMYSm|vT%3i<_d9X&~N*ZCL4iB@YaJuo; zq}-;EGx~T43kq-UHmTn!@sc z3bwcs$rp?~73h*uZl_ysD*WK3_PS1G3N^t3U=KoRm_Gz@C?M>+x9HRMk(cA4m&L`! z=Lb~4*9zt*SHJgsAMAcTy*!1W^B>4T_doWvNw7UwmyA=Wq&kE{*GVHp9Yk5goUO;k zVb_3ARrFPG;&>Jv@P&`z%}t!*M|2127pm{S)gs~f_ID^lOH@nIW9DgU$=FjqNW0pv z&GYdoxe@)RAWWx^j|$N}sj*p)_bFpk`Y=NilvsI(>!Z&KBo&I+wb*kM5Vvkkr#;q< z3CobbF+GJ#MxL?rMldP0@XiC~yQCR57=wW_<$j!SY*$5J+^v{Pn!1{&@R-lHCiK8@ z&O=XQ=V?hjM;h&qCitHmHKJ_$=`v%;jixnQrve^x9{ykWs(;!Q9mlr#{VYVE93oaW z&z+vBD}!tBghkriZy7gX7xJp8c}ajR4;JDu^0#RdQo2itM^~uc==~eBgwx5-m7vLj zP)vE#k%~*N$bT#^>(C1sohq+DwAC{U*z(D)qjgghKKSy#$dPih`R09rfbfI-FLE!` zn!tg71Wr(D7ZV*4R@GqG&7)2K*Zc6_CMJoGu#Yc>9D#{eyZ>u-mrWG@4Hk(je3lnH zu9qvXdq+!`5R1mlzWjV^jvaHl>-^Z+g^s5dy49yem$0$>341=EGuOY=W5PCFBTbNN^19iIQ57C3KcV}z~z#Rvngs#j;g2gswC(TLWlViYW}tB5T#g4 z%vDUYTo1@+&zE&`P%fXc^@prE5z;E@;; zKtpEFYftJq-c0sD6lKYoEQ;O1X4uFZZ;3gdgfAKqIc=Dj6>unXAdM}DD*@a5LHk~o zyJjW@aK;XG%qr<)7Rqh7NdUpnTR6jc;6{FKcK_v_#h{IO{mez>^^70DAWB5whqq!J zevvLUotE;I?IWWf!ieJ-Hx`TqY5)ND>K0NCb7IW40Jk*J* z^#m%kIA~Go2=R|y5zM|*ehJxyuX;lOQZkArKVbQV(XmidUH|8U^q`wP(7%F}=uG}U z2~&~CLebE`c%SCdeU(l&hryL~+Y)6I^d@|||6F15IAGo`G+CdVf zc+!EycZnQH)OBE zyTd8k{(_v9d2}osA$*>Q>Q&OB(7ShxA$}p8ChVnYlXl5My$HlVx@ATprrj0}6)ycK zcQy#bwOms1CnS+xd26}k?J;WI{HR_U+1T^I!$B^S=pJkT705QaMF88VJp!s%`?y9z8f$&Xw(A}3u_(n5G{!)yH&zN)S?c1$SZlo>XieJ zyEFa>_p9B*cY){ct8=dq>uQTf# zd4vB4)(ebwQHlSAu}(6GCe28H32pz^}l%Zqs;Yl|B=l2d9HrCcUf%wxLYs4CBqJ#{gz*u6V$>?9IT@uSf~2Rgk6CNw;C21ZbNkm>ZTc@2zeOSXVE^>i5!2>t%!1cI z{FZA`*o4=dTDG3&{v$3xVr%g;3d(!SFJU}w6x_Re(ohlni)I54Wg{t zWLK{A(}qEIH@pamgtr3serA{THlp_IR(gt0CFguk={|Ochh10)7UV4DcnO7fvL<=x z^WCMg_TI?U8(loaUnAe+Nc9I1JIO#_C`=kJG(&wy%Cr9vRFcY9^8{A3A>GuSW~Zk( zMA#t~0Dw?;3^Ue|lhSp4p%YvYmw-&3ey3}+{6Uhz?l1D|6nYNok6?4N_C!OSR=QtS z2X&QtWlkZshPo#-dXBOlSqh3D;#*_`hyohR>vl$W+QC>HPOs0zwHKN`?zIKqCTw&w&NUGNS|abulHe{D+{q z`WvLw?C4K97cd}6V6f2NtfIAO;=c>qi^+y4#oMjK?5Hy9$Tg1#S~Cxoo-Zdpnt2kG^n}`9)Df-Spvx&Oi+6xXT=N*0l|d`p!ZU ziQo9$y}PYIF~Zqh^?6QZ8YS*JtD^gynifSLMlVYRhBi*f-mJFS<>l%5sp5$V$p*X9?V-0r4bKYvo3n@XkCm4vO-_v? zOsLkR?)>ogb>Ys*m^2>*6%Db0!J?Qvpyd+ODlbslPci9r#W>d~%vcU7J_V;#Um1+` zG0>Q$TrOLUF0%a3g=PaCdQVoUUWXgk>($39-P;tusnMlJ=Dz}#S|E== zl6b3bbYaYguw3Bpv|O(YR2aBk?(jo+QqN*^6f0x+to-@2uj!nu6X{qLK>*PxM!i0C zZwrQ}prOw6Ghz?ApvM`!L3Dzc@6mp<2hO0y{_`lqtt!FcUmBG+PBwl?>0Mwu)Ey{L zU;A{ywkT}jCZpPKH4`_o0$#4*^L7=29%)~!L4*czG!bAva#7ZCDR|6@lBE&cyy5eE zlKHwzv7R9gKZTF<8}3*8uVtI)!HE%AZRD-iW!AJI7oY43@9Z$0^MO@Egj1c?o(BwF ziz1|k#WOgAG?^r1 z>+p=DK?cA-RLIvcdmwq$q?R;ina0SPj@;Mus}W_V2xHnYhOq~=sxzA`yTUOsJ`8`VOSTE=IZ!x`cZYqHbgPijF>J>N7( zqbNsHK50vkB1NI52gyb^PflpU0DRw{&v7Y}Hy2>pV@W2f1EOd2j;H?|WiV%2?Dk7u zS(NrEUDl81<}yY9J#OCwM)N?x&PB-%1{oD*`_ZLiBJ=16uR{n+Lk~!t(&9U#>ZfVd8Iqn&idGd>uo?L@sjm>c|Lk z12d3Y>N9U`342@xaHl&Q@oE5V-f$s`04q983f0#m_WF=X_A89W8C#{uCdTNUZ+))$ zakPyNU)?MDayCKxWh0(-v~1rd8FxocW=Dc6B1%N4^SgQj$?ZMoAMQ-35)IMgf&)M?c@}4QG7=DTq{nHc7yp=CZ z1dh~VkK%OTr23U1mJ*a-DxX0Psvh_13t^YcPl9t?_^$pPEhhwGp}s~f=GFR;4@;@f z@B;R1U6Df?yl#Y=BgYTlP&<|8K27||rx_?{s|L);GM3^{Nn8HZp zFqxiG6s3Nb;PW3O=u;(-o(*q!^2i)jHY%N@;O5Hder~_@$zh4xG#-7?#S^-&M~yc} zh5Y=ltLBnTzt;Y%YNqi2d1M1LOz?MJbZ|Nc6>x19&l_S*2Rgk$DhaP7Y-C)4_uPzf zQm)OY)$AFfE1(0SxkbbN4}CHnlU`RqYFGIE7S9ipx_Q0vkE5JRq4Uc%zV7$?y(x$y zV^)5zwjH~+4?xN z9s@x~w`C_cS}khfI14K4Xgn^iuBxkd^u}3cY=VZI@-8iWHolPtt?JD5lZ1V=@g6yR zj0>bd7Z(dw+@)v#r!xpZaAxgT?4Ton(h`0}fkfF!ZDSu{f*r#{ZRp^oOrO3iB|Fa- z;|+PpW5JKZxJ-kjHf`-7ohmnO=a)Xl9lhI8&$)g6R#6PBIN$QSC8kT=4zj?w&=`!qjkCvvz;ypOfR7P)w^ z-7LFhXd6GLrFa_vGLwR5MRvcV*(r!NhQ@}T-ikBGy!fHaiePD$iA{|Q1$kct2`qHz z6nAyERuqvM6i2^?g@w7W2LLr~3s?pBDk6ce8@CxV;b%4%-rXK-GOk+($sSNK;_FBku zm89B}tpzL-x{dPS-IAjwyL*t7N%7~2E)9OsWJJWHc|}BNa5Xwdx(j7i7AmZhs?#zi z5{y$uQdx?O8x3>+5MR05HwUa-YZa*|UVLOb`T)KHk|~Gmwx8MfBUtM|afuM$0wb7m zR+_lU9=W~Y$uNlxt&(@&1;6t!r69A|W%;k3-%SzLlBzc0 z`b?Jmo`8{LI=d|I3JDAa|iK*D6=I_3q?%xFSLg1 zI^!pA=K}l1joBBj8aa8XHp^;Lf`9xNa&Cv+twW&$_HAwZfHrVcNUrRccn_ z1+L!z$k@LK28nc1VB|Fbwm$wO;B~yEdww1EUn|s&{-Tu;@$d94BLL(OQYx|aCa|&2WPT{qJzbNU!ep>j){o5=6le6 z>~Amqs+mCuOR2)aB!#sK5fuui7LsO!Qzl)lz?Lm!QoQFWbNIkfdkrn|)YbSu8WwxZ zO{}a~wE2Cu)`a3X+KI#LHm(Mi+}bOB6@N~H2}Y)e*}w8_z^Sx`c?CWvu*2{K#yqGo zx!Cu*+8&tdw!eiKqZIQlJg5Cb^hZ^Zh~Mb0l(4m4hc1mP&>oTdt7eS-bEz8mU~oObme{^%56|ou~EPOSFBa7VpUZC z0gVc<@IUeo~q)&?o zU@=bz-qfWm)&0Qn@W_fc9{wx={&-#8>0xHJ-+Ijl#P&1qB-%*KUU*DCPkKCLzF*#t z0U_vrk1(&Vwy6Vm8@#Th3J5J%5ZWd)G0mifB3onY8dA&%g6Hir5gqMH|hnEBL0VVvl~aJjdljF$-X@a zMg=J-bI?2LGw-8mHVF7Jbsk1K4LgWi7U>~QovGT2*t^U&XF#iDs_E$~G+t;U;tZn_@73Y6x>vU%x` z6?l`$@U4JYYe#|GcI^f+rsy|MdB|`PQunKSKkja4IGtj9G6buN&ZSnYi|ieaf{k5q z@ABM@!S(A6Y}Sv~YJcB;9JeqsM|-fPIZZfOgc*FSzIpEdT=YYT(R(z{(~X&x%6ZM1 zY0(|PepBl4dK*@9n6@`rUMd)K^^0!^?U-1rrB*b?LEZe<5taFp!NoC^lc>}YUy?5FjT9tFmC+%%DYNa+L zWr)zMB%y_6L{S%;dk6bJPO!wmT=wPPK1b$%+ffWcO8;2T+7C28T?{!96{%d`0G~j3 z)6g<%$dC{vAKJ22nY)fnxlD>P_Xb&@>wrG+ZpfQ%RX=R2kd@bH3N*M8=BO zi|Z$Z5e`0NcU5&aN_DST8O@4v3vroq3t<_5hBX;d)*AJgWPb~p=qx4}^Ms6pgyY`) zu z^|u7XSP^~b1)*61r(}zd!JOny@$KviSp>L|jSR!u*1IgKwId5jmAi2`qe%u+XCTwU z;a62_a~Z}TqDJ?6lje5hblv1f1(6U@kWpc)z|&nRBV*UIieQR{Rru*|$L2SzxtL&| z7abeg@xniYhexYoN6zxY{nI^*xKW0Gz8D~}tE>O4iCkpWn8wt4?S`(Ftv?<8vIvbw z(FFd5`p4~#m<(3uv2+pv7uVC$R(iZuhnxFEY{o}BxPg2nYK zzOjuMR`}t3{8z#zfLXy||4JCt|1nv5VFjS#|JEhRLI>(-;Rh~J7gK{as*K1{IJ%7F zoZnXx&Y54ABfp9q!HDWAJlvFFdSC9}J*llUYXFDN8meEa<0}s z8M~X?%iKLB$*-a}G_$rTh;U{M0vc<}N#PVAE1vQdL#9a-`uH3*cbJZ~u9ag-fny$i z8aCs;3E85mgVK&vWM6}FH9o^WI#G!=%YOB#gT`1^VttnSVf4$YKja@-;zARB-`7v< z*imICw^KX73Gq-go6e?w^os0U0HSxH>60JLWhFbDeGT&Z$d3;9NWy;WvICuoZaKMi z=UvTpLDrtssbhiK&A3EuWf6!)>$sUlRcn5?Pk^OCtvApB=6suN42uKN-Xs7u7EjXh zG|>-1Rp>w1KB%sI*b5dGwFbuHNN=|})sR(dekHBL=>I~l@Nao%H=w0q==`3$zP>!I zmgoBoi7ylm<9Fw6s3&T%wJ%>VQmx(H)!iq?ABhdSzitwHlFNGcBW4sc&9DmTThb^qz`diS`xzQT# zhZff!yj2#rS>yfS5?}{inV5BfcZw zF5uh!Z8b#76;GcBDp7^zWtzQ%J;D}es(iWWWQNA{SvyhO`X8oyNL?j8Afn=x(zHct z7)3c%RKTPAyKS0gwVpGLqR2_%EowBpk>rW}MFfsR9>#2aOL!HKZtg$bAOe+#;;w?3*If zQk=HPWSlX7cF?h1PVE1D>LL{K&Ze4d!#Y2qN+^N-`~RG(O^Gjg~EsZbW^ipD9*+uf$K4Cq=H zxnYj(#+^eUa_1nRDkJJH|9$VB>+n4c)jji1MPz$dV4Ojf;)iYjgw#m+4puPdwgLSj zubNnwfz=z1DqFmy@X!!7D}kTo6yBjVFYT`CisjAgjS^cO%|(B2vzWb5PcrnxTK4xu zm?ZZkCy>+)-K8*)fo5JCWa@}^R!iI}a6OA*S&ibX6V zKk0=}K_M7m$#QEMW=_j=4tDXgH{_l5u?oFF?CXKmk73#~&>ha8CH{7jDKT2WoJ&sW zD1wk_C4Q6m{-YEWeAg*gP5`2Yl>4S@DAbob$M?&Gk2@2%+H*H2wu_)XL3fn{D8ljl zh41$!&_(kR($}4zJj3?zH-A0f2$4;9tH|N9XT48P;?coFH~9`z4S_35{xiUZC4&-3 zo3Yt|ee&RI&qBF zW$mPrwbqtHO$6De21%1=8zUX5=uMV*>#k-H>d5vP zz8OPyI|HLGKn`U2i>k8-dUX}5DJ(|Oy>)cK%QOwU>>~+Wn?bp?yFpx?yE;9q{;DTa$CFGK2S&xDNk$24GuzOgK{np ztsuRfjYmLjvhn$}jK3F_+!AtM`LVw=u&FUIGIU6>0@nqZq~REsb}_1w!VB5-wbS#J zYPBNKKJcnu^LTORcjX|sa8KU?rH5RRhfJ&l7@AtLVi|n8R7-?$+OVx!2BrQCD8{a)Kc#rtcWIC2(YYu=0edjgP9sFpp0=(eKUE2*>jc+n@q? zKTY!?h-S?Ms1kNuRAjowlnTQZF=#1S3XPx<()Wc1>r=QN?#W;6OL z2|Y0fxO0y=?Qi#F4?$+-Qpt&J>-JT?;d6ITN&7R`s4l(v17J7rOD3#Mu@anT`A z88>nZmkgV5o2{_IQ^TOFu9g}ImZrc~3yltx&sdaLvM=bAFpUK=XGx*;5U2#%A{^-G zEpT(GF(}NVJNzn$I*!S`&mA<1j#FEw4`lJ|^Ii?VA+!l%tC)`Q6kS&`LD*!rp)SSZ z!fOJa=BWFG0rWJE<~c2SnT{ykD23&sE?h7iTM20!s3!XMY*WJK_oA3FzU zScKW==wTvjelr=iu2>(0OLprW-Pv$m4wZ7v>;gB4M5m0(gOK>_@aIy}t&Y`H8crZ% zbo1L-*2^hdvzq`~_{<=PT=3jZ#UgMI*bQbOCzf~T53X2F9_QJ+KHwwQCpU%g4AGP z7i4m>KYOFyVXw`L5P#h};Q56X@OHZ-P-1qabm)G~GS>9sP0ToSI#43Q5iDCjG6r<1 zyJZa^U&>SXTW+bvJNB5oHW0xNpCGimZgaFJSb^??Uz1|jbXP-h<65N`CgZYX8jM3^ zSJ2tNSxr8>9)`mMi8nHw1aDz_?+ZRuMO@tou|Q9z11zdD#ka!jZfeXi(bGK&_vVQ^ z?b#6fYLRy70Mb9>3LcE``^rMcoxj~!hvBT%&cQK#L#nhF)C)iw(B$hY1fwak15v#J z-<0Kg=Zh1uk_^yGnO~&Hl|4?14*DFz9!$a(EAbT!5(<}0xUlYlC%`_JfofaWqfWNEfhlbLb2Ds@#m_oKXUJ0 zdSUbdO-BOnM!b2U2o3t3AQ&HGTzjL}LBTpwM2|gf3<(USB~4unKD6^_G>?@N%R2V zE+a}P6(vB@x|W>|ol!d5vws)e>m=0+2Y~#n1%kb=NXlT+^$#v9N z0Lt8wQ#?o)_j$PRavtm~z!aRPQ85^H^}u0bjlfDm(!3xG(oMQY?(DW6m1QdXq-PG; z7jW?rNj(vW&SZZ>B^q=2mU!8NLql4|nTI;pSkw9gbip(A^U<9DVj%Sjd-T0)ldwku z!O)$tFvVGRJnSI!t*v+U;QlSXfMu%J>v5B@Rq<`V$DQ>YTCkc=so?hUx&dda4;A1r z>~5vZ0E0M|B&lv|71*mTuRX`GB3G>9RzF7}+2HIgGrV-?p|bN%&4si|xxb+z1S}F2 zOBQ37uO?>1n_T3UF8nYp?uWnU&+53X|N94hR8WunjZ{}VH({S=x7sRbdLq7vyftJ? z2@;dF{)x|0nI%sYQ|%pe)%r zxP>}6S+ylPH{St~1KGov%?}z^A&&&(B(s+ngv{wKZ_L(*D^+nzoie`$NZ_*#zQ@&T zeLY@LZ5;akVZ}L=Qc=fIphsO^5%YJ0FQWW3*3|ahxk16yr=ZgTqunNMFFko^CZVSh zlk<_(ZLf{~ks&04%zz`tNla=O_`5r6W>d-%mdkEryHLIgIZyrq88$=4=Im4xR_}|) zZ!?V3+6QZ7$+wYJ=>nqKQ2L_gKw%=9`ds2Mdo6`avM-uO$tdP}7Jandkx0}XQhkn# zzq9uFBxvJ^#%sW$s)6J+j5 zXmAN{4mTo60nJnc2C6XtOBsVbJYc5&a0nZ|e?0yj+kThaCezk^Cm!F<|A=cu`uO@u zMai;5H6<@WD$n?-1{?Pzr2mF?F||EI+58#(N9dB2U*+$o$gl7(T>0jTu!?94mCA7^eb%}7cOyZN?nfVx+L$x~x>^tyJj$vmKZOXBKkU?mdopygE`0+rPi zx3F#q)PBC|6M{n@2|m%_24@G{?ql$@S=PPaEh1sG9v zxo35;K!!nAr&^P|c$6z+&vUa@eX|Uw&nednN1SCQSFNx={#kvzFb``4ixf3m zIY=2lKDmS2WGQx#gfP0BOAD4i?UoNdWtRz&Q=#>Y75@;X*z^@rxbLVa`YnIz{oaTE zNGmThd0`N_?*0!a>=f<^TOdF{&|-km!E9iB4IUs0KsvY|y6}%EN>L%XAjjOs+WGAJ z=wAmEmK)JGoI&Uq$`1%&(sh$n^lmT{o9pDd>t(CQ;o9Sr;gFtdZ>-qZg7jbc*P~uh_&U$wOO;{P3h!F3|a}dH-WoGGsXGBvB2c7p<>_CnJAYP}_#gD0t)$ z$Is_In%83bCJkJDij^-Lbnh)JKexs8f3E|dDy=BUEES;}7{*+oxV&iNODhNv#y<$} z=-mY})V@*#j#N6^A*B940E$3$zfmk;3ReX3DO;=d*_(!|f4FL$#0mL1ToWidl)O|S z_mi9mELAQ#S-D7+a2+=an87R;9t|U~1&sgF{`AZ#ZsOL+=sb67R?kPP;SQrDJP#F^ zsr<9}0#5FYl#3;3$mekh_XV=g`LVN$408Oz1ZU^F@kv7gMcyAWTE+yQfcY<&di4?0 z09J)>xHkZoQg!{E*RBSy?JCKOX7n%2$6 z-dzz8T10-8&ZG00yi<2%x`4@L8oj$ZXP|WgZ7E%-(h>@kqIJqt!{ou4J@Anf#HcEw zPSv)TmeUHAmeK2Am3|mkp+~W?)6eVg;c7e2H48x zBw;iPnvFX(a}Y+nn8^W#;6K4qA&N3hg$HYE=n|Dy)1^$6Gxud`0!yZ0d*p;(03ud^ zy^hvb&{_%?^-|c8>2fAn_!5YCX`?Ov6`*x_BAqZdP7`m!E4|c0ttvHBo2}NJT1HQs ze_rYk1e$5HO|)A}>0a7uufbmK{SDV?ndJ&?hXXVWWefy|nb5Neb%C#pK9tl%P-U{v z%DOV=mf@tF5qHo|q4_JBR-PLXOPn6TUrQ#9e83Sw*iIv zU^kn1C|EKWK_mS%Ah;Pks|+@@OxM8{T4o@Zf(mvI z55b=nM5d)6kW5m_Lx%`#@%0J~At8s1=`iJf)}P0CE6_pa-@`H5WIHbP7t4>QJLNX9vAkd8^)UWbAP6$@LZXWxAVbOYkgCYh!Pi4lzTy1%B>Pf9ZYnAH}3- z*{;*nGg_ZWZvV-oB*dF(WQ0^x71UW+hk8Cp_g2sc=tD&+CHpenk8FnaqFX;|TH%e* z9ifj@(1+=xs1s>xxwM`XyvIu)rw0VwCz$GAQ(yL@$J9)4{viA{r49G#c+Z$S3LaiI z8H1fq(Zeb|M4x7oLLr4te=>z$^SG9N2w2ERGL4D=I9HuNqS6>W3ax}f`>ts|P^Zvm z@RHI@6xXbm9v9ry(J7RMY_2a`aPR71XW4B1S$a}He-4?~NS8>v_Z&;WYl>KnqBJ7-hpw*<(4p-DB;Erm4B)LPDS{#kCnL(dCt zzl#E4aVwa$czprcYdPwIDCcme_C!|1U))PSuuI$zk*W(Ap#uWp$Ho58;-{sE*^$YJ zfcvRRKNF?1B4(sbe>9@m?fS5nel8lSJLrFy&YLbuYc7$Di~9RZ6dwe@uT*+bv?gxR zf2UDHLuJLEg$yM9E&WcA_+R7?)37(a^as(%yhwk9vCtzREf&@5r9ab0gl1l{v<@{6 zC3O?M!(VOl{tcWYFh zcWyW`&qG3pOe@HR0(&Pf@bG-DEH=)i05VspTrF}nH!FPJEICoc3S)q%V+;_aFop)l zP;Po#SxD2ff0q4{T+T}wqs1MJ(W0uHR%OPB;l?2?$s`KN)CwvpIWi|N=M^e1V@wxw zhcbE=o-@%8PA~qV;Cea8wH_!IqWp_Sb&NfdNz}9rhH)r2Br^t) zMeQA%TY4kA4{q7j(jMtJ*xS>w>)_TMT^(L-L2JjGxOJj&ZV-)ggVi{5yFFtT>@y74 zJf{=@f2D8cEh09yg6#A&72XCLgRGuD?B$3Jh}mU9;ruBh4ewxD7AzgZW*I&BN(>mh ziz!$}F_R7^NNhzIC6VZOw|xa*NB`8Izi`@_wbT62%UAIpm3#SWG=pW%ix>j~;()!P z=|~#* zs~lrgJ~te{KY{96l8>ex)n>uuGMb%`c#snwpktC*Tn4EfgILng;xZ@8J7YPjGNU7z ziy8fhkvX(Gk4lucz zopwj%<+s`80do~2D`Ae3vs%C2n@KP&f1Tw*W`gvc{0^aDj8k(=qot>B`xmPR?nWM%F_Tp@8f$^zMC-x zxq5eR4y{vI3_c*+I&2E>TUd_fzE&@Pkna^rKrwaahT_Qipb*^GDr(jJ{9!?Jf23IL z(A^If6~w*; z?}1Z(f$4(T18(_hnK5l-&KgXmo>nd-3e?K(mCc5>6~3tQ)BGjdE37LV)Q^&pwQ#S) z&+u1NlKHDJYC|%1Na3%+nyEu^jPYK6&d&RoKPnRF@-yfpj11b3Z`tb@e>%>eq_``W zHjyW%v=QIIjMQf2l5wjwh-GwmTwut$YYW7S)B^oRCLq)v5C#Y+jB#TgxNhmo8p)ig z+m?O7x>V%vtNgs^JCwARHbhpo8tiRe{t^FJ)aIYKNc@@Cy2(NO%_oXe2h_a_mDEVt zmb7j{8H0tCIim0{RsMyjf5xg%)u5J6>nIZ!1*crg#_ZLsWwQbZRQGHCjX?b^(~`4- z%8a=}HZ#K!NGa0IY^23L=>CEKsPgamPfQ#BAATw`rjrHMokCmE$m&;$>$>FdWOl&m z)`l3}takOU{5O^V!Y`N18@mT#Hk8i4BUNORx;`YLf13b*mCvaBe-8<>i!%lf^-2;U z9Xu^Lie6DxK3T%#A{V~ncqJJ#j^vgU*fE*tQzR9Izl^818it9apbd#{E7lZ_VRf}E zc~xnS$S$5Fa)vkpeqLJ|acM0jlw*p5vTxcoxin9j54VyQ6lcuBR|hLNBB)YOqvR9U z!GXe8h=^BOD85uIf0M*0GA*2n7=9$tiDqrej<}AS5rg&?cv&o6pi1XUOT5%!|GH4f zvaj?*$t>7b&`TGoQk8_MWDe?v2r}Dt(=V&+RUEinS|JRG@uWH{KKj7Hj+!Oxo*$h3 zJSiyE3UmxBOJT8wLQ9;~a_QJ0+H$+Y7xq%5dSM}87BbO_f7fWu3%N;ZkQ#*^Fy;8l z+=R>08U>@C^*y3XHwO(!x~UB1eKROeJu9R4i#yRqn*t8KOlnf8LRwpLV^InvOY4y& z6Y0aoAta#nWk$@|ua--OGHHW!xhjPv3`wq-h()h-g$Rf$X%kb&Wa>o&%jl;Juf;h@YL`0DJV={S3<~|Q zxVKlNt>PnLnaimuw=2>%bOF+Krp5q#4}8Z1N3?_qAS?S%)arm{Ww3y0Sj8X=>X^3N zqTq|)7_lk>iEJQee_T8ouuaPZ z`ZGo<5HsR>A7m?9YOlD%ISXt11#1V2EoPx>=owC%+R@3XD;+F;=(T8c8;0RJ zTsm&wf4E6n@v_B&nSvZcHW#06QG>Wc4M@NZjXq_R6tyGE%uPgmQ2BjdC;x_^K7e<&Sro+Qon7}Z6ij>=e%vr_NLQ=+o& zBpJok>#>>@t9yzoIjkHJE78hf09L;KB)w^jj*Zi;(XexzZjXje(A)F$&QZE+l#Y+n z`=Vi2$nPAb_di1SF@@cJ_apQ%rsI6t?-IX1$@BzBhvht-IL`O`<;uJelNOBA7;pvZ zfB49mXR!WQo}M^PexS)v&gcE|!8|>kr>}-xBWE7K{@1Mi2C+ZCIZxkg5`fhJ{k9ES z?Q&jg{rY^Kz9*250O|V{Qa~U%CqezPdlGEt!}O!OX%T>bVgb8HsA8Oc79FMkJ{1BQ zAj1lz_A7b%#c`?Pf$=T5(=0B&}8~QNxNwRw*HCGxKs7 zAbuqb0wZTm!A@E!voDKNVzcs90B98$d1mpu$?pVH>>OjYdz|h7=c8OvnalIse-rG> z^TJ7MQ)h{-eY_~oi=$1-J+wg3^YM~AU$kfB%yWKA6u<1KR)jRN^V))`t?f_yozaju za%E*q=!xg(Q{=;$gM(CgBtI%caf_(Rsq{@aD+#S}=pC z86ka~*GGN4VU#aFW&hkLem=}?e|vn~F~*%Z>oir1(1J)V;P~B;pF%#~KE~a%?9Q`R zT%aOCGZYoCbw1uX$~|Kog$!cB?q~!dDf0Qo*L&^G+IB- z%c7$kALW4)e5h-jQveUupWrMkF~&y@j`9uT{Dx>3B5#~;1W8xjD8D&0f6BK2KH7bP zZxi%s6BzdKTl4((Xp?-8aO}B$ceSl^VLKn+QQT7@lRQFm{BB3JY*{801(`8^XP)m0 zD?Wbj7{5On_W1Gh19`qL&mS4*kHL?eO-i0WS*?JlPt9MR=TBSiCFAu3oJ*WezdvZZ zSy&eKQ%>+G2tl=09#H+Rf3Rl+Zi1CZ#ESIpy09nYSNtA9DI^G;;Ll9Z5|JT@L8pS6 z=LDaMhSef9kKYv$QmRE_E9?E9x+#R7EG1O<>7Jl@f=`e0)6s|@lKP$XQ0bTR{H&FQ zqg^6St}cX+CEqrS#MdXVu^sKs^EdCN)gfU|nuEu;t&|cN=jWpWf4BaikH05EkAG0a z`{60><}kwSr&av3l#hRYOk3;XuMV}FV=&DU*-9CmLvT+ z+WizQMWlnqEBL#Bo<24v@d&Bg{c`sRFGPy!hJDXGw0(p%#G{63F=LblwcdY3eAs2Vm zpQhd8QdM++1Q6AEX;GK+F4-R9ZGBt;ETo9?DCrv0D+1IDFD2JwEAD ztgpk0jFnYAjJJ(@@>0vEgx;*>?T$KtwXGVHwg{EYV4k~Ae-(8Mq(-WYZ0p$a#PooH1&29;1t$_t9$S2(58GNS8RjOP4xdqRX7GP!mS( zwXWr~Th0}t^{$I4?CPWqt{rr_D@Dz&!?e*gOjo$xOPgE|Qj5EaTHR}@&3zZOyYHqB z_w%$_-a=dCx6@YnYt$*fK-=U$L01^rp)ZLX{|8V@2MEVi07E4e007D}b)$q0%WLwQzAecs$;-Nd zASxmv2qLK4kS~#nq5^hlp^Wh%1BQZAKtXf}4pBfw6cmwp&P}qWT{hR>FFo(vkMniU z{hxF9eEi_U02Ygt0^2UTZ1s{$s=JNge?~JFs`gh0d#dZJgLbsfiWrV%$9z#cWYT!t zjF?8kq{&_*;S2Vf!HtPzG*RvEF(L`GzPc~$iyD1Ci)C~-H!lhd7@Lg7h!G1np548{3_1!t0yE`k(y=0q zK|2;q#^YwpX>6fwMt8(ipwh-oMr2;Z4jPg3t-iFjiEVP5Wj8W^l0Y%930Vneg%uYl z%W`q6JIRq+8;=~^6f>R1wX0ice^UuBBdtAFI2o4_6~UJ^kg?F#!|# zYr2j}n9N@@1>7~fuMD#_D5w%BpwLtNrqnEG8-Ir6ou2E2f_VZH!ltvzf8c{mpVs8; z#;m70j=`}S=A%Yn>Zr&LhjZ?R7!(;@XXOpGy-LRkte_4{1m@;F!7*B7==^LD=cSdP zjHE!>@hvj2=j%8b%Xsz_e=^rfuoNB3(?h2TOd@BOcPH#f(lJ*VPOpv?Y41)Ks62d1 zDEI_jNFx|D6O@q)DJR1``t~a28pcUU-Hb zr2w4G3E7TSV_>3VOTsau3RY9(%sAca@`GltA}bxT)ik1H!5XYBe?kY&r90kZSdnDh zJd5IBgehf8^CirA2(Y&E2`TajRIr|su8#*Igb3yNQi%@vQ|Qug0WPFt3=sf32k5POw*CcHVT&e?km<5rfT#*GFEMn@M&;M?CEXnO;5$&MkH%LTOA|6AF?7MP{_m z+0sTkD8^Y27Oe4f``K{+ti76n(*d037~VYDfUe=5dU+nO0CJFdc)it$BU zO%5G8uizR=3aYQ|=4MC7SFo%Y*Wx+?$Cw=WD(3RQ4HU_UDH>}?$Qz?#n3%XpD7%RuqWbW)B70MGJctpNfASD{o7H++vZu$4o1xXFA?ww{ zbWYj1)>vOM11H((N3yjpV{pzA1&`%9C|O8;qTz8oAyBw>%}U=A6;BG(jxNlRaoAGy zw1!8qhjHlOwzNr^`JZaog`d$CAt|9Y>il#($06H=pOe~P#7@x2FSr@lgz zs*2f8e^n2IOcmXU-YNne%Gnnv>GNc2HZc_ZisGIydd#(P!m?R4 zivLigs3CR?D@I^FJ=eFEUL)RNUX(Or!8C~c7a#Nf0~EDxE0#HPRnWs=+UPC{6t^VV zf1XabIi-5(-Jyy?!mSgUnpB~XV_Ytcm>sjoUU_Xrk!*W}#(=%bsJCjxKxz05sY_ z@G}Yk3Dc=EH=Dtv!#Ajku0+&I@M|%_fIyc`EM&DL*fHD9e%b4a#j?E+)M{6be`;Ty zj5$`+JbiP}?32xoXwpP8m%f=<^e{tJxy7oghoq4Pa<`(&N{~HO^qjLoRa7tJT!Sk7 zSsgN9G|@;e$Q&I@$3Q{O#Il^uu=VVmiBk!-Mt8Jk<70+$)=(E;&_XY3YUUYE+mq35 zGroo+M7UH)O&>)Tg_BG8Jq8ffe>0TcVv^EJOj3He0dUd!GEAWt_X^@_X}^c)tlGf( z_1=OVsHoe4Y4tl$>Dz%B-ohQ2HH10$f&WTSjk)Q4h1*FdNq1jYJA(Ovw%S2VOJTtX z>H@W0L#UVR!W51#ZKi)IoH&G~gQ!g5)U9Z$OQB^e8fZ@i{VD?~tQIWX*I2w);@?C{sP+OFC4_IfZtP}LT~3FqJG8Qta_S@ zd{Vkvu5N`^@ADRYnG%9GerFINTpiWH}CfKwRa=su8@xYMtWNUdJgtNAiV;Y+Vvf0(n9&Vd3lf?a|2 zyyMZp2p%U3hp@Z!sUbWwglALO>sM2F-mChR0km_#io86qt3HtRNa-qlkvtm4D=F+N z{ry3=vh!+J>Fd(tHxEt;zf#bwmKV7$3^W(rBK+m*wvRirDL}s&QrJB?i6Atd4)_cB zfJ^^8jKAEEf28nXf9Xdl4z_0iFG!aQePzN$eu?%GQ4sL##QTAOx3DYVE)$-Pf-<3Y z6gGQOqPX1C)iER{rbH=aO-fALiUh}@oulAayfieU^rNVS(J z)mTl^2~@tAe^!b)l2(foB|TZJmNY8*#H->Iagn%6(yPU_l3p*iOM0^ymh>U9SJJ)W zd9fc5FN&8WzhAt?)OC&PM)w4HMnSamqf#jJo|Dn53@=S?$ zm$)mKmy~z{%+m=xH=vS$SKv$n;7+))4h8h&FQj*-2UijZ-vAYN5vYCyO)N(-fvhgV zm>{B<=vszJt~HqKx&S4vAWB_fl({a&6!&VByDvb6JBX?7UQBaugx76LJ#Go~?*9Q$ zO9u!}1dt)a<&)icU4Pq312GVW|5&xPuGV_G@op77bzQ0`Ma3II6cj;0@G{*_x6$l@ zWLq!9K8SDOg$Q2w06vsBTNM!*$jtot=1)l8KVIJeY+_#EvERRF+`CN~+)~_fcio`v z*4!Y8Ql(|4lGuxq7O`$fleEN}9cjIwL&2@>M%LYJOKqvn8>I&WVJ`e@>#4mHnuhzUW>Zd%6?zt$4SI~lcxhl zC4TO|$3j~w-G4Q7M%K!ZiRsf{m&+`_EmNcWDpuKnz~ahZga7dAl|W%-^~!;R$uf$l zI4EIk3?ryIC}TXYW(0;0`IS)TrpP}tglbN4Rm~aBg2TZCuXEfjpuhoC)~>H#Ftz@S z>Dn`9pMU{c7+4fO0Z>Z^2t=Mc0&4*P0OtV!08mQ<1d~V*7L&|-M}HA1L$(|qvP}`9 z6jDcE$(EPEf?NsMWp)>mXxB>G$Z3wYX%eT2l*V%1)^uAZjamt$qeSWzyLHo~Y15=< z+Qx3$rdOKYhok&&0FWRF%4wrdA7*Ff&CHwk{`bE(eC0czzD`8jMNZJgbLWP4J>EL1 zrBCT*rZv%;&bG!{(|=Ze!pLc^VVUu~mC-S7>p5L>bWDzGPCPxXr%ySBywjS7eiGK;*?i?^3SIg!6H8!T(g4QQ%tWV0x-GTxc>x`MRw2YvQwFLXi(-2*! zpH1fqj&WM*)ss%^jQh*xx>$V^%w2Z&j!JV31wR!8-t%AmCUa;)Y-AU<8!|LS2%021Y5tmW3yZsi6 zH<#N!hAI1YOn3Won&Sv+4!2kBB?os0>2|tcxyat=z9bOEGV>NELSSm<+>3@EO`so2dTfRpG`DsAVrtljgQiju@ zLi;Ew$mLtxrwweRuSZebVg~sWWptaT7 z4VV)J7hC9B-cNaEhxy8v@MbAw(nN(FFn>3184{8gUtj=V_*gGP(WQby4xL6c6(%y8 z3!VL#8W`a1&e9}n@)*R^Im^+5^aGq99C`xc8L2Ne1WWY>>Fx9mmi@ts)>Sv|Ef~2B zXN7kvbe@6II43cH)FLy+yI?xkdQd-GTC)hTvjO{VdXGXsOz-7Xj=I4e57Lj&0e_C+ zAH@(u#l-zKg!>k+E-Qjf-cLWyx_m%Td}$9YvGPN_@+qVd*Q)5cI$TrLpP-Mh>_<6k zysd!BC`cEXVf*Q0Y(UgdE^PYo5;;FDXeF@IGwN8mf~#|e4$?Ec!zTJEQCEM2VQr*k z8Kzplz+)oH5+-jyAK;GP8!A zSKV>V#gDFTsa`xXt|1Uc3i&PSgl%D=JEwjW^F5vD0l6G!z|~>y03#T)?a;@!*(vAwmBFr?|-8vt&)jK z!?QG5DNz%WTH4H>vbUDpIEl_O19mVOmP_8bVz-kCsYEtX_1Ovb zj+KS444hDHKJfNHwq&hQ29#QGU>;3P1P+D_kVfmXiA~y=y{YGCGep{s6iwTA*ge*SZSH9K;{Gc1^NWT z@{>XOdHMwf#oVVr5e4%x1I%+r&CEE*Qu8V$tmu5mm?%|OR}{L++~wCzm$RIp(7a-4 zuUW|Jw)8G^n5G$)e{tS^RU&@6hKR!RWWQzWdvkgoyCMKT%caX_=zlus#?;Tc<%xwM zJewbXg?^RAe+_wMk=A>m=A@r~0~#Z6hmh`q^b!Z`=jde+%aR2&hxQ>`<7bXmDk+!% ze+$*7qh)2_^In4P`ktr>O8z!|UZGd$clcz~c=h>Hr~z=--z_oAmq3RVC-fGwS&sJu z1-B|M{Jx;us@*hy_J0o)`U?9cH0RlBfikrIP@yl=AE9!T32=5+P-i$<+jN!7%+FG| z&!5nrvTOegUa57UpZ*+hJA>p2ga0MxsK21E^Uo8!3b{#gdjViLw zDj?{%qL2b=fc}>G8S&udSPszN3la#if5csvd~EsYTU;zzV}C*VHpkOH)4w1W41*h( zbOQ8mmEBsPEo@ObLg z93$OR0O5mpOQ~kA@~zx=sm%~6;&yQdTLO>ECg3w&$V;K3Rxm$Mx#E3$#)AP`Y5ET>GF+K7Ons=3AJy$clM99)e@XPVK;DaXeI#{!nwqZB>eS#gwM4Gc z+UQjZ#jeu&%Mv~fw1GC37KsP2q#o_EXrxGY9xc+Ai=@m@d~k~Hixz2HYVc*MpSt<2 z$TixLN>0<8uJ7@5d0V_2pQVkF7Vq{{!dIm33#3Ft_}G2)yjM)!d^I{4d6C{M=mM$U zf6tOXHRy?rH1$Si=)u8jv@ewuk!jjLMIV6_5a7L3EjF@9Y$D=$k&f1(*4c#dO{r8e z(v+H}hoI~Q3P)vOmA?n#aMPBi8^%0|sj#w@`5rIzh zQ!tSbr|=trz3XA)gH(s7qlZqzSnr3Gf1k$a6s-R${PJy>^CsjPC{3BNQR^|!p8G=V zW%6Eb%Fa-3=o*=+gf}`(Z);pdp9v&gz7C z*}oPKd5d(eNI!)2=dpg8p7eD2T72>A&r(Oc#kZr8Zl0T=_oWh8{A0N9vXFPxf7T*> z@F=#&(1(wn_rW1wit#=dQbR@h$qP^^nkv#IIQ!Y8pN*0_p744iBi`tUFE&yiA8GoT zkhf%^=TflG&)tw(+<*mIXdUgu%{CxCbK8#JowN2@0SO=M^#R!H6?`{v`CUe5FJ?Sw zyCTwGaWuckZrbd*cS97n*}$HSe?&KIhht~x@pz>vsk20GwyCM?#|=m*99Q+xzrHv4AaMp^qVvE1qqxlUZ9nHsoy&~b@Pi; zbSxIXMqg&hucX*B)AZGlZ<_wNNMB2M8@&ts^)Xsm@z<+UH@_KAm7Vk&fBsM1e8*q} zC%twfR;0hW%s)2}p$g))S6XPbY}b-1+g56mZJ4@bdpGTo?Oxg^+aw*3?Jyme?QuE* z>k?^{mF+lLvMtd2WXr!S_d)uoY)gJo;16IEvvuH(Z&YlEF~4MtgVERw{mtdnP$YGQ zLX5QNiKcH()87Fhz);gaf8Zxp{{AQY07^yr*Rp8*MAN@Z(f^s9xq-6?{;3ChGh2NJ z5h72l13;O%#FbbiB|~{IS`?nriNJPIz>*(s7WJjAq^m9+Eguv+(JTTuX-2FlipGi# z>xbCfU@qZdcZ!5pBz#h2ErNo*n((t*0g$h4ur7sb6@-iGc#L$?z0#Uu)Xh){P%^cBVZ7wOS8%9=n+@X6!d z0j(RK8a`Hw2l5S1eVl@8los!kPhF(7@ijcCcL%PBB!<=~MKK)m$2=`T0Eu_#R=NXI zH=h{{`4iqLa>{Mue;U1>Y8Hp4#o-&#kU!*$UlB)|#anUx3hcmxfhe0Q0&^ZadKv7! zbC8#@-C);d@h~h3LJ*D3;sie9@`|I)B2%(-WLk{fsNVS{3NYNyg}nR)ue=tyK_MEW zlVVgDvV8=;&C^-g=a&0t>2a|ceQr0P|8{y#_POQ$^YjVXUgwtkpQOvO&n@>kdb!Un z_g|vV%RaZ<|2lm`_POQ$>nH%Z&n^1GBO19cTkgk1x9oGv{j_*W>RF15CZPW_^!Tj4^T{T!k9N#2;RO7iBy{i;&QUo$Tz+ znfE#GOwP=ozrTJ1Sc55We021t`blp}YoGj;%5y1uf!uNG{2U zc(N@c!)lX%wI3y3q;Kp>H=-52V;i3A7>>%(TwkwPYfo4kR?qm|#C16kwWU$vA^EoB z6NQd%bM%nHh`l&oU46V-HClA2e;$PpNH>BcwCIK7lE8cr+NK@KmP_V`PLn)Sf8 zDbz3|Fu5lWrRhrFHeWUO$ci zK|;QNMYU4B-{xxq=2gh0MJ_>CzIO%I2C`dQ0}U%zLwzhCD9eXj_~Pck%ya+e`Xnf; z1j}62O+JMJ**YJ(mx~=JE+{p9z;saHl6M^@O>uaJ(zL_pbbfg95AEkMI{P zQrP_-wu~WeK)#DjC~RTz1jWl>>J%&u_A8uVH0UJwtHj+O|MgSsVS$&sSO#aG3~yMr6^X${<>0 zQle|Lj@}|34Nrzqkl>m>`@k4<9*UKfc&#)tI4W!!rdA{x!$&L15^Z=Vs_fD^%wvtV z4GjkS3$YfV7A6gE;|0p94J`((b7fR@!QilW^Ak`-SZ_W1@A@+aUavpvf)AYzv|)!q z4VaP^lJwjZ|A#8&wqkPDwLy5?V^3lqxn2iXkLKsKp3v z)lw?h02Q#9dcl*)Nir~*8P80hEVZkB@JF-{`qDZ}%ic=6I zm%FuV~79YG9K?LnO!Z^jy-SC}sEQ=yjZJve> zhLEVZ{w5(ZoQbyviJ%i_b(}#LLsvu9$Wy~P3VYSGP5*j5?A-{?qgO|N4=ynDG-o(t zyH$VDmx5O`yrrVG6j*nCTSp%*G6XD#7Z}brjGFxGwwDl7VfqSEf=l#B~g+q=IW=b5Z!M<&ucX9YRuprWo1}sWhaiRi-Z__Z`V_?vU@yo}2(i zFdD}DxXjRbRIlL*gGOwBofG%{2tGu67-Ps#wKfT;#rvpD6d}xUOenjnl!5P12Z*7q zw!2cYy^fD{X!wL7>>Y4wID{LA*tcu0;U>}9^SSiBWz#PcPvS>06_ak^GaXZyW_ZJ^ z=DocXy5lp)=I}XgE9)%v+M=maz{HH12<9-a6nE%cQa3OVKU(g8u^m{zqPmtPawHNk zWR7wCpHO$PtcdUx!|AF`o4_oZJa38m07T<0{69Jm_wcovhi@1zG{6_Cwr^I%)O|y^ zYO*wZw@?12&fKV)RzYoo?-}~1q;zC-qb%&GVmhg#?!i<=i!>0|LdgHijnpTlpo4>E zJ*c*hO|z2vk8U1+%7RKMp{yWG^+$Y3922QYvQ(DNhU(N_cuU6$Dzv>0=5xNOeup?c zNo$t6oTaTgSFPlQTvG0VOE^gcRX<`ALi8~FK&RITk_PxKQN!sc(4M3F**1D|x$G9+ z+(ut+b|{%kY$001J2kwwjltaQEs*i>3w*#Zn|y(f7#?GPoIb8Gtu3 z6l++mVQpv&_A5%Vi@5j`T=XJZe@D@ehm?9h2I}XB_@(}4kR&~YHrm3(cAUT?`X&;S z^aR@e0Z>Z|2MApz`fv6F008!r5R-0yTcB1zlqZ!0#k7KfkdSS=y&hcen!76`8u=i8 z2484mW8w=xfFH^@+q=`!9=6HN?9Tr;yF0V{>-UeJ0FZ%A0-r7~^SKXVk(SPwS{9eZ zQbn8-OIociE7X)VHCfZj4Ci&GFlsOiR;iIJRaxoGXw(dGxk43#&53m>S)=uTq|9>^ zv)ObhvxHhb=kS$=qTqy4rO7l7nJURDW4f$LID5`?1J}a&-2B3PE?H*h;zu740{(*5 z&`a#OtS|ymO_x%VPRj~QUFfu4XL{-O9v0OB=uyFEst^tz2VT!z4g<2#lRmMJ`j5ZM7xZ*AM>%2rvSpe(=Ig+{%mm`qu9D$$nuwfAVtg)wU1D1@Oa-0qBDX0)tL}srdd3AKVr| zu!4652w2`d0fsD36d(v8?%fw448z=eKw!vV=GK+cg<@B0$2aAJ0j^IF7?!T;tpbe1 z;%>zpHr&Lcv2JbrpgXly(as#!?0ARvZ(9Tyw9dPLBI6nnUO(iIoc8&R_JI|#ma!w& zAcT?E9qq-QVS__Pcf=Ea+u?_rKX*`?w+8~YR^5P4}7sOkF z9^v<)Wd+*~+BRU@A=_f}TNYc7Hi#bHH2iMhXaTblw9&-j;qmcz7z^KOLL_{r36tEL z;@)&98f?OhrwP%oz<(i#LEKIdh93L_^e1MUFzdwUAZf=#X!!zWeTi=n`C^CXA?1cg z9Q>gxKI!0TcYM;pGp_iegD<(`iw>T3#itznkvl%+;5k=(+QA>Y9v3?#|5p?&G^NcjljeZ~g^f18y^%J9)Cd^>|=NijQzL5oim< zlYvkmuB9`wBAK$LhSPsqg44Xt6)qW^7KbGx93STK5hI&60&Pi2F?cADNrlr=CM*jZ zLoF@q;~O@SuHKr*C$ow|6UMLxJIZx~e9?Ss^Ty`ZaDtBpPPoAs zJW(yH$N4T<;S2#yPeoF?lu&qNOqVhlu1EGea_2aYXH89ap^|@L(Gh7>iYStriu4X0 z;c?T2YBH74HPSR?ZZItAvUReitVH^z=C?2`C}=rO7dV=-77=68sE%uDQcf{6cFi77 zhpm&o07Yne+0~cxtd5_*)sP&)@HC}ize=e%9 z#0xj(imzo}crbrYe63*c7RTYjDhiU1%Z6##t_Qui5BGbp8h+wH(WFEnJTC%R=pic) zGR)Vxl-NNqUE8ZG40R2ST?P81rl{~1FV5^e_8Pg(x$FW_6(mpMLKFJ(*W5>({#DW*Q zoCKbj>CJyx?{us_MShE|Mu(*hn_8mTv>ROv%chy0TJ@sGvER$E`JN~loQ0D;f|Gu7 zWz6bozzKCPos?s8CQ8kPJJs7yy@Vnhlrv7zVopqhG;I`3KjYvJ7U3Q84o~47P9z6E zG=+Dj6AqqAR72W5+#J*NkpVf)wXA6$(M~T?7#4pzGDBrUrkr3p#=R| z)ud>4j>mb%X;#lOggUgWlJKjV=@*U0pX+Y^LM!$sbuI0$Ut`oayK%Cl!#hQF;YI3S zNlkxGOJ@1oTeu+m*V=%8d-n8%+f;C_H)8o;-_FbP`qm5+m$!#sUS3~az?6UCnEncp zrIoW1GYikZ3^9(J+*73a_E2=I+@yTZzO&nHEt<<$te&=8HKwBfgjml-JG}$lI=92@ z4z$bd>F@tEaq6laA2^*uV=f+<_SYxIZ2lu1)15Avq4jrv%t_4M85a1jrdBbg?&OBO z?w|X;yr%s=o>F|n{!ss|&@a-Ga?>Xp`Tt1WnzOgFxn}QvF`pdqH+A0O6M<{R?*8aI zm|Fe9w=3;hq}hV*9V%VFm_Nouyj`+eMRi@5yyP88PxBQT&vbZ!!)Ky@-W>G*(aL2R zRrh*#Vd#O=-{*82{_t)2Q0>X_c9z?Dty^;DE4*(gK1oaCZ038&qGr3{1N+o{&GW)S zR_RrFeoeXT93w9WTJ=k2WmwRsyZJjz~raN31L?*7OZAKosxIC_$obw$Vto-F(G};KG84}n`sf{TwU%2wY3la+hh1Mo zOk8XAThu>BWiTy&7qj>ZQ^xVsJ)L}CZf)Xc&#mN8-WF1DX4>(>Q`45ejQ0=-ZM4zk z5L6XanSS@s%!u+}4U5KdXED2N1@ELz7MFYE%Vl0?GTZp&z)8j5fxVV0(M{Jk-YLI# zD7^e3@2_*4y-s~w)iFmb?A6PWbS|JU~kQ>A{z z<#_KpR{ZVn&J%Zz?8+_T3iQ3CX&uXK`8Ms6*u@`B+O_xJ&pYz;K_cUp%GV7lwA_XQ7h?=EiYO%jA1g4LkyE%H;C7 zPBKh~SnewUyI}=DY{&pStppCf@lAGIC^PvppTgt~O9f-}d3G+pn zHcEm8XU#X20bkb$bjx(06{tEH6~T)57MRE&F1=%5uthQcpfXUA=H!#g@?du$?pR}B zus~7Bs}5H9dx4fr4CvY|pq0)*@1y!kP7|oePX>Iq6EG0Z0Tmgcm@-Wp?51-IwPcVl z;ju?iv_==K$b6Bx4B|cu^pKur092#|ys(EK0ARQEYY^^{l%|QCuAjeEkp14?q>9h4@!6nkbbJ&fg5yu+?X8=+3#!VJj5-STn zB^PM!VxULuP~>AB87AvHdVm8Jad0aGgFcF?DbAA>SBOrobXEl`gda@_j7wDOI$XgD zA?Lm7ffXYk=VyXqs+K2Iu@*=nEBNf4$p*_rnW}xj5^+A_U=u*+w%i1|eiP93x+o@C zhJh7Ihbe;@`y&KjUXYgX_u)8xbzqD+z9U^n!xP?doXqyT+|nlWGZ zf)zbpp(6wDM6oe2=%E;$(+^UFIrO3?4Q`17gDC*02i4ujCr@1I$qFe_?ym&yj++j) Rh /dev/null && printf '%s\n' "$PWD" ) || exit +APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s +' "$PWD" ) || exit # Use the maximum available, or set MAX_FD != -1 to use that value. MAX_FD=maximum @@ -114,7 +115,7 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar # Determine the Java command to use to start the JVM. @@ -205,7 +206,7 @@ fi DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' # Collect all arguments for the java command: -# * DEFAULT_JVM_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, +# * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments, # and any embedded shellness will be escaped. # * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be # treated as '${Hostname}' itself on the command line. @@ -213,7 +214,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ -classpath "$CLASSPATH" \ - -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ + org.gradle.wrapper.GradleWrapperMain \ "$@" # Stop when "xargs" is not available. diff --git a/gradlew.bat b/gradlew.bat index 5eed7ee8452..9b42019c791 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -70,11 +70,11 @@ goto fail :execute @rem Setup the command line -set CLASSPATH= +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* :end @rem End local scope for the variables with windows NT shell From 07a231b6e0eac07e829509994df8049570d4f65c Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:02:03 -0700 Subject: [PATCH 36/44] Release notes for 9.1.5 (#18248) (#18267) * Update release notes for 9.1.5 * finalize release notes * Update docs/release-notes/index.md * Update docs/release-notes/index.md * Update docs/release-notes/index.md --------- (cherry picked from commit f861bb4aa23f86c2d0b0b712faad3ba9f87afee2) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: donoghuc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/release-notes/index.md | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index ad0bb983932..bb763ddc385 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -21,6 +21,28 @@ To check for security updates, go to [Security announcements for the Elastic sta % ### Fixes [logstash-next-fixes] % * +## 9.1.5 [logstash-9.1.5-release-notes] + +No user-facing changes in Logstash core. + +### Plugins [logstash-plugin-9.1.5-changes] + +**Elasticsearch Filter - 4.3.1** + +* Added support for encoded and non-encoded api-key formats on plugin configuration [#203](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/203) + +**Elasticsearch Input - 5.2.1** + +* Added support for encoded and non-encoded api-key formats on plugin configuration [#237](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/237) + +**Jdbc Integration - 5.6.1** + +* Fixes an issue where the `jdbc_static` filter's throughput was artificially limited to 4 concurrent queries, causing the plugin to become a bottleneck in pipelines with more than 4 workers. Each instance of the plugin is now limited to 16 concurrent queries, with increased timeouts to eliminate enrichment failures. [#187](https://github.com/logstash-plugins/logstash-integration-jdbc/pull/187) + +**Elasticsearch Output - 12.0.7** + +* Support both, encoded and non encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223) + ## 9.1.4 [logstash-9.1.4-release-notes] ### Features and enhancements [logstash-9.1.4-features-enhancements] @@ -387,4 +409,4 @@ Check out the [security advisory](https://discuss.elastic.co/c/announcements/sec **Tcp Output - 7.0.0** -* Remove deprecated SSL settings [#58](https://github.com/logstash-plugins/logstash-output-tcp/pull/58) +* Remove deprecated SSL settings [#58](https://github.com/logstash-plugins/logstash-output-tcp/pull/58) \ No newline at end of file From 35fd6ecccb5c194323cb60bf6647efd4f513dca4 Mon Sep 17 00:00:00 2001 From: "mergify[bot]" <37929162+mergify[bot]@users.noreply.github.com> Date: Mon, 6 Oct 2025 15:17:05 -0700 Subject: [PATCH 37/44] [main] (backport #18247) Release notes for 9.0.8 (#18270) * Release notes for 9.0.8 (#18247) * Update release notes for 9.0.8 * finalize release notes * Update docs/release-notes/index.md Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --------- Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: donoghuc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> (cherry picked from commit 82ff1f7681884f0178dbc10178075d4d239e0e36) * fix merge conflicts --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Co-authored-by: donoghuc Co-authored-by: Karen Metts <35154725+karenzone@users.noreply.github.com> --- docs/release-notes/index.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/docs/release-notes/index.md b/docs/release-notes/index.md index bb763ddc385..b9d8ee463aa 100644 --- a/docs/release-notes/index.md +++ b/docs/release-notes/index.md @@ -199,6 +199,16 @@ The Elasticsearch Input now provides [support](https://github.com/logstash-plugi **Tcp Output - 7.0.1** * Call connection check after connect [#61](https://github.com/logstash-plugins/logstash-output-tcp/pull/61) +## 9.0.8 [logstash-9.0.8-release-notes] + +No user-facing changes in Logstash core. + +### Plugins [logstash-plugin-9.0.8-changes] + +**Elasticsearch Output - 12.0.7** + +* Support both, encoded and non-encoded api-key formats on plugin configuration [#1223](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1223) + ## 9.0.7 [logstash-9.0.7-release-notes] From db594c0888c8a248927f2b44f6b42891bc9b4853 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Tue, 7 Oct 2025 08:59:44 +0200 Subject: [PATCH 38/44] ci: remove sonarqube (#18273) this service will be decommissioned shortly --- .buildkite/aarch64_pipeline.yml | 3 --- .buildkite/pull_request_pipeline.yml | 25 ------------------- .../jdk-matrix-tests/generate-steps.py | 1 - ci/unit_tests.sh | 10 +------- logstash-core/build.gradle | 8 ------ sonar-project.properties | 12 --------- 6 files changed, 1 insertion(+), 58 deletions(-) delete mode 100644 sonar-project.properties diff --git a/.buildkite/aarch64_pipeline.yml b/.buildkite/aarch64_pipeline.yml index 3ab0e7ce1e3..e99b8383755 100644 --- a/.buildkite/aarch64_pipeline.yml +++ b/.buildkite/aarch64_pipeline.yml @@ -23,9 +23,6 @@ steps: - label: ":java: Java unit tests" key: "java-unit-tests" - env: - # https://github.com/elastic/logstash/pull/15486 for background - ENABLE_SONARQUBE: "false" command: | set -euo pipefail diff --git a/.buildkite/pull_request_pipeline.yml b/.buildkite/pull_request_pipeline.yml index 19e1a9f9255..b5f62f995e9 100644 --- a/.buildkite/pull_request_pipeline.yml +++ b/.buildkite/pull_request_pipeline.yml @@ -88,8 +88,6 @@ steps: retry: automatic: - limit: 3 - env: - ENABLE_SONARQUBE: true command: | set -euo pipefail source .buildkite/scripts/common/container-agent.sh @@ -110,8 +108,6 @@ steps: retry: automatic: - limit: 3 - env: - ENABLE_SONARQUBE: true command: | set -euo pipefail @@ -122,27 +118,6 @@ steps: - "**/jacocoTestReport.xml" - "**/build/classes/**/*.*" - - label: ":sonarqube: Continuous Code Inspection" - if: | - build.pull_request.id != null || - build.branch == "main" || - build.branch =~ /^[0-9]+\.[0-9]+\$/ - env: - VAULT_SONAR_TOKEN_PATH: "kv/ci-shared/platform-ingest/elastic/logstash/sonar-analyze-token" - agents: - image: "docker.elastic.co/cloud-ci/sonarqube/buildkite-scanner:latest" - command: - - "buildkite-agent artifact download --step ruby-unit-tests coverage/coverage.json ." - - "buildkite-agent artifact download --step java-unit-tests **/jacocoTestReport.xml ." - - "buildkite-agent artifact download --step java-unit-tests **/build/classes/**/*.* ." - - "/scan-source-code.sh" - depends_on: - - "ruby-unit-tests" - - "java-unit-tests" - retry: - manual: - allowed: true - - label: "Observability SRE container smoke test" key: "observability-sre-container-smoke-test" agents: diff --git a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py index cbb4900354b..f8e50149a38 100644 --- a/.buildkite/scripts/jdk-matrix-tests/generate-steps.py +++ b/.buildkite/scripts/jdk-matrix-tests/generate-steps.py @@ -229,7 +229,6 @@ def java_unit_test(self) -> JobRetValues: step_name_human = "Java Unit Test" step_key = f"{self.group_key}-java-unit-test" test_command = ''' -export ENABLE_SONARQUBE="false" ci/unit_tests.sh java ''' diff --git a/ci/unit_tests.sh b/ci/unit_tests.sh index 82a670b071e..d7a760d693e 100755 --- a/ci/unit_tests.sh +++ b/ci/unit_tests.sh @@ -10,8 +10,6 @@ export GRADLE_OPTS="-Xmx4g -Dorg.gradle.jvmargs=-Xmx4g -Dorg.gradle.daemon=false export SPEC_OPTS="--order rand --format documentation" export CI=true export TEST_DEBUG=true -# don't rely on bash booleans for truth checks, since some CI platforms don't have a way to specify env vars as boolean -export ENABLE_SONARQUBE=${ENABLE_SONARQUBE:-"true"} if [ -n "$BUILD_JAVA_HOME" ]; then GRADLE_OPTS="$GRADLE_OPTS -Dorg.gradle.java.home=$BUILD_JAVA_HOME" @@ -19,15 +17,9 @@ fi SELECTED_TEST_SUITE=$1 -SONAR_ARGS=() -if [[ $(echo $ENABLE_SONARQUBE | tr '[:lower:]' '[:upper:]') == "TRUE" ]]; then - SONAR_ARGS=("jacocoTestReport") - export COVERAGE=true -fi - if [[ $SELECTED_TEST_SUITE == $"java" ]]; then echo "Running Java Tests" - ./gradlew javaTests "${SONAR_ARGS[@]}" --console=plain --warning-mode all + ./gradlew javaTests --console=plain --warning-mode all elif [[ $SELECTED_TEST_SUITE == $"ruby" ]]; then echo "Running Ruby unit tests" ./gradlew rubyTests --console=plain --warning-mode all diff --git a/logstash-core/build.gradle b/logstash-core/build.gradle index a45e168f008..de201dc6679 100644 --- a/logstash-core/build.gradle +++ b/logstash-core/build.gradle @@ -23,22 +23,14 @@ buildscript { plugins { id "jacoco" - id "org.sonarqube" version "4.3.0.3225" } apply plugin: 'jacoco' -apply plugin: "org.sonarqube" repositories { mavenCentral() } -sonarqube { - properties { - property 'sonar.coverage.jacoco.xmlReportPaths', "${buildDir}/reports/jacoco/test/jacocoTestReport.xml" - } -} - jacoco { toolVersion = "0.8.9" } diff --git a/sonar-project.properties b/sonar-project.properties deleted file mode 100644 index d95e9fcc176..00000000000 --- a/sonar-project.properties +++ /dev/null @@ -1,12 +0,0 @@ -sonar.projectKey=elastic_logstash_AYm_nEbQaV3I-igkX1q9 -sonar.host.url=https://sonar.elastic.dev - -sonar.exclusions=vendor/**, gradle/**, rakelib/**, logstash-core-plugin-api/**, licenses/**, qa/**, **/spec/** -sonar.tests=logstash-core/src/test, x-pack/src/test, buildSrc/src/test - -# Ruby -sonar.ruby.coverage.reportPaths=coverage/coverage.json - -# Java -sonar.coverage.jacoco.xmlReportPaths=**/jacocoTestReport.xml -sonar.java.binaries=**/build/classes \ No newline at end of file From 64bc3829025764705b7336d1c9bf9d2cd32f2ff6 Mon Sep 17 00:00:00 2001 From: Karen Metts <35154725+karenzone@users.noreply.github.com> Date: Tue, 7 Oct 2025 18:41:49 -0400 Subject: [PATCH 39/44] Doc: Update docs for testing for boolean fields (#18271) --- docs/reference/tips-best-practices.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/reference/tips-best-practices.md b/docs/reference/tips-best-practices.md index f6003263f5d..2d2eed478d9 100644 --- a/docs/reference/tips-best-practices.md +++ b/docs/reference/tips-best-practices.md @@ -60,13 +60,15 @@ filter { # we use a "temporal" field with a predefined arbitrary known value that # lives only in filtering stage. add_field => { "[@metadata][test_field_check]" => "a null value" } + } +filter { + mutate { # we copy the field of interest into that temporal field. # If the field doesn't exist, copy is not executed. copy => { "test_field" => "[@metadata][test_field_check]" } } - # now we now if testField didn't exists, our field will have # the initial arbitrary value if [@metadata][test_field_check] == "a null value" { From 2d8c3cccd2a545731d8ada03b9ce36e4c6b5768b Mon Sep 17 00:00:00 2001 From: Emily S Date: Wed, 15 Oct 2025 09:35:32 +0200 Subject: [PATCH 40/44] Move invokedynamic and log4j isThreadContextMapInheritable from jvm.options to code (#18296) * Set jruby.compile.invokedynamic, log4j2.isThreadContextMapInheritable opts in code * Add invokedynamic to build.gradle settings * Java 11 is no longer supported so don't test for it * Update build.gradle Co-authored-by: Rob Bavey * test not including log4j setting in build.gradle * Revert "test not including log4j setting in build.gradle" This reverts commit 852f97a5beb4601c6a58ac3ebcdd2aeb57796dd4. --------- Co-authored-by: Rob Bavey --- build.gradle | 2 ++ config/jvm.options | 6 ------ .../main/java/org/logstash/launchers/JvmOptionsParser.java | 2 ++ .../java/org/logstash/launchers/JvmOptionsParserTest.java | 4 ++++ 4 files changed, 8 insertions(+), 6 deletions(-) diff --git a/build.gradle b/build.gradle index 923bdc49151..27409b0cb0b 100644 --- a/build.gradle +++ b/build.gradle @@ -90,6 +90,8 @@ allprojects { tasks.withType(Test) { // Add Exports to enable tests to run in JDK17 jvmArgs = [ + "-Djruby.compile.invokedynamic=true", + "-Dlog4j2.isThreadContextMapInheritable=true", "--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", "--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", "--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", diff --git a/config/jvm.options b/config/jvm.options index 084ff034b6d..f339bff690e 100644 --- a/config/jvm.options +++ b/config/jvm.options @@ -41,9 +41,6 @@ # use our provided JNA always versus the system one #-Djna.nosys=true -# Turn on JRuby invokedynamic --Djruby.compile.invokedynamic=true - ## heap dumps # generate a heap dump when an allocation from the Java heap fails @@ -60,9 +57,6 @@ # Entropy source for randomness -Djava.security.egd=file:/dev/urandom -# Copy the logging context from parent threads to children --Dlog4j2.isThreadContextMapInheritable=true - # FasterXML/jackson defaults # # Sets the maximum string length (in chars or bytes, depending on input context). diff --git a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java index 24edfec7332..bb00a81c6b0 100644 --- a/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java +++ b/tools/jvm-options-parser/src/main/java/org/logstash/launchers/JvmOptionsParser.java @@ -53,7 +53,9 @@ public class JvmOptionsParser { private static final String[] MANDATORY_JVM_OPTIONS = new String[]{ "-Djruby.regexp.interruptible=true", + "-Djruby.compile.invokedynamic=true", "-Djdk.io.File.enableADS=true", + "-Dlog4j2.isThreadContextMapInheritable=true", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED", "16-:--add-exports=jdk.compiler/com.sun.tools.javac.parser=ALL-UNNAMED", diff --git a/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java b/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java index 49e28df7f82..b61414ca89c 100644 --- a/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java +++ b/tools/jvm-options-parser/src/test/java/org/logstash/launchers/JvmOptionsParserTest.java @@ -122,6 +122,10 @@ public void testAlwaysMandatoryJvmPresent() { JvmOptionsParser.getMandatoryJvmOptions(11).contains("-Djruby.regexp.interruptible=true")); assertTrue("Contains regexp interruptible for Java 17", JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Djruby.regexp.interruptible=true")); + assertTrue("Contains compile invokedynamic for Java 17", + JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Djruby.compile.invokedynamic=true")); + assertTrue("Contains log4j2 isThreadContextMapInheritable for Java 17", + JvmOptionsParser.getMandatoryJvmOptions(17).contains("-Dlog4j2.isThreadContextMapInheritable=true")); } From 155b4caeedc4395da2acb8a28d41fbdca13313b1 Mon Sep 17 00:00:00 2001 From: Mashhur <99575341+mashhurs@users.noreply.github.com> Date: Wed, 15 Oct 2025 07:26:03 -0700 Subject: [PATCH 41/44] Adds integration test for the `_health_report` and `_node/plugins` APIs. (#18306) * Adds integration test for the _health_report API. * Add _node/plugins API integration test. --- qa/integration/services/monitoring_api.rb | 10 ++++++ qa/integration/specs/monitoring_api_spec.rb | 36 +++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/qa/integration/services/monitoring_api.rb b/qa/integration/services/monitoring_api.rb index 34b52634f11..ceac94822a3 100644 --- a/qa/integration/services/monitoring_api.rb +++ b/qa/integration/services/monitoring_api.rb @@ -74,4 +74,14 @@ def logging_reset resp = Manticore.put("http://localhost:#{@port}/_node/logging/reset", {headers: {"Content-Type" => "application/json"}}).body JSON.parse(resp) end + + def health_report + resp = Manticore.get("http://localhost:#{@port}/_health_report").body + JSON.parse(resp) + end + + def node_plugins + resp = Manticore.get("http://localhost:#{@port}/_node/plugins").body + JSON.parse(resp) + end end diff --git a/qa/integration/specs/monitoring_api_spec.rb b/qa/integration/specs/monitoring_api_spec.rb index 4bcead29634..30598dbf923 100644 --- a/qa/integration/specs/monitoring_api_spec.rb +++ b/qa/integration/specs/monitoring_api_spec.rb @@ -205,6 +205,42 @@ end end + it 'retrieves health report' do + logstash_service = @fixture.get_service("logstash") + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # health_report can fail if the subsystem isn't ready + result = logstash_service.monitoring_api.health_report rescue nil + expect(result).not_to be_nil + expect(result).to be_a(Hash) + expect(result).to include("status") + expect(result["status"]).to match(/^(green|yellow|red)$/) + end + end + + it 'retrieves node plugins information' do + logstash_service = @fixture.get_service("logstash") + logstash_service.start_with_stdin + logstash_service.wait_for_logstash + Stud.try(max_retry.times, [StandardError, RSpec::Expectations::ExpectationNotMetError]) do + # node_plugins can fail if the subsystem isn't ready + result = logstash_service.monitoring_api.node_plugins rescue nil + expect(result).not_to be_nil + expect(result).to be_a(Hash) + expect(result).to include("plugins") + plugins = result["plugins"] + expect(plugins).to be_a(Array) + expect(plugins.size).to be > 0 + # verify plugin structure and that stdin plugin is present + stdin_plugin = plugins.find { |p| p["name"] == "logstash-input-stdin" } + expect(stdin_plugin).not_to be_nil + expect(stdin_plugin).to include("name") + expect(stdin_plugin["name"]).to eq("logstash-input-stdin") + expect(stdin_plugin).to include("version") + end + end + shared_examples "pipeline metrics" do # let(:pipeline_id) { defined?(super()) or fail NotImplementedError } let(:settings_overrides) do From 50d6b3db1f72518183b0b8250994f147f3933f6e Mon Sep 17 00:00:00 2001 From: Cas Donoghue Date: Fri, 17 Oct 2025 13:41:42 -0700 Subject: [PATCH 42/44] Handle GH org for elastic_integration plugin (#18315) Previously the RN generator would look for EVERY plugin in the `logstash-plugins` GH org. The elastic integration plugin is in the `elastic` org. Update the logic to handle this. --- tools/release/generate_release_notes_md.rb | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tools/release/generate_release_notes_md.rb b/tools/release/generate_release_notes_md.rb index cedf0e6ec63..d2c16c339f1 100755 --- a/tools/release/generate_release_notes_md.rb +++ b/tools/release/generate_release_notes_md.rb @@ -91,15 +91,17 @@ plugin_changes.each do |plugin, versions| _, type, name = plugin.split("-") header = "**#{name.capitalize} #{type.capitalize} - #{versions.last}**" + # Determine the correct GitHub organization + org = plugin.include?('elastic_integration') ? 'elastic' : 'logstash-plugins' start_changelog_file = Tempfile.new(plugin + 'start') end_changelog_file = Tempfile.new(plugin + 'end') - changelog = `curl https://raw.githubusercontent.com/logstash-plugins/#{plugin}/v#{versions.last}/CHANGELOG.md`.split("\n") + changelog = `curl https://raw.githubusercontent.com/#{org}/#{plugin}/v#{versions.last}/CHANGELOG.md`.split("\n") report << "#{header}\n" changelog.each do |line| break if line.match(/^## #{versions.first}/) next if line.match(/^##/) line.gsub!(/^\+/, "") - line.gsub!(/ #(?\d+)\s*$/, " https://github.com/logstash-plugins/#{plugin}/issues/\\k[#\\k]") + line.gsub!(/ #(?\d+)\s*$/, " https://github.com/#{org}/#{plugin}/issues/\\k[#\\k]") line.gsub!(/\[#(?\d+)\]\((?[^)]*)\)/, "[#\\k](\\k)") line.gsub!(/^\s+-/, "*") report << line From 31a643c12390b293ff8f728fc3de4c3da5e55d4b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Oct 2025 10:26:53 +0200 Subject: [PATCH 43/44] Bump the github-actions group across 1 directory with 2 updates (#18320) Bumps the github-actions group with 2 updates in the / directory: [anchore/scan-action](https://github.com/anchore/scan-action) and [actions/setup-node](https://github.com/actions/setup-node). Updates `anchore/scan-action` from 7.0.0 to 7.0.2 - [Release notes](https://github.com/anchore/scan-action/releases) - [Changelog](https://github.com/anchore/scan-action/blob/main/RELEASE.md) - [Commits](https://github.com/anchore/scan-action/compare/f6601287cdb1efc985d6b765bbf99cb4c0ac29d8...a5605eb0943e46279cb4fbd9d44297355d3520ab) Updates `actions/setup-node` from 5 to 6 - [Release notes](https://github.com/actions/setup-node/releases) - [Commits](https://github.com/actions/setup-node/compare/v5...v6) --- updated-dependencies: - dependency-name: anchore/scan-action dependency-version: 7.0.2 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: github-actions - dependency-name: actions/setup-node dependency-version: '6' dependency-type: direct:production update-type: version-update:semver-major dependency-group: github-actions ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/critical_vulnerability_scan.yml | 2 +- .github/workflows/lint_docs.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/critical_vulnerability_scan.yml b/.github/workflows/critical_vulnerability_scan.yml index a500c9448e2..dc437bb41c9 100644 --- a/.github/workflows/critical_vulnerability_scan.yml +++ b/.github/workflows/critical_vulnerability_scan.yml @@ -17,7 +17,7 @@ jobs: - run: tar -zxf ../build/logstash-*.tar.gz working-directory: ./scan - name: scan image - uses: anchore/scan-action@f6601287cdb1efc985d6b765bbf99cb4c0ac29d8 # v7.0.0 + uses: anchore/scan-action@a5605eb0943e46279cb4fbd9d44297355d3520ab # v7.0.2 with: path: "./scan" fail-build: true diff --git a/.github/workflows/lint_docs.yml b/.github/workflows/lint_docs.yml index f1620d0d69f..c70b98fe699 100644 --- a/.github/workflows/lint_docs.yml +++ b/.github/workflows/lint_docs.yml @@ -15,7 +15,7 @@ jobs: uses: actions/checkout@v5 - name: Set up Node.js - uses: actions/setup-node@v5 + uses: actions/setup-node@v6 with: node-version: 16.13.2 cache: npm From 50ee5c6bc13fad28dc327c287b0f57854312a668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=81lex=20C=C3=A1mara=20Lara?= Date: Tue, 21 Oct 2025 09:42:37 +0200 Subject: [PATCH 44/44] Add encoded/non-encoded auth method to x-pack unit tests (#18307) --- .../elasticsearch_source_spec.rb | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/x-pack/spec/config_management/elasticsearch_source_spec.rb b/x-pack/spec/config_management/elasticsearch_source_spec.rb index b16916ca5e6..6d51df92246 100644 --- a/x-pack/spec/config_management/elasticsearch_source_spec.rb +++ b/x-pack/spec/config_management/elasticsearch_source_spec.rb @@ -170,6 +170,30 @@ expect { described_class.new(system_settings) }.to_not raise_error end end + + context "when api_key is set (encoded or not)" do + [ + { desc: "non-encoded", value: "foo:bar" }, + { desc: "encoded", value: Base64.strict_encode64("foo:bar") } + ].each do |api_key_case| + context "with #{api_key_case[:desc]} api_key" do + let(:settings) do + { + "xpack.management.enabled" => true, + "xpack.management.pipeline.id" => "main", + "xpack.management.elasticsearch.api_key" => api_key_case[:value], + } + end + + it "will rely on #{api_key_case[:desc]} api_key for authentication" do + # the http client used by xpack module is the same as the one used by the ES output plugin + # and the HttpClientBuilder.setup_api_key method will handle both encoded and non-encoded api_key values. + # These tests prevent future regressions if the plugin client is changed. + expect { described_class.new(system_settings) }.to_not raise_error + end + end + end + end end context "valid settings" do