From 4bc5afcc94f84c23ac90e0565eb589720f7d60fe Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 3 Mar 2025 16:49:08 -0800 Subject: [PATCH 1/3] Adds elastic-transport client support where opens LS-core a way to start using newer ES ruby client. --- CHANGELOG.md | 3 +++ lib/logstash/filters/elasticsearch.rb | 4 +++- lib/logstash/filters/elasticsearch/client.rb | 14 ++++++++++++-- logstash-filter-elasticsearch.gemspec | 4 ++-- 4 files changed, 20 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 566edee..3211413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.1.1 + - Adds elastic-transport support [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191) + ## 4.1.0 - Added support for custom headers [#188](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/188) diff --git a/lib/logstash/filters/elasticsearch.rb b/lib/logstash/filters/elasticsearch.rb index a534faf..dab33e9 100644 --- a/lib/logstash/filters/elasticsearch.rb +++ b/lib/logstash/filters/elasticsearch.rb @@ -6,7 +6,6 @@ require "monitor" require_relative "elasticsearch/client" -require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore" class LogStash::Filters::Elasticsearch < LogStash::Filters::Base config_name "elasticsearch" @@ -174,6 +173,9 @@ def register test_connection! setup_serverless + if get_client.es_transport_client_type == "elasticsearch_transport" + require_relative "elasticsearch/patches/_elasticsearch_transport_http_manticore" + end end # def register def filter(event) diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index c7c3b7d..12924a8 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -1,7 +1,6 @@ # encoding: utf-8 require "elasticsearch" require "base64" -require "elasticsearch/transport/transport/http/manticore" module LogStash @@ -9,6 +8,7 @@ module Filters class ElasticsearchClient attr_reader :client + attr_reader :es_transport_client_type BUILD_FLAVOR_SERVERLESS = 'serverless'.freeze DEFAULT_EAV_HEADER = { "Elastic-Api-Version" => "2023-10-31" }.freeze @@ -44,7 +44,7 @@ def initialize(logger, hosts, options = {}) client_options = { hosts: hosts, - transport_class: ::Elasticsearch::Transport::Transport::HTTP::Manticore, + transport_class: get_transport_client_class, transport_options: transport_options, ssl: ssl_options, retry_on_failure: options[:retry_on_failure], @@ -98,6 +98,16 @@ def setup_api_key(api_key) token = ::Base64.strict_encode64(api_key.value) { 'Authorization' => "ApiKey #{token}" } end + + def get_transport_client_class + require "elasticsearch/transport/transport/http/manticore" + es_transport_client_type = "elasticsearch_transport" + ::Elasticsearch::Transport::Transport::HTTP::Manticore + rescue ::LoadError + require "elastic/transport/transport/http/manticore" + es_transport_client_type = "elastic_transport" + ::Elastic::Transport::Transport::HTTP::Manticore + end end end end diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index a05f9c3..50cfa8a 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -1,13 +1,13 @@ Gem::Specification.new do |s| s.name = 'logstash-filter-elasticsearch' - s.version = '4.1.0' + s.version = '4.1.1' s.licenses = ['Apache License (2.0)'] s.summary = "Copies fields from previous log events in Elasticsearch to current events " s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" s.authors = ["Elastic"] s.email = 'info@elastic.co' - s.homepage = "http://www.elastic.co/guide/en/logstash/current/index.html" + s.homepage = "https://elastic.co/logstash" s.require_paths = ["lib"] # Files From 244b4e8889465192bb20bbbf93b6bbfc4e2db558 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Mon, 3 Mar 2025 22:52:42 -0800 Subject: [PATCH 2/3] Fix failed unit tests. --- CHANGELOG.md | 2 +- spec/filters/elasticsearch_spec.rb | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3211413..af2acb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 4.1.1 - - Adds elastic-transport support [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191) + - Adds elastic-transport client support [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191) ## 4.1.0 - Added support for custom headers [#188](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/188) diff --git a/spec/filters/elasticsearch_spec.rb b/spec/filters/elasticsearch_spec.rb index 9a7945e..adb5d3e 100644 --- a/spec/filters/elasticsearch_spec.rb +++ b/spec/filters/elasticsearch_spec.rb @@ -103,6 +103,7 @@ before(:each) do allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client) + allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport') allow(client).to receive(:search).and_return(response) allow(plugin).to receive(:test_connection!) allow(plugin).to receive(:setup_serverless) @@ -347,6 +348,7 @@ before do allow(plugin).to receive(:get_client).and_return(client_double) + allow(client_double).to receive(:es_transport_client_type).and_return('elasticsearch_transport') allow(client_double).to receive(:client).and_return(transport_double) end @@ -821,6 +823,7 @@ def wait_receive_request before(:each) do allow(LogStash::Filters::ElasticsearchClient).to receive(:new).and_return(client) + allow(client).to receive(:es_transport_client_type).and_return('elasticsearch_transport') allow(plugin).to receive(:test_connection!) allow(plugin).to receive(:setup_serverless) plugin.register From d6e26ea1d3661b472fdfa6e37377a3626d3a6945 Mon Sep 17 00:00:00 2001 From: Mashhur Date: Tue, 11 Mar 2025 13:08:54 -0700 Subject: [PATCH 3/3] Code review changes: define specific elasticsearch gem version range, add comments on the logics to support both elastic and elasticsearch transport gems. --- CHANGELOG.md | 2 +- lib/logstash/filters/elasticsearch/client.rb | 4 ++++ logstash-filter-elasticsearch.gemspec | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index af2acb5..5a6d0ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,5 @@ ## 4.1.1 - - Adds elastic-transport client support [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191) + - Add elastic-transport client support used in elasticsearch-ruby 8.x [#191](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/191) ## 4.1.0 - Added support for custom headers [#188](https://github.com/logstash-plugins/logstash-filter-elasticsearch/pull/188) diff --git a/lib/logstash/filters/elasticsearch/client.rb b/lib/logstash/filters/elasticsearch/client.rb index 12924a8..120d8e5 100644 --- a/lib/logstash/filters/elasticsearch/client.rb +++ b/lib/logstash/filters/elasticsearch/client.rb @@ -100,6 +100,10 @@ def setup_api_key(api_key) end def get_transport_client_class + # LS-core includes `elasticsearch` gem. The gem is composed of two separate gems: `elasticsearch-api` and `elasticsearch-transport` + # And now `elasticsearch-transport` is old, instead we have `elastic-transport`. + # LS-core updated `elasticsearch` > 8: https://github.com/elastic/logstash/pull/17161 + # Following source bits are for the compatibility to support both `elasticsearch-transport` and `elastic-transport` gems require "elasticsearch/transport/transport/http/manticore" es_transport_client_type = "elasticsearch_transport" ::Elasticsearch::Transport::Transport::HTTP::Manticore diff --git a/logstash-filter-elasticsearch.gemspec b/logstash-filter-elasticsearch.gemspec index 50cfa8a..29670c2 100644 --- a/logstash-filter-elasticsearch.gemspec +++ b/logstash-filter-elasticsearch.gemspec @@ -21,7 +21,7 @@ Gem::Specification.new do |s| # Gem dependencies s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99" - s.add_runtime_dependency 'elasticsearch', ">= 7.14.9" # LS >= 6.7 and < 7.14 all used version 5.0.5 + s.add_runtime_dependency 'elasticsearch', ">= 7.14.9", '< 9' s.add_runtime_dependency 'manticore', ">= 0.7.1" s.add_runtime_dependency 'logstash-mixin-ca_trusted_fingerprint_support', '~> 1.0' s.add_development_dependency 'cabin', ['~> 0.6']