diff --git a/CHANGELOG.md b/CHANGELOG.md index d5a53f11..e792e059 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 11.13.1 + - Avoid crash by ensuring ILM settings are injected in the correct location depending on the default (or custom) template format, template_api setting and ES version [#1102](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1102) + ## 11.13.0 - add technology preview support for allowing events to individually encode a default pipeline with `[@metadata][target_ingest_pipeline]` (as part of a technology preview, this feature may change without notice) [#1113](https://github.com/logstash-plugins/logstash-output-elasticsearch/pull/1113) diff --git a/lib/logstash/outputs/elasticsearch/template_manager.rb b/lib/logstash/outputs/elasticsearch/template_manager.rb index 6cf76199..495db258 100644 --- a/lib/logstash/outputs/elasticsearch/template_manager.rb +++ b/lib/logstash/outputs/elasticsearch/template_manager.rb @@ -46,15 +46,38 @@ def self.add_ilm_settings_to_template(plugin, template) # definition - remove any existing definition of 'template' template.delete('template') if template.include?('template') if plugin.maximum_seen_major_version < 8 template['index_patterns'] = "#{plugin.ilm_rollover_alias}-*" - settings = template_settings(plugin, template) + settings = resolve_template_settings(plugin, template) if settings && (settings['index.lifecycle.name'] || settings['index.lifecycle.rollover_alias']) plugin.logger.info("Overwriting index lifecycle name and rollover alias as ILM is enabled") end settings.update({ 'index.lifecycle.name' => plugin.ilm_policy, 'index.lifecycle.rollover_alias' => plugin.ilm_rollover_alias}) end - def self.template_settings(plugin, template) - plugin.maximum_seen_major_version < 8 ? template['settings']: template['template']['settings'] + def self.resolve_template_settings(plugin, template) + if template.key?('template') + plugin.logger.trace("Resolving ILM template settings: under 'template' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version) + composable_index_template_settings(template) + elsif template.key?('settings') + plugin.logger.trace("Resolving ILM template settings: under 'settings' key", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version) + legacy_index_template_settings(template) + else + template_endpoint = template_endpoint(plugin) + plugin.logger.trace("Resolving ILM template settings: template doesn't have 'settings' or 'template' fields, falling back to auto detection", :template => template, :template_api => plugin.template_api, :es_version => plugin.maximum_seen_major_version, :template_endpoint => template_endpoint) + template_endpoint == INDEX_TEMPLATE_ENDPOINT ? + composable_index_template_settings(template) : + legacy_index_template_settings(template) + end + end + + # Sets ['settings'] field to be compatible with _template API structure + def self.legacy_index_template_settings(template) + template['settings'] ||= {} + end + + # Sets the ['template']['settings'] fields if not exist to be compatible with _index_template API structure + def self.composable_index_template_settings(template) + template['template'] ||= {} + template['template']['settings'] ||= {} end # Template name - if template_name set, use it diff --git a/logstash-output-elasticsearch.gemspec b/logstash-output-elasticsearch.gemspec index a6140703..631c7060 100644 --- a/logstash-output-elasticsearch.gemspec +++ b/logstash-output-elasticsearch.gemspec @@ -1,6 +1,6 @@ Gem::Specification.new do |s| s.name = 'logstash-output-elasticsearch' - s.version = '11.13.0' + s.version = '11.13.1' s.licenses = ['apache-2.0'] s.summary = "Stores logs in Elasticsearch" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/unit/outputs/elasticsearch/template_manager_spec.rb b/spec/unit/outputs/elasticsearch/template_manager_spec.rb index 9a9dcd5f..413ff2ca 100644 --- a/spec/unit/outputs/elasticsearch/template_manager_spec.rb +++ b/spec/unit/outputs/elasticsearch/template_manager_spec.rb @@ -1,4 +1,4 @@ -require "logstash/devutils/rspec/spec_helper" +require_relative "../../../../spec/spec_helper" require "logstash/outputs/elasticsearch/template_manager" describe LogStash::Outputs::ElasticSearch::TemplateManager do @@ -33,33 +33,85 @@ end end - describe "index template with ilm settings" do + context "index template with ilm settings" do let(:plugin_settings) { {"manage_template" => true, "template_overwrite" => true} } let(:plugin) { LogStash::Outputs::ElasticSearch.new(plugin_settings) } - describe "in version 8+" do - let(:file_path) { described_class.default_template_path(8) } - let(:template) { described_class.read_template_file(file_path)} + describe "with custom template" do - it "should update settings" do - expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) - described_class.add_ilm_settings_to_template(plugin, template) - expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil) - expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) - expect(template.include?('settings')).to be_falsey + describe "in version 8+" do + let(:file_path) { described_class.default_template_path(8) } + let(:template) { described_class.read_template_file(file_path)} + + it "should update settings" do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) + described_class.add_ilm_settings_to_template(plugin, template) + expect(template['template']['settings']['index.lifecycle.name']).not_to eq(nil) + expect(template['template']['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) + expect(template.include?('settings')).to be_falsey + end + end + + describe "in version < 8" do + let(:file_path) { described_class.default_template_path(7) } + let(:template) { described_class.read_template_file(file_path)} + + it "should update settings" do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) + described_class.add_ilm_settings_to_template(plugin, template) + expect(template['settings']['index.lifecycle.name']).not_to eq(nil) + expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) + expect(template.include?('template')).to be_falsey + end end end - describe "in version < 8" do - let(:file_path) { described_class.default_template_path(7) } - let(:template) { described_class.read_template_file(file_path)} + context "resolve template setting" do + let(:plugin_settings) { super().merge({"template_api" => template_api}) } + + describe "with composable template API" do + let(:template_api) { "composable" } - it "should update settings" do - expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) - described_class.add_ilm_settings_to_template(plugin, template) - expect(template['settings']['index.lifecycle.name']).not_to eq(nil) - expect(template['settings']['index.lifecycle.rollover_alias']).not_to eq(nil) - expect(template.include?('template')).to be_falsey + it 'resolves composable index template API compatible setting' do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) # required to log + template = {} + described_class.resolve_template_settings(plugin, template) + expect(template["template"]["settings"]).not_to eq(nil) + end + end + + describe "with legacy template API" do + let(:template_api) { "legacy" } + + it 'resolves legacy index template API compatible setting' do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) # required to log + template = {} + described_class.resolve_template_settings(plugin, template) + expect(template["settings"]).not_to eq(nil) + end + end + + describe "with `template_api => 'auto'`" do + let(:template_api) { "auto" } + + describe "with ES < 8 versions" do + + it 'resolves legacy index template API compatible setting' do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(7) + template = {} + described_class.resolve_template_settings(plugin, template) + expect(template["settings"]).not_to eq(nil) + end + end + + describe "with ES >= 8 versions" do + it 'resolves composable index template API compatible setting' do + expect(plugin).to receive(:maximum_seen_major_version).at_least(:once).and_return(8) + template = {} + described_class.resolve_template_settings(plugin, template) + expect(template["template"]["settings"]).not_to eq(nil) + end + end end end end