diff --git a/CHANGELOG.md b/CHANGELOG.md index 25abffa3..fb1e0b70 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +## 4.7.0 + - Added api_key support [#131](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/131) + ## 4.6.2 - Added scroll clearing and better handling of scroll expiration [#128](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/128) diff --git a/docs/index.asciidoc b/docs/index.asciidoc index 1037efe8..b54c6ce5 100644 --- a/docs/index.asciidoc +++ b/docs/index.asciidoc @@ -68,6 +68,16 @@ Further documentation describing this syntax can be found https://github.com/jmettraux/rufus-scheduler#parsing-cronlines-and-time-strings[here]. +[id="plugins-{type}s-{plugin}-auth"] +==== Authentication + +Authentication to a secure Elasticsearch cluster is possible using _one_ of the following options: + +* <> AND <> +* <> +* <> + + [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Input Configuration Options @@ -76,6 +86,7 @@ This plugin supports the following configuration options plus the <> |<>|No | <> |a valid filesystem path|No | <> |<>|No | <> |<>|No @@ -100,6 +111,16 @@ input plugins.   +[id="plugins-{type}s-{plugin}-api_key"] +===== `api_key` + + * Value type is <> + * There is no default value for this setting. + +Authenticate using Elasticsearch API key. Note that this option also requires enabling the `ssl` option. + +Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key API]. + [id="plugins-{type}s-{plugin}-ca_file"] ===== `ca_file` @@ -315,4 +336,4 @@ empty string authentication will be disabled. [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[] -:default_codec!: \ No newline at end of file +:default_codec!: diff --git a/lib/logstash/inputs/elasticsearch.rb b/lib/logstash/inputs/elasticsearch.rb index f10dafbf..a4cf8bda 100644 --- a/lib/logstash/inputs/elasticsearch.rb +++ b/lib/logstash/inputs/elasticsearch.rb @@ -70,11 +70,6 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base # Port defaults to 9200 config :hosts, :validate => :array - # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. - # - # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation] - config :cloud_id, :validate => :string - # The index or alias to search. config :index, :validate => :string, :default => "logstash-*" @@ -140,11 +135,20 @@ class LogStash::Inputs::Elasticsearch < LogStash::Inputs::Base # Basic Auth - password config :password, :validate => :password + # Cloud ID, from the Elastic Cloud web console. If set `hosts` should not be used. + # + # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_id[Logstash-to-Cloud documentation] + config :cloud_id, :validate => :string + # Cloud authentication string (":" format) is an alternative for the `user`/`password` configuration. # # For more info, check out the https://www.elastic.co/guide/en/logstash/current/connecting-to-cloud.html#_cloud_auth[Logstash-to-Cloud documentation] config :cloud_auth, :validate => :password + # Authenticate using Elasticsearch API key. + # format is id:api_key (as returned by https://www.elastic.co/guide/en/elasticsearch/reference/current/security-api-create-api-key.html[Create API key]) + config :api_key, :validate => :password + # Set the address of a forward HTTP proxy. config :proxy, :validate => :uri_or_empty @@ -177,28 +181,17 @@ def register @slices < 1 && fail(LogStash::ConfigurationError, "Elasticsearch Input Plugin's `slices` option must be greater than zero, got `#{@slices}`") end - transport_options = {} - + validate_authentication fill_user_password_from_cloud_auth + fill_hosts_from_cloud_id - if @user && @password - token = Base64.strict_encode64("#{@user}:#{@password.value}") - transport_options[:headers] = { :Authorization => "Basic #{token}" } - end - fill_hosts_from_cloud_id - @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s + transport_options = {:headers => {}} + transport_options[:headers].merge!(setup_basic_auth(user, password)) + transport_options[:headers].merge!(setup_api_key(api_key)) - hosts = if @ssl - @hosts.map do |h| - host, port = h.split(":") - { :host => host, :scheme => 'https', :port => port } - end - else - @hosts - end - ssl_options = { :ssl => true, :ca_file => @ca_file } if @ssl && @ca_file - ssl_options ||= {} + hosts = setup_hosts + ssl_options = setup_ssl @logger.warn "Supplied proxy setting (proxy => '') has no effect" if @proxy.eql?('') @@ -351,25 +344,67 @@ def hosts_default?(hosts) hosts.nil? || ( hosts.is_a?(Array) && hosts.empty? ) end - def fill_hosts_from_cloud_id - return unless @cloud_id + def validate_authentication + authn_options = 0 + authn_options += 1 if @cloud_auth + authn_options += 1 if (@api_key && @api_key.value) + authn_options += 1 if (@user || (@password && @password.value)) - if @hosts && !hosts_default?(@hosts) - raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.' + if authn_options > 1 + raise LogStash::ConfigurationError, 'Multiple authentication options are specified, please only use one of user/password, cloud_auth or api_key' end - @hosts = parse_host_uri_from_cloud_id(@cloud_id) + + if @api_key && @api_key.value && @ssl != true + raise(LogStash::ConfigurationError, "Using api_key authentication requires SSL/TLS secured communication using the `ssl => true` option") + end + end + + def setup_ssl + @ssl && @ca_file ? { :ssl => true, :ca_file => @ca_file } : {} + end + + def setup_hosts + @hosts = Array(@hosts).map { |host| host.to_s } # potential SafeURI#to_s + if @ssl + @hosts.map do |h| + host, port = h.split(":") + { :host => host, :scheme => 'https', :port => port } + end + else + @hosts + end + end + + def setup_basic_auth(user, password) + return {} unless user && password && password.value + + token = ::Base64.strict_encode64("#{user}:#{password.value}") + { Authorization: "Basic #{token}" } + end + + def setup_api_key(api_key) + return {} unless (api_key && api_key.value) + + token = ::Base64.strict_encode64(api_key.value) + { Authorization: "ApiKey #{token}" } end def fill_user_password_from_cloud_auth return unless @cloud_auth - if @user || @password - raise LogStash::ConfigurationError, 'Both cloud_auth and user/password specified, please only use one.' - end @user, @password = parse_user_password_from_cloud_auth(@cloud_auth) params['user'], params['password'] = @user, @password end + def fill_hosts_from_cloud_id + return unless @cloud_id + + if @hosts && !hosts_default?(@hosts) + raise LogStash::ConfigurationError, 'Both cloud_id and hosts specified, please only use one of those.' + end + @hosts = parse_host_uri_from_cloud_id(@cloud_id) + end + def parse_host_uri_from_cloud_id(cloud_id) begin # might not be available on older LS require 'logstash/util/cloud_setting_id' diff --git a/logstash-input-elasticsearch.gemspec b/logstash-input-elasticsearch.gemspec index 19445327..7511ac72 100644 --- a/logstash-input-elasticsearch.gemspec +++ b/logstash-input-elasticsearch.gemspec @@ -1,7 +1,7 @@ Gem::Specification.new do |s| s.name = 'logstash-input-elasticsearch' - s.version = '4.6.2' + s.version = '4.7.0' s.licenses = ['Apache License (2.0)'] s.summary = "Reads query results from an Elasticsearch cluster" s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program" diff --git a/spec/inputs/elasticsearch_spec.rb b/spec/inputs/elasticsearch_spec.rb index 4027767c..4832f5e9 100644 --- a/spec/inputs/elasticsearch_spec.rb +++ b/spec/inputs/elasticsearch_spec.rb @@ -583,7 +583,37 @@ def synchronize_method!(object, method_name) let(:config) { super.merge({ 'cloud_auth' => 'elastic:my-passwd-00', 'user' => 'another' }) } it "should fail" do - expect { plugin.register }.to raise_error LogStash::ConfigurationError, /cloud_auth and user/ + expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/ + end + end + end if LOGSTASH_VERSION > '6.0' + + describe "api_key" do + context "without ssl" do + let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar') }) } + + it "should fail" do + expect { plugin.register }.to raise_error LogStash::ConfigurationError, /api_key authentication requires SSL\/TLS/ + end + end + + context "with ssl" do + let(:config) { super.merge({ 'api_key' => LogStash::Util::Password.new('foo:bar'), "ssl" => true }) } + + it "should set authorization" do + plugin.register + client = plugin.send(:client) + auth_header = client.transport.options[:transport_options][:headers][:Authorization] + + expect( auth_header ).to eql "ApiKey #{Base64.strict_encode64('foo:bar')}" + end + + context 'user also set' do + let(:config) { super.merge({ 'api_key' => 'foo:bar', 'user' => 'another' }) } + + it "should fail" do + expect { plugin.register }.to raise_error LogStash::ConfigurationError, /Multiple authentication options are specified/ + end end end end if LOGSTASH_VERSION > '6.0'