remove code related to ES 6.x and earlier (#1194)

* fixes secure integration tests by using credentials to check ES being up
* remove ECS content for ES 6.x
* remove document_type handling for 6.x
* remove two different ways to define certain parameters like routing vs _routing
* remove ES < 7 specific version handling
* remove ES < 7 specific ILM feature checking
* remove ES < 7 specific template handling
* remove ES 6.x template
* large cleanup related to ES < 7 specific tests

Author: jsvd

.ci/docker-run.sh

@@ -7,7 +7,7 @@ cd .ci
 
 if [ "$INTEGRATION" == "true" ]; then
     # remove the `--attach logstash` if you want to see all logs including elasticsearch container logs
-    docker-compose up --exit-code-from logstash --attach logstash
+    docker compose up --exit-code-from logstash --attach logstash
 else
-    docker-compose up --exit-code-from logstash logstash
+    docker compose up --exit-code-from logstash logstash
 fi

.ci/docker-setup.sh

@@ -81,9 +81,9 @@ cd .ci
 
 export BUILDKIT_PROGRESS=plain
 if [ "$INTEGRATION" == "true" ]; then
-    docker-compose down
-    docker-compose build --quiet
+    docker compose down
+    docker compose build
 else
-    docker-compose down
-    docker-compose build logstash --quiet
+    docker compose down
+    docker compose build logstash
 fi

.ci/logstash-run.sh

@@ -13,7 +13,7 @@ else
 fi
 
 # CentOS 7 using curl defaults does not enable TLSv1.3
-CURL_OPTS="-k --tlsv1.2 --tls-max 1.3"
+CURL_OPTS="-s -u admin:elastic -k --tlsv1.2 --tls-max 1.3"
 
 wait_for_es() {
   count=120
@@ -22,7 +22,7 @@ wait_for_es() {
     [[ $count -eq 0 ]] && exit 1
     sleep 1
   done
-  echo $(curl $CURL_OPTS -vi $ES_URL | jq -r .version.number)
+  echo $(curl $CURL_OPTS $ES_URL | jq -r .version.number)
 }
 
 if [[ "$INTEGRATION" != "true" ]]; then

.travis.yml

@@ -9,6 +9,7 @@ jobs:
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next
     - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=main
     - stage: "Secure Integration Tests"
       env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true
     - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current

Rakefile

@@ -1,7 +1,6 @@
 require "logstash/devutils/rake"
 
 task :'vendor-ecs-schemata' do
-  download_ecs_schemata(:v1, elasticsearch_major: 6, ecs_release_tag: 'v1.10.0') # WARNING: v1.11 breaks 6.x (see: https://github.com/elastic/ecs/issues/1649)
   download_ecs_schemata(:v1, elasticsearch_major: 7, ecs_release_tag: 'v1.12.1')
   download_ecs_schemata(:v1, elasticsearch_major: 8, ecs_release_tag: 'v1.12.1', generated_for: 7)
   download_ecs_schemata(:v1, elasticsearch_major: 9, ecs_release_tag: 'v1.12.1', generated_for: 7)

docs/index.asciidoc

@@ -612,8 +612,7 @@ Elasticsearch with the same ID.
 
 NOTE: This option is deprecated due to the
 https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal
-of types in Elasticsearch 6.0]. It will be removed in the next major version of
-Logstash.
+of types in Elasticsearch 6.0].
 
 NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`.
 
@@ -622,9 +621,7 @@ similar events to the same 'type'. String expansion `%{foo}` works here.
 If you don't set a value for this option:
 
 - for elasticsearch clusters 8.x: no value will be used;
-- for elasticsearch clusters 7.x: the value of '_doc' will be used;
-- for elasticsearch clusters 6.x: the value of 'doc' will be used;
-- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used.
+- for elasticsearch clusters 7.x: the value of '_doc' will be used.
 
 [id="plugins-{type}s-{plugin}-ecs_compatibility"]
 ===== `ecs_compatibility`
@@ -1039,8 +1036,6 @@ NOTE: Deprecates <<plugins-{type}s-{plugin}-failure_type_logging_whitelist>>.
 
 This setting asks Elasticsearch for the list of all cluster nodes and adds them
 to the hosts list.
-For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will
-be added to the hosts list, excluding master-only nodes.
 
 [id="plugins-{type}s-{plugin}-sniffing_delay"]
 ===== `sniffing_delay`

lib/logstash/outputs/elasticsearch.rb

@@ -14,39 +14,16 @@
 # .Compatibility Note
 # [NOTE]
 # ================================================================================
-# Starting with Elasticsearch 5.3, there's an {ref}modules-http.html[HTTP setting]
-# called `http.content_type.required`. If this option is set to `true`, and you
-# are using Logstash 2.4 through 5.2, you need to update the Elasticsearch output
-# plugin to version 6.2.5 or higher.
-#
-# ================================================================================
 #
 # This plugin is the recommended method of storing logs in Elasticsearch.
 # If you plan on using the Kibana web interface, you'll want to use this output.
 #
-# This output only speaks the HTTP protocol. HTTP is the preferred protocol for interacting with Elasticsearch as of Logstash 2.0.
-# We strongly encourage the use of HTTP over the node protocol for a number of reasons. HTTP is only marginally slower,
-# yet far easier to administer and work with. When using the HTTP protocol one may upgrade Elasticsearch versions without having
-# to upgrade Logstash in lock-step.
+# This output only speaks the HTTP protocol.
 #
 # You can learn more about Elasticsearch at <https://www.elastic.co/products/elasticsearch>
 #
-# ==== Template management for Elasticsearch 5.x
-# Index template for this version (Logstash 5.0) has been changed to reflect Elasticsearch's mapping changes in version 5.0.
-# Most importantly, the subfield for string multi-fields has changed from `.raw` to `.keyword` to match ES default
-# behavior.
-#
-# ** Users installing ES 5.x and LS 5.x **
-# This change will not affect you and you will continue to use the ES defaults.
-#
-# ** Users upgrading from LS 2.x to LS 5.x with ES 5.x **
-# LS will not force upgrade the template, if `logstash` template already exists. This means you will still use
-# `.raw` for sub-fields coming from 2.x. If you choose to use the new template, you will have to reindex your data after
-# the new template is installed.
-#
 # ==== Retry Policy
 #
-# The retry policy has changed significantly in the 2.2.0 release.
 # This plugin uses the Elasticsearch bulk API to optimize its imports into Elasticsearch. These requests may experience
 # either partial or total failures.
 #
@@ -129,8 +106,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
   # - delete: deletes a document by id (An id is required for this action)
   # - create: indexes a document, fails if a document by that id already exists in the index.
   # - update: updates a document by id. Update has a special case where you can upsert -- update a
-  #   document if not already present. See the `upsert` option. NOTE: This does not work and is not supported
-  #   in Elasticsearch 1.x. Please upgrade to ES 2.x or greater to use this feature with Logstash!
+  #   document if not already present. See the `upsert` option.
   # - A sprintf style string to change the action based on the content of the event. The value `%{[foo]}`
   #   would use the foo field for the action
   #
@@ -148,7 +124,7 @@ class LogStash::Outputs::ElasticSearch < LogStash::Outputs::Base
 
   config :document_type,
     :validate => :string,
-    :deprecated => "Document types are being deprecated in Elasticsearch 6.0, and removed entirely in 7.0. You should avoid this feature"
+    :deprecated => "Document types were deprecated in Elasticsearch 7.0, and no longer configurable since 8.0. You should avoid this feature."
 
   # From Logstash 1.3 onwards, a template is applied to Elasticsearch during
   # Logstash's startup if one with the name `template_name` does not already exist.
@@ -483,7 +459,7 @@ def event_action_tuple(event)
         join_value = event.get(@join_field)
         parent_value = event.sprintf(@parent)
         event.set(@join_field, { "name" => join_value, "parent" => parent_value })
-        params[routing_field_name] = event.sprintf(@parent)
+        params[:routing] = event.sprintf(@parent)
       else
         params[:parent] = event.sprintf(@parent)
       end
@@ -495,7 +471,7 @@ def event_action_tuple(event)
     if action == 'update'
       params[:_upsert] = LogStash::Json.load(event.sprintf(@upsert)) if @upsert != ""
       params[:_script] = event.sprintf(@script) if @script != ""
-      params[retry_on_conflict_action_name] = @retry_on_conflict
+      params[:retry_on_conflict] = @retry_on_conflict
     end
 
     event_control = event.get("[@metadata][_ingest_document]")
@@ -552,7 +528,7 @@ def common_event_params(event)
     params = {
         :_id => resolve_document_id(event, event_id),
         :_index => resolve_index!(event, event_index),
-        routing_field_name => resolve_routing(event, event_routing)
+        :routing => resolve_routing(event, event_routing)
     }
 
     target_pipeline = resolve_pipeline(event, event_pipeline)
@@ -615,16 +591,7 @@ def resolve_pipeline(event, event_pipeline)
     require "logstash/outputs/elasticsearch/#{name}"
   end
 
-  def retry_on_conflict_action_name
-    maximum_seen_major_version >= 7 ? :retry_on_conflict : :_retry_on_conflict
-  end
-
-  def routing_field_name
-    :routing
-  end
-
   # Determine the correct value for the 'type' field for the given event
-  DEFAULT_EVENT_TYPE_ES6 = "doc".freeze
   DEFAULT_EVENT_TYPE_ES7 = "_doc".freeze
 
   def get_event_type(event)
@@ -633,9 +600,7 @@ def get_event_type(event)
              event.sprintf(@document_type)
            else
              major_version = maximum_seen_major_version
-             if major_version == 6
-               DEFAULT_EVENT_TYPE_ES6
-             elsif major_version == 7
+             if major_version == 7
                DEFAULT_EVENT_TYPE_ES7
              else
                nil
@@ -653,9 +618,9 @@ def get_event_type(event)
   # @param noop_required_client [nil]: required `nil` for legacy reasons.
   # @return [Boolean]
   def use_event_type?(noop_required_client)
-    # always set type for ES 6
-    # for ES 7 only set it if the user defined it
-    (maximum_seen_major_version < 7) || (maximum_seen_major_version == 7 && @document_type)
+    # never use event type unless
+    # ES is 7.x and the user defined it
+    maximum_seen_major_version == 7 && @document_type
   end
 
   def install_template

lib/logstash/outputs/elasticsearch/data_stream_support.rb

@@ -127,7 +127,6 @@ def invalid_data_stream_params(params)
           value.to_s == 'true'
         when 'manage_template'
           value.to_s == 'false'
-        when 'ecs_compatibility' then true # required for LS <= 6.x
         else
           name.start_with?('data_stream_') ||
               shared_params.include?(name) ||

lib/logstash/outputs/elasticsearch/http_client/pool.rb

@@ -52,7 +52,6 @@ def initialize(original_error, url)
     ROOT_URI_PATH = '/'.freeze
     LICENSE_PATH = '/_license'.freeze
 
-    VERSION_6_TO_7 = ::Gem::Requirement.new([">= 6.0.0", "< 7.0.0"])
     VERSION_7_TO_7_14 = ::Gem::Requirement.new([">= 7.0.0", "< 7.14.0"])
 
     DEFAULT_OPTIONS = {
@@ -550,11 +549,9 @@ def elasticsearch?(response)
       return false if version_info['version'].nil?
 
       version = ::Gem::Version.new(version_info["version"]['number'])
-      return false if version < ::Gem::Version.new('6.0.0')
+      return false if version < ::Gem::Version.new('7.0.0')
 
-      if VERSION_6_TO_7.satisfied_by?(version)
-        return valid_tagline?(version_info)
-      elsif VERSION_7_TO_7_14.satisfied_by?(version)
+      if VERSION_7_TO_7_14.satisfied_by?(version)
         build_flavor = version_info["version"]['build_flavor']
         return false if build_flavor.nil? || build_flavor != 'default' || !valid_tagline?(version_info)
       else

lib/logstash/outputs/elasticsearch/ilm.rb

@@ -21,13 +21,7 @@ def ilm_in_use?
               "Serverless Elasticsearch cluster does not support Index Lifecycle Management.") if @ilm_enabled == 'auto'
             false
           elsif @ilm_enabled == 'auto'
-            if ilm_on_by_default?
-              ilm_alias_set?
-            else
-              @logger.info("ILM auto configuration (`ilm_enabled => auto` or unset) resolved to `false`."\
-                " Elasticsearch cluster is before 7.0.0, which is the minimum version required to automatically run Index Lifecycle Management")
-              false
-            end
+            ilm_alias_set?
           elsif @ilm_enabled.to_s == 'true'
             ilm_alias_set?
           else
@@ -42,10 +36,6 @@ def ilm_alias_set?
       default_index?(@index) || !default_rollover_alias?(@ilm_rollover_alias)
     end
 
-    def ilm_on_by_default?
-      maximum_seen_major_version >= 7
-    end
-
     def default_index?(index)
       index == @default_index
     end