Merge branch 'main' into cursor

Author: jsvd

.ci/Dockerfile.elasticsearch

@@ -13,7 +13,7 @@ COPY --chown=elasticsearch:elasticsearch spec/fixtures/test_certs/* $es_path/con
 
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.enabled: true" >> $es_yml; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.key: $es_path/config/test_certs/es.key" >> $es_yml; fi
-RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/es.crt" >> $es_yml; fi
+RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate: $es_path/config/test_certs/es.chain.crt" >> $es_yml; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.certificate_authorities: [ '$es_path/config/test_certs/ca.crt' ]" >> $es_yml; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] ; then echo "xpack.security.http.ssl.verification_mode: certificate" >> $es_yml; fi
 RUN if [ "$SECURE_INTEGRATION" = "true" ] && [ -n "$ES_SSL_SUPPORTED_PROTOCOLS" ] ; then echo "xpack.security.http.ssl.supported_protocols: ${ES_SSL_SUPPORTED_PROTOCOLS}" >> $es_yml; fi

.ci/docker-setup.sh

@@ -0,0 +1,11 @@
+# user_agent requires /etc/protocols, which is provided by netbase.
+# https://github.com/jruby/jruby/issues/3955
+if [ ! -f "/etc/protocols" ]; then
+  if [ $(command -v apt-get) ]; then
+    echo "installing netbase with apt-get"
+    sudo apt-get install -y netbase
+  else
+    echo "installing netbase with yum"
+    sudo yum install -y netbase
+  fi
+fi

.ci/setup.sh

@@ -4,15 +4,15 @@ import:
 before_install:
 - sudo sysctl -w vm.max_map_count=262144 # due ES bootstrap requirements
 
-env:
-- INTEGRATION=false ELASTIC_STACK_VERSION=7.x
-- INTEGRATION=false ELASTIC_STACK_VERSION=8.x
-- INTEGRATION=false ELASTIC_STACK_VERSION=7.x SNAPSHOT=true
-- INTEGRATION=false ELASTIC_STACK_VERSION=8.x SNAPSHOT=true
-- INTEGRATION=false ELASTIC_STACK_VERSION=7.16.3 MANTICORE_VERSION=0.7.1 ELASTICSEARCH_VERSION=7.15.0
-- INTEGRATION=true ELASTIC_STACK_VERSION=7.x
-- INTEGRATION=true ELASTIC_STACK_VERSION=7.x SNAPSHOT=true LOG_LEVEL=info
-- INTEGRATION=true ELASTIC_STACK_VERSION=8.x SNAPSHOT=true LOG_LEVEL=info
-- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x LOG_LEVEL=info
-- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.x ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3
-- SECURE_INTEGRATION=true INTEGRATION=true ELASTIC_STACK_VERSION=7.16.3 MANTICORE_VERSION=0.7.1 ELASTICSEARCH_VERSION=7.14.1 LOG_LEVEL=info
+jobs:
+  include:
+    - stage: "Integration Tests"
+      env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.previous
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.next
+    - env: INTEGRATION=true SNAPSHOT=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.future
+    - stage: "Secure Integration Tests"
+      env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=8.current SNAPSHOT=true
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current
+    - env: SECURE_INTEGRATION=true INTEGRATION=true LOG_LEVEL=info ELASTIC_STACK_VERSION=7.current ES_SSL_SUPPORTED_PROTOCOLS=TLSv1.3

.travis.yml

@@ -1,3 +1,25 @@
+## 5.0.1
+  - Fix: prevent plugin crash when hits contain illegal structure [#218](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/218)
+    - When a hit cannot be converted to an event, the input now emits an event tagged with `_elasticsearch_input_failure` with an `[event][original]` containing a JSON-encoded string representation of the entire hit.
+
+## 5.0.0
+  - SSL settings that were marked deprecated in version `4.17.0` are now marked obsolete, and will prevent the plugin from starting.
+  - These settings are:
+    - `ssl`, which should bre replaced by `ssl_enabled`
+    - `ca_file`, which should bre replaced by `ssl_certificate_authorities`
+    - `ssl_certificate_verification`, which should bre replaced by `ssl_verification_mode`
+    - [#213](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/213)
+  - Add support for custom headers [#207](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/207)
+
+## 4.20.5
+  - Add `x-elastic-product-origin` header to Elasticsearch requests [#211](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/211)
+
+## 4.20.4
+  - Fix issue where the `index` parameter was being ignored when using `response_type => aggregations` [#209](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/209)
+
+## 4.20.3
+  - [DOC] Update link to bypass redirect, resolving directly to correct content  [#206](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/206)
+
 ## 4.20.2
   - fix case when aggregation returns an error [#204](https://github.com/logstash-plugins/logstash-input-elasticsearch/pull/204)
 

CHANGELOG.md

@@ -23,7 +23,7 @@ include::{include_path}/plugin_header.asciidoc[]
 
 Read from an Elasticsearch cluster, based on search query results.
 This is useful for replaying test logs, reindexing, etc.
-You can periodically schedule ingestion using a cron syntax 
+You can periodically schedule ingestion using a cron syntax
 (see `schedule` setting) or run the query one time to load
 data into Logstash.
 
@@ -93,10 +93,23 @@ The plugin logs a warning when ECS is enabled and `target` isn't set.
 
 TIP: Set the `target` option to avoid potential schema conflicts.
 
+[id="plugins-{type}s-{plugin}-failure-handling"]
+==== Failure handling
+
+When this input plugin cannot create a structured `Event` from a hit result, it will instead create an `Event` that is tagged with `_elasticsearch_input_failure` whose `[event][original]` is a JSON-encoded string representation of the entire hit.
+
+Common causes are:
+
+ - When the hit result contains top-level fields that are {logstash-ref}/processing.html#reserved-fields[reserved in Logstash] but do not have the expected shape. Use the <<plugins-{type}s-{plugin}-target>> directive to avoid conflicts with the top-level namespace.
+ - When <<plugins-{type}s-{plugin}-docinfo>> is enabled and the docinfo fields cannot be merged into the hit result. Combine <<plugins-{type}s-{plugin}-target>> and <<plugins-{type}s-{plugin}-docinfo_target>> to avoid conflict.
+
 [id="plugins-{type}s-{plugin}-options"]
 ==== Elasticsearch Input configuration options
 
-This plugin supports the following configuration options plus the <<plugins-{type}s-{plugin}-common-options>> and the <<plugins-{type}s-{plugin}-deprecated-options>> described later.
+This plugin supports these configuration options plus the <<plugins-{type}s-{plugin}-common-options>> described later.
+
+NOTE: As of version `5.0.0` of this plugin, a number of previously deprecated settings related to SSL have been removed.
+Please check out <<plugins-{type}s-{plugin}-obsolete-options>> for details.
 
 [cols="<,<,<",options="header",]
 |=======================================================================
@@ -106,6 +119,7 @@ This plugin supports the following configuration options plus the <<plugins-{typ
 | <<plugins-{type}s-{plugin}-cloud_auth>> |<<password,password>>|No
 | <<plugins-{type}s-{plugin}-cloud_id>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-connect_timeout_seconds>> | <<number,number>>|No
+| <<plugins-{type}s-{plugin}-custom_headers>> |<<hash,hash>>|No
 | <<plugins-{type}s-{plugin}-docinfo>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-docinfo_fields>> |<<array,array>>|No
 | <<plugins-{type}s-{plugin}-docinfo_target>> |<<string,string>>|No
@@ -199,8 +213,18 @@ For more info, check out the
 The maximum amount of time, in seconds, to wait while establishing a connection to Elasticsearch.
 Connect timeouts tend to occur when Elasticsearch or an intermediate proxy is overloaded with requests and has exhausted its connection pool.
 
+[id="plugins-{type}s-{plugin}-custom_headers"]
+===== `custom_headers`
+
+  * Value type is <<hash,hash>>
+  * Default value is empty
+
+Pass a set of key value pairs as the headers sent in each request to an elasticsearch node.
+The headers will be used for any kind of request.
+These custom headers will override any headers previously set by the plugin such as the User Agent or Authorization headers.
+
 [id="plugins-{type}s-{plugin}-docinfo"]
-===== `docinfo` 
+===== `docinfo`
 
   * Value type is <<boolean,boolean>>
   * Default value is `false`
@@ -251,7 +275,7 @@ Example
 
 
 [id="plugins-{type}s-{plugin}-docinfo_fields"]
-===== `docinfo_fields` 
+===== `docinfo_fields`
 
   * Value type is <<array,array>>
   * Default value is `["_index", "_type", "_id"]`
@@ -262,7 +286,7 @@ option lists the metadata fields to save in the current event. See
 more information.
 
 [id="plugins-{type}s-{plugin}-docinfo_target"]
-===== `docinfo_target` 
+===== `docinfo_target`
 
   * Value type is <<string,string>>
   * Default value depends on whether <<plugins-{type}s-{plugin}-ecs_compatibility>> is enabled:
@@ -286,7 +310,7 @@ this option names the field under which to store the metadata fields as subfield
 Controls this plugin's compatibility with the {ecs-ref}[Elastic Common Schema (ECS)].
 
 [id="plugins-{type}s-{plugin}-hosts"]
-===== `hosts` 
+===== `hosts`
 
   * Value type is <<array,array>>
   * There is no default value for this setting.
@@ -296,18 +320,18 @@ can be either IP, HOST, IP:port, or HOST:port. The port defaults to
 9200.
 
 [id="plugins-{type}s-{plugin}-index"]
-===== `index` 
+===== `index`
 
   * Value type is <<string,string>>
   * Default value is `"logstash-*"`
 
-The index or alias to search. See {ref}/multi-index.html[Multi Indices
-documentation] in the Elasticsearch documentation for more information on how to
-reference multiple indices.
-
+The index or alias to search.
+Check out {ref}/api-conventions.html#api-multi-index[Multi Indices
+documentation] in the Elasticsearch documentation for info on
+referencing multiple indices.
 
 [id="plugins-{type}s-{plugin}-password"]
-===== `password` 
+===== `password`
 
   * Value type is <<password,password>>
   * There is no default value for this setting.
@@ -327,7 +351,7 @@ An empty string is treated as if proxy was not set, this is useful when using
 environment variables e.g. `proxy => '${LS_PROXY:}'`.
 
 [id="plugins-{type}s-{plugin}-query"]
-===== `query` 
+===== `query`
 
   * Value type is <<string,string>>
   * Default value is `'{ "sort": [ "_doc" ] }'`
@@ -375,7 +399,7 @@ The default is 0 (no retry). This value should be equal to or greater than zero.
 NOTE: Partial failures - such as errors in a subset of all slices - can result in the entire query being retried, which can lead to duplication of data. Avoiding this would require Logstash to store the entire result set of a query in memory which is often not possible.
 
 [id="plugins-{type}s-{plugin}-schedule"]
-===== `schedule` 
+===== `schedule`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -387,7 +411,7 @@ There is no schedule by default. If no schedule is given, then the statement is
 exactly once.
 
 [id="plugins-{type}s-{plugin}-scroll"]
-===== `scroll` 
+===== `scroll`
 
   * Value type is <<string,string>>
   * Default value is `"1m"`
@@ -410,7 +434,7 @@ The query requires at least one `sort` field, as described in the <<plugins-{typ
 `scroll` uses {ref}/paginate-search-results.html#scroll-search-results[scroll] API to search, which is no longer recommended.
 
 [id="plugins-{type}s-{plugin}-size"]
-===== `size` 
+===== `size`
 
   * Value type is <<number,number>>
   * Default value is `1000`
@@ -478,6 +502,8 @@ Enable SSL/TLS secured communication to Elasticsearch cluster.
 Leaving this unspecified will use whatever scheme is specified in the URLs listed in <<plugins-{type}s-{plugin}-hosts>> or extracted from the <<plugins-{type}s-{plugin}-cloud_id>>.
 If no explicit protocol is specified plain HTTP will be used.
 
+When not explicitly set, SSL will be automatically enabled if any of the specified hosts use HTTPS.
+
 [id="plugins-{type}s-{plugin}-ssl_key"]
 ===== `ssl_key`
   * Value type is <<path,path>>
@@ -598,7 +624,7 @@ It is also possible to target an entry in the event's metadata, which will be av
 
 
 [id="plugins-{type}s-{plugin}-user"]
-===== `user` 
+===== `user`
 
   * Value type is <<string,string>>
   * There is no default value for this setting.
@@ -608,56 +634,21 @@ option when authenticating to the Elasticsearch server. If set to an
 empty string authentication will be disabled.
 
 
-[id="plugins-{type}s-{plugin}-deprecated-options"]
-==== Elasticsearch Input deprecated configuration options
+[id="plugins-{type}s-{plugin}-obsolete-options"]
+==== Elasticsearch Input Obsolete Configuration Options
 
-This plugin supports the following deprecated configurations.
+WARNING: As of version `5.0.0` of this plugin, some configuration options have been replaced.
+The plugin will fail to start if it contains any of these obsolete options.
 
-WARNING: Deprecated options are subject to removal in future releases.
 
-[cols="<,<,<",options="header",]
+[cols="<,<",options="header",]
 |=======================================================================
-|Setting|Input type|Replaced by
-| <<plugins-{type}s-{plugin}-ca_file>> |a valid filesystem path|<<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
-| <<plugins-{type}s-{plugin}-ssl>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_enabled>>
-| <<plugins-{type}s-{plugin}-ssl_certificate_verification>> |<<boolean,boolean>>|<<plugins-{type}s-{plugin}-ssl_verification_mode>>
+|Setting|Replaced by
+| ca_file | <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>
+| ssl | <<plugins-{type}s-{plugin}-ssl_enabled>>
+| ssl_certificate_verification | <<plugins-{type}s-{plugin}-ssl_verification_mode>>
 |=======================================================================
 
-[id="plugins-{type}s-{plugin}-ca_file"]
-===== `ca_file`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_certificate_authorities>>]
-
-* Value type is <<path,path>>
-* There is no default value for this setting.
-
-SSL Certificate Authority file in PEM encoded format, must also include any chain certificates as necessary.
-
-[id="plugins-{type}s-{plugin}-ssl"]
-===== `ssl`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_enabled>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `false`
-
-If enabled, SSL will be used when communicating with the Elasticsearch
-server (i.e. HTTPS will be used instead of plain HTTP).
-
-
-[id="plugins-{type}s-{plugin}-ssl_certificate_verification"]
-===== `ssl_certificate_verification`
-deprecated[4.17.0, Replaced by <<plugins-{type}s-{plugin}-ssl_verification_mode>>]
-
-* Value type is <<boolean,boolean>>
-* Default value is `true`
-
-Option to validate the server's certificate. Disabling this severely compromises security.
-When certificate validation is disabled, this plugin implicitly trusts the machine
-resolved at the given address without validating its proof-of-identity.
-In this scenario, the plugin can transmit credentials to or process data from an untrustworthy
-man-in-the-middle or other compromised infrastructure.
-More information on the importance of certificate verification:
-**https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf**.
-
 [id="plugins-{type}s-{plugin}-common-options"]
 include::{include_path}/{type}.asciidoc[]