diff --git a/docs/plugins/outputs/elasticsearch.asciidoc b/docs/plugins/outputs/elasticsearch.asciidoc index 9c7816d5..b598cd28 100644 --- a/docs/plugins/outputs/elasticsearch.asciidoc +++ b/docs/plugins/outputs/elasticsearch.asciidoc @@ -66,9 +66,10 @@ Set the value to port :443 instead. For more info on sending data from {ls} to {es-serverless}, check out the {serverless-docs}/elasticsearch/what-is-elasticsearch-serverless[{es-serverless} docs]. [id="plugins-{type}s-{plugin}-ess"] -==== Hosted {es} Service on Elastic Cloud +==== {ls} to {ech} -{ess-leadin} +You can run Elasticsearch on your own hardware or use Elastic Cloud Hosted, available on AWS, GCP, and Azure. +Try Elastic Cloud Hosted for free: https://cloud.elastic.co/registration. ==== Compatibility with the Elastic Common Schema (ECS) @@ -197,7 +198,22 @@ This plugin uses the Elasticsearch bulk API to optimize its imports into Elastic either partial or total failures. The bulk API sends batches of requests to an HTTP endpoint. Error codes for the HTTP request are handled differently than error codes for individual documents. -HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely. + +HTTP requests to the bulk API are expected to return a 200 response code. All other response codes are retried indefinitely, +including 413 (Payload Too Large) responses. + +If you want to handle large payloads differently, you can configure 413 responses to go to the Dead Letter Queue instead: + +[source,ruby] +----- +output { + elasticsearch { + hosts => ["localhost:9200"] + dlq_custom_codes => [413] # Send 413 errors to DLQ instead of retrying + } +----- + +This will capture oversized payloads in the DLQ for analysis rather than retrying them. The following document errors are handled as follows: @@ -326,8 +342,10 @@ When a string value on an event contains one or more byte sequences that are not [id="plugins-{type}s-{plugin}-options"] ==== Elasticsearch Output Configuration Options -This plugin supports the following configuration options plus the -<> and the <> described later. +This plugin supports these configuration options plus the <> described later. + +NOTE: As of version 12.0.0 of this plugin, a number of previously deprecated SSL settings have been removed. +Please check out <> for details. [cols="<,<,<",options="header",] |======================================================================= @@ -442,7 +460,7 @@ For more details on actions, check out the {ref}/docs-bulk.html[Elasticsearch bu * There is no default value for this setting. Authenticate using Elasticsearch API key. -Note that this option also requires SSL/TLS, which can be enabled by supplying a <>, a list of HTTPS <>, or by setting < true`>>. +Note that this option also requires SSL/TLS, which can be enabled by supplying a <>, a list of HTTPS <>, or by setting < true`>>. Format is `id:api_key` where `id` and `api_key` are as returned by the Elasticsearch {ref}/security-api-create-api-key.html[Create API key API]. @@ -611,8 +629,7 @@ Elasticsearch with the same ID. NOTE: This option is deprecated due to the https://www.elastic.co/guide/en/elasticsearch/reference/6.0/removal-of-types.html[removal -of types in Elasticsearch 6.0]. It will be removed in the next major version of -Logstash. +of types in Elasticsearch 6.0]. NOTE: This value is ignored and has no effect for Elasticsearch clusters `8.x`. @@ -621,9 +638,7 @@ similar events to the same 'type'. String expansion `%{foo}` works here. If you don't set a value for this option: - for elasticsearch clusters 8.x: no value will be used; -- for elasticsearch clusters 7.x: the value of '_doc' will be used; -- for elasticsearch clusters 6.x: the value of 'doc' will be used; -- for elasticsearch clusters 5.x and below: the event's 'type' field will be used, if the field is not present the value of 'doc' will be used. +- for elasticsearch clusters 7.x: the value of '_doc' will be used. [id="plugins-{type}s-{plugin}-ecs_compatibility"] ===== `ecs_compatibility` @@ -1038,8 +1053,6 @@ NOTE: Deprecates <>. This setting asks Elasticsearch for the list of all cluster nodes and adds them to the hosts list. -For Elasticsearch 5.x and 6.x any nodes with `http.enabled` (on by default) will -be added to the hosts list, excluding master-only nodes. [id="plugins-{type}s-{plugin}-sniffing_delay"] ===== `sniffing_delay` @@ -1325,98 +1338,24 @@ https://www.elastic.co/blog/elasticsearch-versioning-support[versioning support blog] and {ref}/docs-index_.html#_version_types[Version types] in the Elasticsearch documentation. -[id="plugins-{type}s-{plugin}-deprecated-options"] -==== Elasticsearch Output Deprecated Configuration Options - -This plugin supports the following deprecated configurations. +[id="plugins-{type}s-{plugin}-obsolete-options"] +==== Elasticsearch Output Obsolete Configuration Options -WARNING: Deprecated options are subject to removal in future releases. +WARNING: As of version `12.0.0` of this plugin, some configuration options have been replaced. +The plugin will fail to start if it contains any of these obsolete options. -[cols="<,<,<",options="header",] +[cols="<,<",options="header",] |======================================================================= -|Setting|Input type|Replaced by -| <> |a valid filesystem path|<> -| <> |a valid filesystem path|<> -| <> |<>|<> -| <> |<>|<> -| <> |<>|<> -| <> |a valid filesystem path|<> -| <> |<>|<> +|Setting|Replaced by +| cacert | <> +| keystore | <> +| keystore_password | <> +| ssl | <> +| ssl_certificate_verification | <> +| truststore | <> +| truststore_password | <> |======================================================================= - -[id="plugins-{type}s-{plugin}-cacert"] -===== `cacert` -deprecated[11.14.0, Replaced by <>] - -* Value type is a list of <> -* There is no default value for this setting. - -The .cer or .pem file to validate the server's certificate. - -[id="plugins-{type}s-{plugin}-keystore"] -===== `keystore` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -The keystore used to present a certificate to the server. -It can be either .jks or .p12 - -NOTE: You cannot use this setting and <> at the same time. - -[id="plugins-{type}s-{plugin}-keystore_password"] -===== `keystore_password` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -Set the keystore password - -[id="plugins-{type}s-{plugin}-ssl"] -===== `ssl` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -Enable SSL/TLS secured communication to Elasticsearch cluster. -Leaving this unspecified will use whatever scheme is specified in the URLs listed in <> or extracted from the <>. -If no explicit protocol is specified plain HTTP will be used. - -[id="plugins-{type}s-{plugin}-ssl_certificate_verification"] -===== `ssl_certificate_verification` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* Default value is `true` - -Option to validate the server's certificate. Disabling this severely compromises security. -For more information on disabling certificate verification please read -https://www.cs.utexas.edu/~shmat/shmat_ccs12.pdf - -[id="plugins-{type}s-{plugin}-truststore"] -===== `truststore` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -The truststore to validate the server's certificate. -It can be either `.jks` or `.p12`. -Use either `:truststore` or `:cacert`. - -[id="plugins-{type}s-{plugin}-truststore_password"] -===== `truststore_password` -deprecated[11.14.0, Replaced by <>] - -* Value type is <> -* There is no default value for this setting. - -Set the truststore password - [id="plugins-{type}s-{plugin}-common-options"] include::{include_path}/{type}.asciidoc[]