From e2bf45094c6919cfee081c76a4748d89b15c067f Mon Sep 17 00:00:00 2001 From: logstashmachine <43502315+logstashmachine@users.noreply.github.com> Date: Sat, 28 Jun 2025 06:04:18 +0000 Subject: [PATCH] updated versioned plugin docs --- .../elastic_integration-index.asciidoc | 6 + .../elastic_integration-v8.17.2.asciidoc | 568 ++++++++++++++++ .../elastic_integration-v8.18.1.asciidoc | 634 ++++++++++++++++++ .../elastic_integration-v9.0.1.asciidoc | 634 ++++++++++++++++++ 4 files changed, 1842 insertions(+) create mode 100644 docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc create mode 100644 docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc create mode 100644 docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc diff --git a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc index c7a60ec5..19b52c7f 100644 --- a/docs/versioned-plugins/filters/elastic_integration-index.asciidoc +++ b/docs/versioned-plugins/filters/elastic_integration-index.asciidoc @@ -5,8 +5,11 @@ include::{include_path}/version-list-intro.asciidoc[] |======================================================================= | Version | Release Date +| <> | 2025-06-28 | <> | 2025-04-28 +| <> | 2025-06-28 | <> | 2025-04-17 +| <> | 2025-06-28 | <> | 2025-01-28 | <> | 2025-01-08 | <> | 2025-01-28 @@ -33,8 +36,11 @@ include::{include_path}/version-list-intro.asciidoc[] | <> | 2023-04-14 |======================================================================= +include::elastic_integration-v9.0.1.asciidoc[] include::elastic_integration-v9.0.0.asciidoc[] +include::elastic_integration-v8.18.1.asciidoc[] include::elastic_integration-v8.18.0.asciidoc[] +include::elastic_integration-v8.17.2.asciidoc[] include::elastic_integration-v8.17.1.asciidoc[] include::elastic_integration-v8.17.0.asciidoc[] include::elastic_integration-v8.16.1.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc new file mode 100644 index 00000000..9ab21960 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.17.2.asciidoc @@ -0,0 +1,568 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.17.2 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.17.2/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `communityid` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `dateindexname` | _none_ +| `dissect` | _none_ +| `dotexpander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `htmlstrip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `keyvalue` | _none_ +| `lowercase` | _none_ +| `networkdirection` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registereddomain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc new file mode 100644 index 00000000..eadc56e5 --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v8.18.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v8.18.1 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v8.18.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[] diff --git a/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc b/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc new file mode 100644 index 00000000..56b1990d --- /dev/null +++ b/docs/versioned-plugins/filters/elastic_integration-v9.0.1.asciidoc @@ -0,0 +1,634 @@ +:plugin: elastic_integration +:type: filter + +/////////////////////////////////////////// +START - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// +:version: v9.0.1 +:release_date: 2025-06-28 +:changelog_url: https://github.com/elastic/logstash-filter-elastic_integration/blob/v9.0.1/CHANGELOG.md +:include_path: ../include/6.x +/////////////////////////////////////////// +END - GENERATED VARIABLES, DO NOT EDIT! +/////////////////////////////////////////// + +:elastic-integration-name: Elastic Integration + +[id="{version}-plugins-{type}s-{plugin}"] + +=== {elastic-integration-name} filter plugin {version} + +include::{include_path}/plugin_header-nonstandard.asciidoc[] + +.Elastic Enterprise License +**** +Use of this plugin requires an active Elastic Enterprise https://www.elastic.co/subscriptions[subscription]. +**** + +==== Description + +Use this filter to process Elastic integrations powered by {es} Ingest Node in {ls}. + +.Extending Elastic integrations with {ls} +**** +This plugin can help you take advantage of the extensive, built-in capabilities of {integrations-docs}[Elastic {integrations}]—​such as managing data collection, +transformation, and visualization—​and then use {ls} for additional data processing and output options. +For more info about extending Elastic integrations with {ls}, check out {logstash-ref}/ea-integrations.html[Using {ls} with Elastic Integrations]. +**** + +When you configure this filter to point to an {es} cluster, it detects which ingest pipeline (if any) should be executed for each event, +using an explicitly-defined <<{version}-plugins-{type}s-{plugin}-pipeline_name>> or auto-detecting the event's data-stream and its default pipeline. + +It then loads that pipeline's definition from {es} and run that pipeline inside Logstash without transmitting the event to {es}. +Events that are successfully handled by their ingest pipeline will have `[@metadata][target_ingest_pipeline]` set to `_none` so that any downstream {es} output in the Logstash pipeline will avoid running the event's default pipeline _again_ in {es}. + +NOTE: Some multi-pipeline configurations such as logstash-to-logstash over http(s) do not maintain the state of `[@metadata]` fields. + In these setups, you may need to explicitly configure your downstream pipeline's {es} output with `pipeline => "_none"` to avoid re-running the default pipeline. + +Events that _fail_ ingest pipeline processing will be tagged with `_ingest_pipeline_failure`, and their `[@metadata][_ingest_pipeline_failure]` will be populated with details as a key/value map. + +[id="{version}-plugins-{type}s-{plugin}-requirements"] +===== Requirements and upgrade guidance + +- This plugin requires Java 17 minimum with {ls} `8.x` versions and Java 21 minimum with {ls} `9.x` versions. +- When you upgrade the {stack}, upgrade {ls} (or this plugin specifically) _before_ you upgrade {kib}. + (Note that this requirement is a departure from the typical {stack} https://www.elastic.co/guide/en/elastic-stack/current/installing-elastic-stack.html#install-order-elastic-stack[installation order].) ++ +The {es}-{ls}-{kib} installation order recommended here ensures the best experience with {agent}-managed pipelines, and embeds functionality from a version of {es} Ingest Node that is compatible with the plugin version (`major`.`minor`). + +[id="{version}-plugins-{type}s-{plugin}-es-tips"] +===== Using `filter-elastic_integration` with `output-elasticsearch` + +Elastic {integrations} are designed to work with {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[data streams] and {logstash-ref}/plugins-outputs-elasticsearch.html#_compatibility_with_the_elastic_common_schema_ecs[ECS-compatible] output. +Be sure that these features are enabled in the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch`] plugin. + +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data_stream[`data-stream`] to `true`. + + (Check out {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-data-streams[Data streams] for additional data streams settings.) +* Set {logstash-ref}/plugins-outputs-elasticsearch.html#plugins-outputs-elasticsearch-ecs_compatibility[`ecs-compatibility`] to `v1` or `v8`. + +Check out the {logstash-ref}/plugins-outputs-elasticsearch.html[`output-elasticsearch` plugin] docs for additional settings. + +[id="{version}-plugins-{type}s-{plugin}-minimum_configuration"] +==== Minimum configuration + +You will need to configure this plugin to connect to {es}, and may need to also need to provide local GeoIp databases. + +[source,ruby] +-------------------------------------------------- +filter { + elastic_integration { + cloud_id => "YOUR_CLOUD_ID_HERE" + cloud_auth => "YOUR_CLOUD_AUTH_HERE" + geoip_database_directory => "/etc/your/geoip-databases" + } +} +-------------------------------------------------- + +Read on for a guide to configuration, or jump to the <<{version}-plugins-{type}s-{plugin}-options, complete list of configuration options>>. + +[id="{version}-plugins-{type}s-{plugin}-connecting_to_elasticsearch"] +==== Connecting to {es} + +This plugin communicates with {es} to identify which ingest pipeline should be run for a given event, and to retrieve the ingest pipeline definitions themselves. +You must configure this plugin to point to {es} using exactly one of: + +* A Cloud Id (see <<{version}-plugins-{type}s-{plugin}-cloud_id>>) +* A list of one or more host URLs (see <<{version}-plugins-{type}s-{plugin}-hosts>>) + +Communication will be made securely over SSL unless you explicitly configure this plugin otherwise. + +You may need to configure how this plugin establishes trust of the server that responds, +and will likely need to configure how this plugin presents its own identity or credentials. + +===== SSL Trust Configuration + +When communicating over SSL, this plugin fully-validates the proof-of-identity presented by {es} using the system trust store. +You can provide an _alternate_ source of trust with one of: + +* A PEM-formatted list of trusted certificate authorities (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>>) +* A JKS- or PKCS12-formatted Keystore containing trusted certificates (see <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>) + +You can also configure which aspects of the proof-of-identity are verified (see <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>). + +===== SSL Identity Configuration + +When communicating over SSL, you can also configure this plugin to present a certificate-based proof-of-identity to the {es} cluster it connects to using one of: + +* A PKCS8 Certificate/Key pair (see <<{version}-plugins-{type}s-{plugin}-ssl_certificate>>) +* A JKS- or PKCS12-formatted Keystore (see <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>) + +===== Request Identity + +You can configure this plugin to present authentication credentials to {es} in one of several ways: + +* ApiKey: (see <<{version}-plugins-{type}s-{plugin}-api_key>>) +* Cloud Auth: (see <<{version}-plugins-{type}s-{plugin}-cloud_auth>>) +* HTTP Basic Auth: (see <<{version}-plugins-{type}s-{plugin}-username>> and <<{version}-plugins-{type}s-{plugin}-password>>) + +NOTE: Your request credentials are only as secure as the connection they are being passed over. + They provide neither privacy nor secrecy on their own, and can easily be recovered by an adversary when SSL is disabled. + +[id="{version}-plugins-{type}s-{plugin}-minimum_required_privileges"] +==== Minimum required privileges + +This plugin communicates with Elasticsearch to resolve events into pipeline definitions and needs to be configured with credentials with appropriate privileges to read from the relevant APIs. +At the startup phase, this plugin confirms that current user has sufficient privileges, including: + +[cols="<1,<1",options="header"] +|======================================================================= +| Privilege name | Description + +| `monitor` | A read-only privilege for cluster operations such as cluster health or state. Plugin requires it when checks {es} license. +| `read_pipeline` | A read-only get and simulate access to ingest pipeline. It is required when plugin reads {es} ingest pipeline definitions. +| `manage_index_templates` | All operations on index templates privilege. It is required when plugin resolves default pipeline based on event data stream name. + +|======================================================================= + +[NOTE] +-- +This plugin cannot determine if an anonymous user has the required privileges when it connects to an {es} cluster that has security features disabled or when the user does not provide credentials. +The plugin starts in an unsafe mode with a runtime error indicating that API permissions are insufficient, and prevents events from being processed by the ingest pipeline. + +To avoid these issues, set up user authentication and ensure that security in {es} is enabled (default). +-- + +[id="{version}-plugins-{type}s-{plugin}-supported_ingest_processors"] +==== Supported Ingest Processors + +This filter can run {es} Ingest Node pipelines that are _wholly_ comprised of the supported subset of processors. +It has access to the Painless and Mustache scripting engines where applicable: + +[cols="<1,<1,<4",options="header"] +|======================================================================= +|Source | Processor | Caveats +.35+h|Ingest Common + +| `append` | _none_ +| `bytes` | _none_ +| `community_id` | _none_ +| `convert` | _none_ +| `csv` | _none_ +| `date` | _none_ +| `date_index_name` | _none_ +| `dissect` | _none_ +| `dot_expander` | _none_ +| `drop` | _none_ +| `fail` | _none_ +| `fingerprint` | _none_ +| `foreach` | _none_ +| `grok` | _none_ +| `gsub` | _none_ +| `html_strip` | _none_ +| `join` | _none_ +| `json` | _none_ +| `kv` | _none_ +| `lowercase` | _none_ +| `network_direction` | _none_ +| `pipeline` | resolved pipeline _must_ be wholly-composed of supported processors +| `registered_domain` | _none_ +| `remove` | _none_ +| `rename` | _none_ +| `reroute` | _none_ +| `script` | `lang` must be `painless` (default) +| `set` | _none_ +| `sort` | _none_ +| `split` | _none_ +| `trim` | _none_ +| `uppercase` | _none_ +| `uri_parts` | _none_ +| `urldecode` | _none_ +| `user_agent` | side-loading a custom regex file is not supported; the processor will use the default user agent definitions as specified in https://www.elastic.co/guide/en/elasticsearch/reference/current/user-agent-processor.html[Elasticsearch processor definition] + +h| Redact | `redact` | _none_ + +h| GeoIp +| `geoip` | requires MaxMind GeoIP2 databases, which may be provided by Logstash's Geoip Database Management _OR_ configured using <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> + +|======================================================================= + +[id="{version}-plugins-{type}s-{plugin}-field_mappings"] +===== Field Mappings + +:esid: {es} Ingest Document + +During execution the Ingest pipeline works with a temporary mutable _view_ of the Logstash event called an ingest document. +This view contains all of the as-structured fields from the event with minimal type conversions. + +It also contains additional metadata fields as required by ingest pipeline processors: + +* `_version`: a `long`-value integer equivalent to the event's `@version`, or a sensible default value of `1`. +* `_ingest.timestamp`: a `ZonedDateTime` equivalent to the event's `@timestamp` field + +After execution completes the event is sanitized to ensure that Logstash-reserved fields have the expected shape, providing sensible defaults for any missing required fields. +When an ingest pipeline has set a reserved field to a value that cannot be coerced, the value is made available in an alternate location on the event as described below. + +[cols="<1,<1,<5",options="header"] +|======================================================================= +| {ls} field | type | value + +| `@timestamp` | `Timestamp` | +First coercible value of the ingest document's `@timestamp`, `event.created`, `_ingest.timestamp`, or `_now` fields; or the current timestamp. +When the ingest document has a value for `@timestamp` that cannot be coerced, it will be available in the event's `_@timestamp` field. + +| `@version` | String-encoded integer | +First coercible value of the ingest document's `@version`, or `_version` fields; or the current timestamp. +When the ingest document has a value for `@version` that cannot be coerced, it will be available in the event's `_@version` field. + +| `@metadata` | key/value map | +The ingest document's `@metadata`; or an empty map. +When the ingest document has a value for `@metadata` that cannot be coerced, it will be available in the event's `_@metadata` field. + +| `tags` | a String or a list of Strings | +The ingest document's `tags`. +When the ingest document has a value for `tags` that cannot be coerced, it will be available in the event's `_tags` field. +|======================================================================= + +Additionally, these {es} IngestDocument Metadata fields are made available on the resulting event _if-and-only-if_ they were set during pipeline execution: + +[cols="<1,<5",options="header"] +|======================================================================= +| {es} document metadata | {ls} field + +| `_id` | `[@metadata][_ingest_document][id]` +| `_index` | `[@metadata][_ingest_document][index]` +| `_routing` | `[@metadata][_ingest_document][routing]` +| `_version` | `[@metadata][_ingest_document][version]` +| `_version_type` | `[@metadata][_ingest_document][version_type]` +| `_ingest.timestamp` | `[@metadata][_ingest_document][timestamp]` +|======================================================================= + + +[id="{version}-plugins-{type}s-{plugin}-resolving"] +==== Resolving Pipeline Definitions + +:cached-entry-ttl: 24 hours +:cache-reload-frequency: 1 minute + +This plugin uses {es} to resolve pipeline names into their pipeline definitions. +When configured _without_ an explicit <<{version}-plugins-{type}s-{plugin}-pipeline_name>>, or when a pipeline uses the Reroute Processor, it also uses {es} to establish mappings of data stream names to their respective default pipeline names. + +It uses hit/miss caches to avoid querying Elasticsearch for every single event. +It also works to update these cached mappings _before_ they expire. +The result is that when {es} is responsive this plugin is able to pick up changes quickly without impacting its own performance, and it can survive periods of {es} issues without interruption by continuing to use potentially-stale mappings or definitions. + +To achieve this, mappings are cached for a maximum of {cached-entry-ttl}, and cached values are reloaded every {cache-reload-frequency} with the following effect: + +* when a reloaded mapping is non-empty and is the _same_ as its already-cached value, its time-to-live is reset to ensure that subsequent events can continue using the confirmed-unchanged value +* when a reloaded mapping is non-empty and is _different_ from its previously-cached value, the entry is _updated_ so that subsequent events will use the new value +* when a reloaded mapping is newly _empty_, the previous non-empty mapping is _replaced_ with a new empty entry so that subsequent events will use the empty value +* when the reload of a mapping _fails_, this plugin emits a log warning but the existing cache entry is unchanged and gets closer to its expiry. + +[id="{version}-plugins-{type}s-{plugin}-troubleshooting"] +==== Troubleshooting + +Troubleshooting ingest pipelines associated with data streams requires a pragmatic approach, involving thorough analysis and debugging techniques. +To identify the root cause of issues with pipeline execution, you need to enable debug-level logging. +The debug logs allow monitoring the plugin's behavior and help to detect issues. +The plugin operates through following phases: pipeline _resolution_, ingest pipeline _creation_, and pipeline _execution_. + +[ingest-pipeline-resolution-errors] +===== Ingest Pipeline Resolution Errors + +*Plugin does not resolve ingest pipeline associated with data stream* + +If you encounter `No pipeline resolved for event ...` messages in the debug logs, the error indicates that the plugin is unable to resolve the ingest pipeline from the data stream. +To further diagnose and resolve the issue, verify whether the data stream's index settings include a `default_pipeline` or `final_pipeline` configuration. +You can inspect the index settings by running a `POST _index_template/_simulate_index/{type}-{dataset}-{namespace}` query in the {kib} Dev Tools. +Make sure to replace `{type}-{dataset}-{namespace}` with values corresponding to your data stream. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +*Ingest pipeline does not exist* + +If you notice `pipeline not found: ...` messages in the debug logs or `Pipeline {pipeline-name} could not be loaded` warning messages, it indicates that the plugin has successfully resolved the ingest pipeline from `default_pipeline` or `final_pipeline`, but the specified pipeline does not exist. +To confirm whether pipeline exists, run a `GET _ingest/pipeline/{ingest-pipeline-name}` query in the {kib} Dev Tools console. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#pipelines-for-fleet-elastic-agent[Ingest pipelines for fleet] and {integrations-docs}[Elastic {integrations}] resources. + +[ingest-pipeline-creation-errors] +===== Ingest Pipeline Creation Errors + +If you encounter `failed to create ingest pipeline {pipeline-name} from pipeline configuration` error messages, it indicates that the plugin is unable to create an ingest pipeline from the resolved pipeline configuration. +This issue typically arises when the pipeline configuration contains unsupported or invalid processor(s) that the plugin cannot execute. +In such situations, the log output includes information about the issue. +For example, the following error message indicating `inference` processor in the pipeline configuration which is not supported processor type. + + [source] + ---- + 2025-01-21 12:29:13 [2025-01-21T20:29:13,986][ERROR][co.elastic.logstash.filters.elasticintegration.IngestPipelineFactory][main] failed to create ingest pipeline logs-my.custom-1.0.0 from pipeline configuration + 2025-01-21 12:29:13 org.elasticsearch.ElasticsearchParseException: No processor type exists with name [inference] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.newConfigurationException(ConfigurationUtils.java:470) ~[logstash-filter-elastic_integration-0.1.16.jar:?] + 2025-01-21 12:29:13 at org.elasticsearch.ingest.ConfigurationUtils.readProcessor(ConfigurationUtils.java:635) + ---- + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[ingest-pipeline-execution-errors] +===== Ingest Pipeline Execution Errors + +These errors typically fall into two main categories, each requiring specific investigation and resolution steps: + +*Logstash catches issues while running ingest pipelines* + +When errors occur during the execution of ingest pipelines, {ls} attaches the `_ingest_pipeline_failure` tag to the event, making it easier to identify and investigate problematic events. +The detailed logs are available in the {ls} logs for your investigation. +The root cause may depend on configuration, environment or integration you are running. +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +*Errors internally occurred in the ingest pipeline* + +If an ingest pipeline is configured with `on_failure` conditions, failures during pipeline execution are internally handled by the ingest pipeline itself and not be visible to {ls}. +This means that errors are captured and processed within the pipeline, rather than being passed to {ls} for logging or tagging. +To identify and analyze such cases, go to the {kib} -> Stack Management -> Ingest pipelines and find the ingest pipeline you are using. +Click on it and navigate to the _Failure processors_ section. If processors are configured, they may specify which field contains the failure details. +For example, the pipeline might store error information in a `error.message` field or a custom field defined in the _Failure processors_ configuration. +Go to the {kib} Dev Tools and search for the data (`GET {index-ingest-pipeline-is-writing}/_search`) and look for the fields mentioned in the failure processors . +The fields have error details which help you to analyze the root cause. + +For further guidance, we recommend exploring {fleet-guide}/integrations.html[Manage Elastic Agent Integrations], {es} {ref}/ingest.html#handling-pipeline-failures[Handling pipeline failures] resources. + +[id="{version}-plugins-{type}s-{plugin}-options"] +==== {elastic-integration-name} Filter Configuration Options + +This plugin supports the following configuration options plus the <<{version}-plugins-{type}s-{plugin}-common-options>> described later. + +[cols="<,<,<",options="header",] +|======================================================================= +|Setting |Input type|Required +| <<{version}-plugins-{type}s-{plugin}-api_key>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_auth>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-cloud_id>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-geoip_database_directory>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-hosts>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-pipeline_name>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +| <<{version}-plugins-{type}s-{plugin}-proxy>> | {logstash-ref}/configuration-file-structure.html#uri[uri]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities>> |{logstash-ref}/configuration-file-structure.html#array[array]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_enabled>> | {logstash-ref}/configuration-file-structure.html#boolean[boolean]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> | {logstash-ref}/configuration-file-structure.html#path[path]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> | {logstash-ref}/configuration-file-structure.html#password[password]|No +| <<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>> | {logstash-ref}/configuration-file-structure.html#string[string], one of `["full", "certificate", "none"]`|No +| <<{version}-plugins-{type}s-{plugin}-username>> | {logstash-ref}/configuration-file-structure.html#string[string]|No +|======================================================================= + +// Variables for re-use in per-option docs +:prohibit-ssl-disabled-effective: Cannot be combined with configurations that disable SSL +:prohibit-ssl-disabled-explicit: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false`. +:prohibit-ssl-verify-none: Cannot be combined with `<<{version}-plugins-{type}s-{plugin}-ssl_verification_mode>>=>none`. + +[id="{version}-plugins-{type}s-{plugin}-api_key"] +===== `api_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +The encoded form of an API key that is used to authenticate this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-cloud_auth"] +===== `cloud_auth` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. + +Cloud authentication string (":" format) is an alternative +for the `username`/`password` pair and can be obtained from Elastic Cloud web console. + +[id="{version}-plugins-{type}s-{plugin}-cloud_id"] +===== `cloud_id` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-explicit} + +Cloud Id, from the Elastic Cloud web console. + +When connecting with a Cloud Id, communication to {es} is secured with SSL. + +For more details, check out the +{logstash-ref}/connecting-to-cloud.html[Logstash-to-Cloud documentation]. + +[id="{version}-plugins-{type}s-{plugin}-geoip_database_directory"] +===== `geoip_database_directory` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. + +When running in a Logstash process that has Geoip Database Management enabled, integrations that use the Geoip Processor wil use managed Maxmind databases by default. +By using managed databases you accept and agree to the https://www.maxmind.com/en/geolite2/eula[MaxMind EULA]. + +You may instead configure this plugin with the path to a local directory containing database files. + +This plugin will discover all regular files with the `.mmdb` suffix in the provided directory, and make each available by its file name to the GeoIp processors in integration pipelines. +It expects the files it finds to be in the MaxMind DB format with one of the following database types: + +* `AnonymousIp` +* `ASN` +* `City` +* `Country` +* `ConnectionType` +* `Domain` +* `Enterprise` +* `Isp` + +[NOTE] +==== +Most integrations rely on databases being present named _exactly_: + +* `GeoLite2-ASN.mmdb`, +* `GeoLite2-City.mmdb`, or +* `GeoLite2-Country.mmdb` +==== + +[id="{version}-plugins-{type}s-{plugin}-hosts"] +===== `hosts` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#uri[uri]s +* There is no default value for this setting. +* Constraints: +** When any URL contains a protocol component, all URLs must have the same protocol as each other. +** `https`-protocol hosts use HTTPS and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => false`>>. +** `http`-protocol hosts use unsecured HTTP and cannot be combined with <<{version}-plugins-{type}s-{plugin}-ssl_enabled, `ssl_enabled => true`>>. +** When any URL omits a port component, the default `9200` is used. +** When any URL contains a path component, all URLs must have the same path as each other. + +A non-empty list of {es} hosts to connect. + +Examples: + +- `"127.0.0.1"` +- `["127.0.0.1:9200","127.0.0.2:9200"]` +- `["http://127.0.0.1"]` +- `["https://127.0.0.1:9200"]` +- `["https://127.0.0.1:9200/subpath"]` (If using a proxy on a subpath) + +When connecting with a list of hosts, communication to {es} is secured with SSL unless configured otherwise. + +[WARNING] +.Disabling SSL is dangerous +============ +The security of this plugin relies on SSL to avoid leaking credentials and to avoid running illegitimate ingest pipeline definitions. + +There are two ways to disable SSL: + +* Provide a list of `http`-protocol hosts +* Set `<<{version}-plugins-{type}s-{plugin}-ssl_enabled>>=>false` + +============ + +[id="{version}-plugins-{type}s-{plugin}-password"] +===== `password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when request auth is configured with <<{version}-plugins-{type}s-{plugin}-username>> + +A password when using HTTP Basic Authentication to connect to {es}. + +[id="{version}-plugins-{type}s-{plugin}-pipeline_name"] +===== `pipeline_name` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, the event's initial pipeline will _not_ be auto-detected from the event's data stream fields. +* Value may be a {logstash-ref}/event-dependent-configuration.html#sprintf[sprintf-style] template; if any referenced fields cannot be resolved the event will not be routed to an ingest pipeline. + +[id="{version}-plugins-{type}s-{plugin}-proxy"] +===== `proxy` + +* Value type is {logstash-ref}/configuration-file-structure.html#uri[uri] +* There is no default value for this setting. + +Address of the HTTP forward proxy used to connect to the {es} cluster. +An empty string is treated as if proxy was not set. +Environment variables may be used to set this value, e.g. `proxy => '${LS_PROXY:}'`. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate"] +===== `ssl_certificate` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_key>> and <<{version}-plugins-{type}s-{plugin}-ssl_key_passphrase>> are also required. +* {prohibit-ssl-disabled-effective} + +Path to a PEM-encoded certificate or certificate chain with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_certificate_authorities"] +===== `ssl_certificate_authorities` + +* Value type is a list of {logstash-ref}/configuration-file-structure.html#path[path]s +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +One or more PEM-formatted files defining certificate authorities. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_enabled"] +===== `ssl_enabled` + +* Value type is {logstash-ref}/configuration-file-structure.html#boolean[boolean] +* There is no default value for this setting. + +Secure SSL communication to {es} is enabled unless: + +* it is explicitly disabled with `ssl_enabled => false`; OR +* it is implicitly disabled by providing `http`-protocol <<{version}-plugins-{type}s-{plugin}-hosts>>. + +Specifying `ssl_enabled => true` can be a helpful redundant safeguard to ensure this plugin cannot be configured to use non-ssl communication. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key"] +===== `ssl_key` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A path to a PKCS8-formatted SSL certificate key. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_password"] +===== `ssl_keystore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_keystore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_keystore_path"] +===== `ssl_keystore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_keystore_password>> is also required. +* {prohibit-ssl-disabled-effective} + +A path to a JKS- or PKCS12-formatted keystore with which to identify this plugin to {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_key_passphrase"] +===== `ssl_key_passphrase` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection identity is configured with <<{version}-plugins-{type}s-{plugin}-ssl_certificate>> +* {prohibit-ssl-disabled-effective} + +A password or passphrase of the <<{version}-plugins-{type}s-{plugin}-ssl_key>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_path"] +===== `ssl_truststore_path` + +* Value type is {logstash-ref}/configuration-file-structure.html#path[path] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-ssl_truststore_password>> is required. +* {prohibit-ssl-disabled-effective} +* {prohibit-ssl-verify-none} + +A path to JKS- or PKCS12-formatted keystore where trusted certificates are located. + +This setting can be used to _override_ the system trust store for verifying the SSL certificate presented by {es}. + +[id="{version}-plugins-{type}s-{plugin}-ssl_truststore_password"] +===== `ssl_truststore_password` + +* Value type is {logstash-ref}/configuration-file-structure.html#password[password] +* There is no default value for this setting. +* Required when connection trust is configured with <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>> +* {prohibit-ssl-disabled-effective} + +Password for the <<{version}-plugins-{type}s-{plugin}-ssl_truststore_path>>. + +[id="{version}-plugins-{type}s-{plugin}-ssl_verification_mode"] +===== `ssl_verification_mode` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* {prohibit-ssl-disabled-effective} + +Level of verification of the certificate provided by {es}. + +SSL certificates presented by {es} are fully-validated by default. + +* Available modes: +** `none`: performs no validation, implicitly trusting any server that this plugin connects to (insecure) +** `certificate`: validates the server-provided certificate is signed by a trusted certificate authority and that the server can prove possession of its associated private key (less secure) +** `full` (default): performs the same validations as `certificate` and also verifies that the provided certificate has an identity claim matching the server we are attempting to connect to (most secure) + +[id="{version}-plugins-{type}s-{plugin}-username"] +===== `username` + +* Value type is {logstash-ref}/configuration-file-structure.html#string[string] +* There is no default value for this setting. +* When present, <<{version}-plugins-{type}s-{plugin}-password>> is also required. + +A user name when using HTTP Basic Authentication to connect to {es}. + +  + +[id="{version}-plugins-{type}s-{plugin}-common-options"] +include::{include_path}/{type}.asciidoc[]