LOG-1195: Add Openshift Ingest Plugin for ES

Add default openshift_schema pipeline definiton for all [app-, infra-, audit-] indices.

Author: lukas-vlcek

elasticsearch/Dockerfile

@@ -31,6 +31,7 @@ USER 0
 
 ARG ES_ARCHIVE_URL
 ARG PROMETHEUS_EXPORTER_URL
+ARG INGEST_PLUGIN_URL
 ARG OPENDISTRO_URL
 ARG OPENSHIFT_CI
 
@@ -44,6 +45,7 @@ ENV ES_PATH_CONF=/etc/elasticsearch/ \
     JAVA_HOME=/usr/lib/jvm/jre \
     NODE_QUORUM=1 \
     PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \
+    INGEST_PLUGIN_VER=6.8.1.0-redhat-00002 \
     OPENDISTRO_VER=0.10.0.4-redhat-00001 \
     PLUGIN_LOGLEVEL=INFO \
     RECOVER_AFTER_NODES=1 \
@@ -80,6 +82,7 @@ RUN /var/tmp/install-es.sh
 
 COPY  ${upstream_code}/sgconfig/ ${HOME}/sgconfig/
 COPY  ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/
+COPY  ${upstream_code}/ingest_pipelines/ ${ES_HOME}/ingest_pipelines/
 COPY  ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/
 COPY  ${upstream_code}/init/ ${ES_HOME}/init/
 COPY  ${upstream_code}/probe/ ${ES_HOME}/probe/

elasticsearch/Dockerfile.in

@@ -44,6 +44,7 @@ USER 0
 
 ARG ES_ARCHIVE_URL
 ARG PROMETHEUS_EXPORTER_URL
+ARG INGEST_PLUGIN_URL
 ARG OPENDISTRO_URL
 ARG OPENSHIFT_CI
 
@@ -57,6 +58,7 @@ ENV ES_PATH_CONF=/etc/elasticsearch/ \
     JAVA_HOME=/usr/lib/jvm/jre \
     NODE_QUORUM=1 \
     PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-00001 \
+    INGEST_PLUGIN_VER=6.8.1.0-redhat-00002 \
     OPENDISTRO_VER=0.10.0.4-redhat-00001 \
     PLUGIN_LOGLEVEL=INFO \
     RECOVER_AFTER_NODES=1 \
@@ -96,6 +98,7 @@ RUN /var/tmp/install-es.sh
 
 COPY --from=builder ${upstream_code}/sgconfig/ ${HOME}/sgconfig/
 COPY --from=builder ${upstream_code}/index_templates/ ${ES_HOME}/index_templates/
+COPY --from=builder ${upstream_code}/ingest_pipelines/ ${ES_HOME}/ingest_pipelines/
 COPY --from=builder ${upstream_code}/index_patterns/ ${ES_HOME}/index_patterns/
 COPY --from=builder ${upstream_code}/init/ ${ES_HOME}/init/
 COPY --from=builder ${upstream_code}/probe/ ${ES_HOME}/probe/

elasticsearch/ci-env.sh

@@ -4,14 +4,20 @@ set -o xtrace
 
 PROMETHEUS_EXPORTER_URL=${PROMETHEUS_EXPORTER_URL:-$MAVEN_REPO_URL/org/elasticsearch/plugin/prometheus/prometheus-exporter/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip}
 OPENDISTRO_URL=${OPENDISTRO_URL:-$MAVEN_REPO_URL/com/amazon/opendistroforelasticsearch/opendistro_security/$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip}
+INGEST_PLUGIN_URL=${INGEST_PLUGIN_URL:-https://github.com/ViaQ/elasticsearch-openshift-ingest-plugin/releases/download/$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip}
 
 if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
     # This flag is set during CI runs. If no ARG was passed in,
     # default to maven.org.
     export ES_ARCHIVE_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/elasticsearch-oss-$ES_VER/elasticsearch-oss-$ES_VER.zip
     export OPENDISTRO_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/opendistro_security-$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip
 
+    INGEST_PLUGIN_VER=$(echo $INGEST_PLUGIN_VER | cut -d'-' -f1)
+    export INGEST_PLUGIN_URL=https://github.com/ViaQ/elasticsearch-openshift-ingest-plugin/releases/download/$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip
+    # Modify this if you need to CI-test the plugin sourcing from different repo
+    # export INGEST_PLUGIN_URL=https://github.com/lukas-vlcek/elasticsearch-openshift-ingest-plugin/releases/download/v$INGEST_PLUGIN_VER/openshift-ingest-plugin-$INGEST_PLUGIN_VER.zip
+
     PROMETHEUS_EXPORTER_VER=$(echo $PROMETHEUS_EXPORTER_VER | cut -d'-' -f1)
     export PROMETHEUS_EXPORTER_URL=https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip
 fi
-es_plugins=($OPENDISTRO_URL $PROMETHEUS_EXPORTER_URL)
+es_plugins=($OPENDISTRO_URL $PROMETHEUS_EXPORTER_URL $INGEST_PLUGIN_URL)

elasticsearch/fetch-artifacts-koji.yaml

@@ -1,3 +1,4 @@
 - nvr: org.elasticsearch-elasticsearch-6.8.1.redhat_00006-1
 - nvr: com.amazon.opendistroforelasticsearch-opendistro_security-0.10.0.4_redhat_00001-1
 - nvr: org.elasticsearch.plugin.prometheus-prometheus-exporter-6.8.1.0_redhat_00001-1
+- nvr: org.elasticsearch.plugin.ingest-openshift-ingest-plugin-6.8.1.0_redhat_00002-1

elasticsearch/index_templates/com.redhat.viaq-openshift-operations.template.json

@@ -973,6 +973,7 @@
   },
   "order": 10,
   "settings": {
-    "index.refresh_interval": "5s"
+    "index.refresh_interval": "5s",
+    "index.default_pipeline": "openshift_schema.json"
   }
 }

elasticsearch/index_templates/com.redhat.viaq-openshift-project.template.json

@@ -972,6 +972,7 @@
   },
   "order": 10,
   "settings": {
-    "index.refresh_interval": "5s"
+    "index.refresh_interval": "5s",
+    "index.default_pipeline": "openshift_schema.json"
   }
 }

elasticsearch/ingest_pipelines/openshift_schema.json

@@ -0,0 +1,6 @@
+{
+  "description": "Openshift ingest pipeline for custom schema",
+  "processors": [
+    { "openshift-ingestion-processor": {} }
+  ]
+}

elasticsearch/init/0100-seed-pipelines

@@ -0,0 +1,42 @@
+#!/bin/bash -e
+#
+# Copyright 2018 Red Hat, Inc. and/or its affiliates
+# and other contributors as indicated by the @author tags.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+source "logging"
+
+info Adding ingest pipelines
+shopt -s failglob
+for ingest_file in ${ES_HOME}/ingest_pipelines/*.json
+do
+
+    template=`basename $ingest_file`
+    # Check if ingest template already exists
+    response_code=$(es_util --query=_ingest/pipeline/$template \
+        ${DEBUG:+-v} -s \
+        --request HEAD --head --output /dev/null \
+        -w '%{response_code}')
+    if [ "${response_code}" == "200" ]; then
+        info "Ingest pipeline '$template' found in the cluster, overriding it"
+    else
+        info "Create ingest pipeline '$template'"
+    fi
+    es_util --query=_ingest/pipeline/$template \
+        ${DEBUG:+-v} -s -X PUT \
+        -d@${ingest_file}
+
+done
+shopt -u failglob
+info Finished adding ingest pipelines

elasticsearch/init/0100-seed-index-templates renamed to elasticsearch/init/0200-seed-index-templates

@@ -23,6 +23,7 @@ done
 #fix location from config
 if [[ "${ES_HOME}" != "/usr/share/elasticsearch" ]]; then
   ln -s ${ES_HOME}/index_templates /usr/share/elasticsearch/index_templates
+  ln -s ${ES_HOME}/ingest_pipelines /usr/share/elasticsearch/ingest_pipelines
   ln -s ${ES_HOME}/index_patterns /usr/share/elasticsearch/index_patterns
 fi