LOG-1272: Fix various flaws in elasticsearch5 image related to jaskson library

          Converge on single dockerfile that can consume content built by PNC
          for CI and also is buildable downstream.

(cherry picked from commit 3d7d8c6)

Author: jcantrill

.gitignore

@@ -1,3 +1,4 @@
+/prometheus-exporter-*.zip
 *~
 _output/
 

elasticsearch/Dockerfile

@@ -0,0 +1 @@
+Dockerfile.rhel8

elasticsearch/Dockerfile.origin

@@ -0,0 +1 @@
+Dockerfile.rhel8

elasticsearch/Dockerfile.origin

@@ -2,10 +2,26 @@ FROM registry.ci.openshift.org/ocp/builder:rhel-8-base-openshift-4.7
 
 MAINTAINER OpenShift Development <[email protected]>
 
+LABEL \
+        License="GPLv2+" \
+        io.k8s.description="Elasticsearch container for EFK aggregated logging storage" \
+        io.k8s.display-name="Elasticsearch 6" \
+        io.openshift.tags="logging,elk,elasticsearch" \
+        vendor="Red Hat" \
+        name="openshift/ose-logging-elasticsearch6" \
+        com.redhat.component="logging-elasticsearch6-container" \
+        io.openshift.maintainer.product="OpenShift Container Platform" \
+        io.openshift.maintainer.component="Logging" 
+
 EXPOSE 9200
 EXPOSE 9300
 USER 0
 
+ARG ES_ARCHIVE_URL
+ARG PROMETHEUS_EXPORTER_URL
+ARG OPENDISTRO_URL
+ARG OPENSHIFT_CI
+
 ENV ES_PATH_CONF=/etc/elasticsearch/ \
     ES_HOME=/usr/share/elasticsearch \
     ES_VER=6.8.1.redhat-00006 \
@@ -14,16 +30,17 @@ ENV ES_PATH_CONF=/etc/elasticsearch/ \
     JAVA_VER=11 \
     JAVA_HOME=/usr/lib/jvm/jre \
     NODE_QUORUM=1 \
-    PROMETHEUS_EXPORTER_VER=6.8.1.0-redhat-1 \
+    PROMETHEUS_EXPORTER_VER=6.8.1.0 \
     OPENDISTRO_VER=0.10.0.4-redhat-00001 \
     PLUGIN_LOGLEVEL=INFO \
     RECOVER_AFTER_NODES=1 \
     RECOVER_EXPECTED_NODES=1 \
     RECOVER_AFTER_TIME=5m \
     DHE_TMP_KEY_SIZE=2048 \
-    RELEASE_STREAM=prod
+    RELEASE_STREAM=prod \
+    OPENSHIFT_CI=${OPENSHIFT_CI:-false}
 
-ARG MAVEN_REPO_URL=file:///artifacts/
+ARG MAVEN_REPO_URL=${MAVEN_REPO_URL:-file:///artifacts/}
 
 RUN packages="java-${JAVA_VER}-openjdk-headless \
               python36 python3-pyyaml  \
@@ -44,19 +61,17 @@ RUN mkdir /artifacts
 # In an OSBS build, this will COPY artifacts from fetch-artifacts-koji.yaml. In a CI build, it will just
 # copy the README.MD.
 COPY artifacts/* /artifacts
+COPY *.zip /
 RUN /var/tmp/install-es.sh
 
 ADD sgconfig/ ${HOME}/sgconfig/
 ADD index_templates/ ${ES_HOME}/index_templates/
 ADD index_patterns/ ${ES_HOME}/index_patterns/
 ADD init/ ${ES_HOME}/init/
 ADD probe/ ${ES_HOME}/probe/
-ADD init.sh run.sh prep-install* install.sh ci-env.sh ${HOME}/
+ADD init.sh run.sh ci-env.sh install.sh ${HOME}/
 COPY utils/** /usr/local/bin/
 
-ARG PROMETHEUS_EXPORTER_URL=${MAVEN_REPO_URL}org/elasticsearch/plugin/prometheus/prometheus-exporter/${PROMETHEUS_EXPORTER_VER}/prometheus-exporter-${PROMETHEUS_EXPORTER_VER}.zip
-ARG OPENDISTRO_URL=${MAVEN_REPO_URL}com/amazon/opendistroforelasticsearch/opendistro_security/${OPENDISTRO_VER}/opendistro_security-${OPENDISTRO_VER}.zip
-
 RUN ${HOME}/install.sh && rm -rf /artifacts
 
 WORKDIR ${HOME}

elasticsearch/Dockerfile.rhel8

@@ -2,14 +2,15 @@
 
 set -o xtrace
 
+#PROMETHEUS_EXPORTER_URL=${PROMETHEUS_EXPORTER_URL:-$MAVEN_REPO_URL/org/elasticsearch/plugin/prometheus/prometheus-exporter/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip}
+PROMETHEUS_EXPORTER_URL=file:///prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip
+OPENDISTRO_URL=${OPENDISTRO_URL:-$MAVEN_REPO_URL/com/amazon/opendistroforelasticsearch/opendistro_security/$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip}
+
 if [[ "${OPENSHIFT_CI:-}" == "true" ]]; then
     # This flag is set during CI runs. If no ARG was passed in,
     # default to maven.org.
-	export RELEASE_STREAM=origin
-    export ES_VER=6.8.1
-    export PROMETHEUS_EXPORTER_VER=6.8.1.0
-    export OPENDISTRO_VER=0.10.1.0
-	export MAVEN_REPO_URL="https://repo1.maven.org/maven2/"
-	export PROMETHEUS_EXPORTER_URL=https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/6.8.1.0/prometheus-exporter-6.8.1.0.zip
-    export OPENDISTRO_URL=https://github.com/jcantrill/security/releases/download/v0.10.1.0/opendistro_security-0.10.1.0.zip
+    export ES_ARCHIVE_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/elasticsearch-oss-$ES_VER/elasticsearch-oss-$ES_VER.zip
+    export OPENDISTRO_URL=https://github.com/openshift/origin-aggregated-logging/releases/download/opendistro_security-$OPENDISTRO_VER/opendistro_security-$OPENDISTRO_VER.zip
+    export PROMETHEUS_EXPORTER_URL=https://github.com/vvanholl/elasticsearch-prometheus-exporter/releases/download/$PROMETHEUS_EXPORTER_VER/prometheus-exporter-$PROMETHEUS_EXPORTER_VER.zip
 fi
+es_plugins=($OPENDISTRO_URL $PROMETHEUS_EXPORTER_URL)

elasticsearch/ci-env.sh

@@ -1,4 +1,2 @@
 - nvr: org.elasticsearch-elasticsearch-6.8.1.redhat_00006-1
-- nvr: org.elasticsearch.plugin.prometheus-elasticsearch-prometheus-exporter-6.8.1.0_redhat_1-2
 - nvr: com.amazon.opendistroforelasticsearch-opendistro_security-0.10.0.4_redhat_00001-1
-

elasticsearch/dummy.zip

@@ -12,6 +12,7 @@ cd /var/tmp
 ES_ARCHIVE_URL=${ES_ARCHIVE_URL:-${MAVEN_REPO_URL}org/elasticsearch/distribution/zip/elasticsearch-oss/${ES_VER}/elasticsearch-oss-${ES_VER}.zip}
 curl -L -v -s -o es.zip ${ES_ARCHIVE_URL}
 unzip es.zip
+mv elasticsearch-$(echo $ES_VER | cut -d'.' -f1-3) elasticsearch-${ES_VER}
 pushd elasticsearch-${ES_VER}
 mkdir -p ${ES_HOME}/bin
 install -p -m 755 bin/elasticsearch bin/elasticsearch-cli bin/elasticsearch-keystore \
@@ -44,4 +45,4 @@ chmod u+rwx,g+rwx ${ES_HOME}/logs
 mkdir -p /var/run/elasticsearch
 chmod u+rwx,g+rwx /var/run/elasticsearch
 mkdir /elasticsearch && chmod og+w /elasticsearch
-rm -rf elasticsearch-${ES_VER} es.zip extra-jvm.options
+rm -rf elasticsearch-$(echo $ES_VER | cut -d'.' -f1-3) es.zip extra-jvm.options

elasticsearch/fetch-artifacts-koji.yaml

@@ -6,8 +6,6 @@ source $(dirname "$0")/ci-env.sh
 
 ln -s /usr/local/bin/logging ${HOME}/logging
 
-source ${HOME}/prep-install.${RELEASE_STREAM}
-
 echo "removing module: ingest-geoip"
 rm -rf ${ES_HOME}/modules/ingest-geoip
 

elasticsearch/install-es.sh

@@ -0,0 +1 @@
+SHA512 (prometheus-exporter-6.8.1.0.zip) = 46ebce3ae296836917fcfafbfd5142e1bdd2747847a1d9f62129c37ed72440a8a084f56001a7ad269c84c6ff36f3e4da971caa1a5bf548c2f3d7a180bc45a0f6

elasticsearch/install.sh

@@ -28,4 +28,9 @@ echo "-
 echo "Building image $dir - this may take a few minutes until you see any output..."
 echo "-                                                                                                              -"
 echo "----------------------------------------------------------------------------------------------------------------"
-podman build -f $dfpath -t "$fullimagename" $dir
+buildargs=""
+if [ "$dir" = "elasticsearch" ] ; then
+  buildargs="--build-arg OPENSHIFT_CI=true"
+fi
+
+podman build $buildargs -f $dfpath -t "$fullimagename" $dir

elasticsearch/prep-install.origin

@@ -304,17 +304,12 @@ if [ "${PUSH_ONLY:-false}" = false ] ; then
       mountarg=""
     fi
 
-    suffix=""
-    if [ "$img" = "logging-elasticsearch6" ] ; then
-        suffix=".origin"
-    fi
-
     echo "----------------------------------------------------------------------------------------------------------------"
     echo "-                                                                                                              -"
     echo "Building image $img - this may take a few minutes until you see any output..."
     echo "-                                                                                                              -"
     echo "----------------------------------------------------------------------------------------------------------------"
-    $IMAGE_BUILDER build $IMAGE_BUILDER_OPTS $mountarg -f $dfpath$suffix -t "$fullimagename" $dir
+    $(pwd)/hack/build-component-image.sh $dir $fullimagename
     dir=""
     img=""
   done

elasticsearch/prep-install.prod

@@ -0,0 +1 @@
+Dockerfile.rhel8