Merge pull request #556 from nhosoi/file_buffer_1

Merged by openshift-bot

Author: OpenShift Bot

fluentd/configs.d/openshift/es-copy-config.conf

@@ -20,6 +20,8 @@
       flush_interval "#{ENV['ES_FLUSH_INTERVAL'] || '5s'}"
       max_retry_wait "#{ENV['ES_RETRY_WAIT'] || '300'}"
       disable_retry_limit true
+      buffer_type file
+      buffer_path '/var/lib/fluentd/buffer-es-copy-config'
       buffer_queue_limit "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
       buffer_chunk_limit "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
       # the systemd journald 0.0.8 input plugin will just throw away records if the buffer

fluentd/configs.d/openshift/es-ops-copy-config.conf

@@ -20,6 +20,8 @@
       flush_interval "#{ENV['OPS_FLUSH_INTERVAL'] || ENV['ES_FLUSH_INTERVAL'] || '5s'}"
       max_retry_wait "#{ENV['OPS_RETRY_WAIT'] || ENV['ES_RETRY_WAIT'] || '300'}"
       disable_retry_limit true
+      buffer_type file
+      buffer_path '/var/lib/fluentd/buffer-es-ops-copy-config'
       buffer_queue_limit "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
       buffer_chunk_limit "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
       # the systemd journald 0.0.8 input plugin will just throw away records if the buffer

fluentd/configs.d/openshift/output-es-config.conf

@@ -20,6 +20,8 @@
       flush_interval "#{ENV['ES_FLUSH_INTERVAL'] || '5s'}"
       max_retry_wait "#{ENV['ES_RETRY_WAIT'] || '300'}"
       disable_retry_limit true
+      buffer_type file
+      buffer_path '/var/lib/fluentd/buffer-output-es-config'
       buffer_queue_limit "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
       buffer_chunk_limit "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
       # the systemd journald 0.0.8 input plugin will just throw away records if the buffer

fluentd/configs.d/openshift/output-es-ops-config.conf

@@ -20,6 +20,8 @@
       flush_interval "#{ENV['OPS_FLUSH_INTERVAL'] || ENV['ES_FLUSH_INTERVAL'] || '5s'}"
       max_retry_wait "#{ENV['OPS_RETRY_WAIT'] || ENV['ES_RETRY_WAIT'] || '300'}"
       disable_retry_limit true
+      buffer_type file
+      buffer_path '/var/lib/fluentd/buffer-output-es-ops-config'
       buffer_queue_limit "#{ENV['BUFFER_QUEUE_LIMIT'] || '1024' }"
       buffer_chunk_limit "#{ENV['BUFFER_SIZE_LIMIT'] || '1m' }"
       # the systemd journald 0.0.8 input plugin will just throw away records if the buffer

fluentd/run.sh

@@ -61,11 +61,7 @@ IPADDR4=`/usr/sbin/ip -4 addr show dev eth0 | grep inet | sed -e "s/[ \t]*inet \
 IPADDR6=`/usr/sbin/ip -6 addr show dev eth0 | grep inet6 | sed "s/[ \t]*inet6 \([a-f0-9:]*\).*/\1/"`
 export IPADDR4 IPADDR6
 
-BUFFER_SIZE_LIMIT=${BUFFER_SIZE_LIMIT:-1048576}
-MUX_CPU_LIMIT=${MUX_CPU_LIMIT:-500m}
-MUX_MEMORY_LIMIT=${MUX_MEMORY_LIMIT:-2Gi}
-FLUENTD_CPU_LIMIT=${FLUENTD_CPU_LIMIT:-100m}
-FLUENTD_MEMORY_LIMIT=${FLUENTD_MEMORY_LIMIT:-512Mi}
+BUFFER_SIZE_LIMIT=${BUFFER_SIZE_LIMIT:-16777216}
 
 CFG_DIR=/etc/fluent/configs.d
 if [ "${USE_MUX:-}" = "true" ] ; then
@@ -129,30 +125,55 @@ else
 fi
 export K8S_FILTER_REMOVE_KEYS
 
-if [ "${USE_MUX:-}" = "true" ] ; then
-    TOTAL_MEMORY_LIMIT=`echo $MUX_MEMORY_LIMIT |  sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc`
+# How many outputs?
+if [ -n "${MUX_CLIENT_MODE:-}" -o "${USE_MUX_CLIENT:-}" = "true" ] ; then
+    # A fluentd collector configured as a mux client has just one output: sending to a mux.
+    NUM_OUTPUTS=1
 else
-    TOTAL_MEMORY_LIMIT=`echo $FLUENTD_MEMORY_LIMIT |  sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc`
-fi
-BUFFER_SIZE_LIMIT=`echo $BUFFER_SIZE_LIMIT |  sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc`
-if [ $BUFFER_SIZE_LIMIT -eq 0 ]; then
-    BUFFER_SIZE_LIMIT=1048576
+    # fluentd usually has 2 outputs.
+    NUM_OUTPUTS=2
+    if [ "$ES_COPY" = "true" ]; then
+        NUM_OUTPUTS=`expr $NUM_OUTPUTS \* 2`
+    fi
 fi
 
-DIV=1
-if [ "$ES_HOST" != "$OPS_HOST" ] || [ "$ES_PORT" != "$OPS_PORT" ] ; then
-    # using ops cluster
-    DIV=`expr $DIV \* 2`
+# If FILE_BUFFER_PATH exists and it is not a directory, mkdir fails with the error.
+FILE_BUFFER_PATH=/var/lib/fluentd
+mkdir -p $FILE_BUFFER_PATH
+
+# Get the available disk size; use 1/4 of it
+DF_LIMIT=$(df -B1 $FILE_BUFFER_PATH | grep -v Filesystem | awk '{print $2}')
+DF_LIMIT=${DF_LIMIT:-0}
+DF_LIMIT=$(expr $DF_LIMIT / 4) || :
+if [ $DF_LIMIT -eq 0 ]; then
+    echo "ERROR: No disk space is available for file buffer in $FILE_BUFFER_PATH."
+    exit 1
 fi
-if [ -n "${MUX_CLIENT_MODE:-}" -o "${USE_MUX_CLIENT:-}" = "true" ] ; then
-    DIV=`expr $DIV \* 2`
+# Determine final total given the number of outputs we have.
+TOTAL_LIMIT=$(echo ${FILE_BUFFER_LIMIT:-2Gi} | sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc) || :
+if [ $TOTAL_LIMIT -le 0 ]; then
+    echo "ERROR: Invalid file buffer limit ($FILE_BUFFER_LIMIT) is given.  Failed to convert to bytes."
+    exit 1
 fi
+TOTAL_LIMIT=$(expr $TOTAL_LIMIT \* $NUM_OUTPUTS) || :
+if [ $DF_LIMIT -lt $TOTAL_LIMIT ]; then
+    echo "WARNING: Available disk space ($DF_LIMIT bytes) is less than the user specified file buffer limit ($FILE_BUFFER_LIMIT times $NUM_OUTPUTS)."
+    TOTAL_LIMIT=$DF_LIMIT
+fi
+
+BUFFER_SIZE_LIMIT=$(echo $BUFFER_SIZE_LIMIT |  sed -e "s/[Kk]/*1024/g;s/[Mm]/*1024*1024/g;s/[Gg]/*1024*1024*1024/g;s/i//g" | bc)
+BUFFER_SIZE_LIMIT=${BUFFER_SIZE_LIMIT:-16777216}
 
-# MEMORY_LIMIT per buffer
-MEMORY_LIMIT=`expr $TOTAL_MEMORY_LIMIT / $DIV`
-BUFFER_QUEUE_LIMIT=`expr $MEMORY_LIMIT / $BUFFER_SIZE_LIMIT`
-if [ $BUFFER_QUEUE_LIMIT -eq 0 ]; then
-    BUFFER_QUEUE_LIMIT=1024
+# TOTAL_BUFFER_SIZE_LIMIT per buffer
+TOTAL_BUFFER_SIZE_LIMIT=$(expr $TOTAL_LIMIT / $NUM_OUTPUTS) || :
+if [ -z $TOTAL_BUFFER_SIZE_LIMIT -o $TOTAL_BUFFER_SIZE_LIMIT -eq 0 ]; then
+    echo "ERROR: Calculated TOTAL_BUFFER_SIZE_LIMIT is 0. TOTAL_LIMIT $TOTAL_LIMIT is too small compared to NUM_OUTPUTS $NUM_OUTPUTS. Please increase FILE_BUFFER_LIMIT $FILE_BUFFER_LIMIT and/or the volume size of $FILE_BUFFER_PATH."
+    exit 1
+fi
+BUFFER_QUEUE_LIMIT=$(expr $TOTAL_BUFFER_SIZE_LIMIT / $BUFFER_SIZE_LIMIT) || :
+if [ -z $BUFFER_QUEUE_LIMIT -o $BUFFER_QUEUE_LIMIT -eq 0 ]; then
+    echo "ERROR: Calculated BUFFER_QUEUE_LIMIT is 0. TOTAL_BUFFER_SIZE_LIMIT $TOTAL_BUFFER_SIZE_LIMIT is too small compared to BUFFER_SIZE_LIMIT $BUFFER_SIZE_LIMIT. Please increase FILE_BUFFER_LIMIT $FILE_BUFFER_LIMIT and/or the volume size of $FILE_BUFFER_PATH."
+    exit 1
 fi
 export BUFFER_QUEUE_LIMIT BUFFER_SIZE_LIMIT
 
@@ -167,12 +188,15 @@ OPS_COPY_PASSWORD="${OPS_COPY_PASSWORD:-$ES_COPY_PASSWORD}"
 export OPS_COPY_HOST OPS_COPY_PORT OPS_COPY_SCHEME OPS_COPY_CLIENT_CERT \
        OPS_COPY_CLIENT_KEY OPS_COPY_CA OPS_COPY_USERNAME OPS_COPY_PASSWORD
 
-if [ "$ES_COPY" = "true" ] ; then
+if [ "$ES_COPY" = "true" -a "${USE_MUX_CLIENT:-}" != "true" ] ; then
     # user wants to split the output of fluentd into two different elasticsearch
     # user will provide the necessary COPY environment variables as above
     cp $CFG_DIR/{openshift,dynamic}/es-copy-config.conf
     cp $CFG_DIR/{openshift,dynamic}/es-ops-copy-config.conf
 else
+    if [ "$ES_COPY" = "true" ] ; then
+        echo "WARNING: When USE_MUX_CLIENT is true, logs are forwarded to MUX; COPY won't work with it."
+    fi
     # create empty files for the ES copy config
     echo > $CFG_DIR/dynamic/es-copy-config.conf
     echo > $CFG_DIR/dynamic/es-ops-copy-config.conf

hack/testing/init-log-stack

@@ -11,16 +11,38 @@ ES_HOSTNAME=${ES_HOST:+openshift_logging_es_hostname=$ES_HOST}
 ES_OPS_ALLOW_EXTERNAL=${ES_OPS_HOST:+openshift_logging_es_ops_allow_external=True}
 ES_OPS_HOSTNAME=${ES_OPS_HOST:+openshift_logging_es_ops_hostname=$ES_OPS_HOST}
 
+SET_MUX_CPU_LIMIT=""
+SET_MUX_MEMORY_LIMIT=""
+SET_MUX_BUFFER_QUEUE_LIMIT=""
+SET_MUX_BUFFER_SIZE_LIMIT=""
+SET_MUX_FILE_BUFFER_STORAGE_TYPE=""
+SET_MUX_FILE_BUFFER_PVC_NAME=""
+SET_MUX_FILE_BUFFER_PVC_SIZE=""
 if [ "$MUX_ALLOW_EXTERNAL" = true ] ; then
     SET_MUX_CPU_LIMIT="openshift_logging_mux_cpu_limit=${MUX_CPU_LIMIT:-500m}"
     SET_MUX_MEMORY_LIMIT="openshift_logging_mux_memory_limit=${MUX_MEMORY_LIMIT:-2Gi}"
     SET_MUX_BUFFER_QUEUE_LIMIT="openshift_logging_mux_buffer_queue_limit=${MUX_BUFFER_QUEUE_LIMIT:-1024}"
     SET_MUX_BUFFER_SIZE_LIMIT="openshift_logging_mux_buffer_size_limit=${MUX_BUFFER_SIZE_LIMIT:-1048576}"
-else
-    SET_MUX_CPU_LIMIT=""
-    SET_MUX_MEMORY_LIMIT=""
-    SET_MUX_BUFFER_QUEUE_LIMIT=""
-    SET_MUX_BUFFER_SIZE_LIMIT=""
+    SET_MUX_FILE_BUFFER_STORAGE_TYPE="openshift_logging_mux_file_buffer_storage_type=${MUX_FILE_BUFFER_STORAGE_TYPE:-emptydir}"
+    if [ "${MUX_FILE_BUFFER_STORAGE_TYPE:-}" = "pvc" ]; then
+      SET_MUX_FILE_BUFFER_PVC_NAME="openshift_logging_mux_file_buffer_pvc_name=logging-mux-pvc"
+      SET_MUX_FILE_BUFFER_PVC_SIZE="openshift_logging_mux_file_buffer_pvc_size=6Gi"
+      echo "### Prepare pvc storage for testing"
+      echo "apiVersion: \"v1\"
+kind: \"PersistentVolume\"
+metadata:
+  name: logging-muxpv-1
+spec:
+  capacity:
+    storage: \"6Gi\"
+  accessModes:
+    - \"ReadWriteOnce\"
+  hostPath:
+    path: ${FILE_BUFFER_PATH:-/var/lib/fluentd}" > /tmp/host-pv.yaml
+          cat /tmp/host-pv.yaml
+          cat /tmp/host-pv.yaml | oc create -f -
+      echo "###################################"
+    fi
 fi
 
 source $OS_O_A_L_DIR/hack/testing/build-images
@@ -64,7 +86,9 @@ $SET_MUX_CPU_LIMIT
 $SET_MUX_MEMORY_LIMIT
 $SET_MUX_BUFFER_QUEUE_LIMIT
 $SET_MUX_BUFFER_SIZE_LIMIT
-
+$SET_MUX_FILE_BUFFER_STORAGE_TYPE
+$SET_MUX_FILE_BUFFER_PVC_NAME
+$SET_MUX_FILE_BUFFER_PVC_SIZE
 EOL
 
 echo "### Created host inventory file ###"

hack/testing/logging.sh

@@ -213,7 +213,6 @@ rm -f $lfds
 # when fluentd starts up it may take a while before it catches up with all of the logs
 # let's wait until that happens
 wait_for_fluentd_ready
-wait_for_fluentd_to_catch_up
 
 # add admin user and normal user for kibana and token auth testing
 export LOG_ADMIN_USER=admin
@@ -244,6 +243,7 @@ if [ "$ENABLE_OPS_CLUSTER" = "true" ] ; then
         curl_es_input $esopspod /$sg_opsindex/rolesmapping/0 -XPUT -d@- | \
         python -mjson.tool
 fi
+wait_for_fluentd_to_catch_up
 
 # verify that $LOG_ADMIN_USER user has access to cluster stats
 sleep 5

hack/testing/prep-host

@@ -11,7 +11,7 @@ OS_ANSIBLE_BRANCH=${OS_ANSIBLE_BRANCH:-master}
 OS_ANSIBLE_DIR=$WORKDIR/openhift-ansible
 
 sudo yum makecache fast
-sudo yum install python2-pip $RUAMEL_YAML_RPM $ANSIBLE_RPM -y
+sudo yum install python2-pip $RUAMEL_YAML_RPM $ANSIBLE_RPM bc -y
 
 git clone $OS_ANSIBLE_REPO $OS_ANSIBLE_DIR -b $OS_ANSIBLE_BRANCH --depth=1
 

hack/testing/test-es-copy.sh

@@ -13,6 +13,11 @@ if ! type get_running_pod > /dev/null 2>&1 ; then
     . ${OS_O_A_L_DIR:-../..}/deployer/scripts/util.sh
 fi
 
+if [ "$USE_MUX_CLIENT" = "true" ] ; then
+    echo "Skipping -- This test is exclusive to USE_MUX_CLIENT != true."
+    exit 0
+fi
+
 if [[ $# -ne 1 || "$1" = "false" ]]; then
   # assuming not using OPS cluster
   CLUSTER="false"
@@ -45,15 +50,21 @@ write_and_verify_logs() {
     return $rc
 }
 
-restart_fluentd() {
-    # delete daemonset which also stops fluentd
-    oc delete daemonset logging-fluentd
-    # wait for fluentd to stop
+undeploy_fluentd() {
+    fpod=`get_running_pod fluentd`
+
+    # undeploy fluentd
+    oc label node --all logging-infra-fluentd-
+
     wait_for_pod_ACTION stop $fpod
-    # create the daemonset which will also start fluentd
-    oc process logging-fluentd-template | oc create -f -
-    # wait for fluentd to start
-    wait_for_pod_ACTION start fluentd
+}
+
+redeploy_fluentd() {
+  # redeploy fluentd
+  oc label node --all logging-infra-fluentd=true
+
+  # wait for fluentd to start
+  wait_for_pod_ACTION start fluentd
 }
 
 TEST_DIVIDER="------------------------------------------"
@@ -64,33 +75,36 @@ TEST_DIVIDER="------------------------------------------"
 # cause messages to be written to the system log - verify that OPS contains
 # two copies
 
-fpod=`get_running_pod fluentd`
+undeploy_fluentd
 
-# first, make sure copy is off
 cfg=`mktemp`
-oc get template logging-fluentd-template -o yaml | \
+# first, make sure copy is off
+oc get daemonset logging-fluentd -o yaml | \
     sed '/- name: ES_COPY/,/value:/ s/value: .*$/value: "false"/' | \
     oc replace -f -
-restart_fluentd
-fpod=`get_running_pod fluentd`
 
-# save original template config
-origconfig=`mktemp`
-oc get template logging-fluentd-template -o yaml > $origconfig
+redeploy_fluentd
 
 # run test to make sure fluentd is working normally - no copy
 write_and_verify_logs 1 || {
     oc get events -o yaml > $ARTIFACT_DIR/all-events.yaml 2>&1
     exit 1
 }
 
+undeploy_fluentd
+
+# save original daemonset config
+origconfig=`mktemp`
+oc get daemonset logging-fluentd -o yaml > $origconfig
+
 cleanup() {
     # may have already been cleaned up
     if [ ! -f $origconfig ] ; then return 0 ; fi
+    undeploy_fluentd
     # put back original configuration
     oc replace --force -f $origconfig
     rm -f $origconfig
-    restart_fluentd
+    redeploy_fluentd
 }
 trap "cleanup" INT TERM EXIT
 
@@ -99,36 +113,43 @@ nocopy=`mktemp`
 sed '/_COPY/,/value/d' $origconfig > $nocopy
 # for every ES_ or OPS_ setting, create a copy called ES_COPY_ or OPS_COPY_
 envpatch=`mktemp`
-sed -n '/^        - env:/,/^          image:/ {
-/^          image:/d
-/^        - env:/d
+sed -n '/^ *- env:/,/^ *image:/ {
+/^ *image:/d
+/^ *- env:/d
 /name: K8S_HOST_URL/,/value/d
+/name: .*JOURNAL.*/,/value/d
+/name: .*BUFFER.*/,/value/d
+/name: .*MUX.*/,/value/d
+/name: FLUENTD_.*_LIMIT/,/valueFrom:/d
+/resourceFieldRef:/,/containerName: fluentd-elasticsearch/d
+/divisor:/,/resource: limits./d
 s/ES_/ES_COPY_/
 s/OPS_/OPS_COPY_/
 p
 }' $nocopy > $envpatch
 
 # add the scheme, and turn on verbose
 cat >> $envpatch <<EOF
-          - name: ES_COPY
-            value: "true"
-          - name: ES_COPY_SCHEME
-            value: https
-          - name: OPS_COPY_SCHEME
-            value: https
-          - name: VERBOSE
-            value: "true"
+        - name: ES_COPY
+          value: "true"
+        - name: ES_COPY_SCHEME
+          value: https
+        - name: OPS_COPY_SCHEME
+          value: https
+        - name: VERBOSE
+          value: "true"
 EOF
 
 # add this back to the dc config
+docopy=`mktemp`
 cat $nocopy | \
-    sed '/^        - env:/r '$envpatch | \
-    oc replace -f -
+    sed '/^ *- env:/r '$envpatch > $docopy
 
-rm -f $envpatch $nocopy
+cat $docopy | \
+    oc replace -f -
 
-restart_fluentd
-fpod=`get_running_pod fluentd`
+redeploy_fluentd
+rm -f $envpatch $nocopy $docopy
 
 write_and_verify_logs 2 || {
     oc get events -o yaml > $ARTIFACT_DIR/all-events.yaml 2>&1
@@ -139,9 +160,6 @@ write_and_verify_logs 2 || {
 oc replace --force -f $origconfig
 rm -f $origconfig
 
-restart_fluentd
-fpod=`get_running_pod fluentd`
-
 write_and_verify_logs 1 || {
     oc get events -o yaml > $ARTIFACT_DIR/all-events.yaml 2>&1
     exit 1