fix: stability and bug fixes (#138)

* fix: remove erroneous dash in shebang

* fix: missing ''' in Jenkinsfiles

* fix: updated node label

* fix: documentation updates

* test: do change

* fix: update to createpw() function to stop Jenkins hang

* fix: update Jenkinsfiles

* test: add tracing to kube start

* test: more tracing

* test: update the createpw() function

* test: move the createpw logic

* test: move tr to sed

* test: still futzing with password generation

* test: zeroing in on PW fix

* fix: revert tests and apply fix for createpw()

* fix: issues with eks and 1.22, revert to 1.21 for eks

* chore: update nginx ic version and chart version

* fix: typo in configuration

* feat: update start script with new additions

* fix: add pulumi args to calls to pulumi to get stack

* fix: downgrade pulumi-docker to address ecr issue

* chore: remove post-run command block from jenkinsfiles

* chore: remove version info for ic from aws jenkinsfile

* chore: bump nginx ic version for repo deploy

* fix: version 5.2.0 moves to new api ver for auth

* fix: trying to find a working aws/eks pulumi combo

* fix: trying to find a working aws/eks pulumi combo

* fix: specify and lock all python modules

* fix: add stack identifier to call to application

* feat: add in information on versions in use

* fix: cleanup random temp file

* fix: move version info to AWS script for debugging

* test: working on kubectl version issue

* feat: add version information to start

* fix: address issue with curl command for kubectl

* test: trying new K8 version with older kubectl

* fix: cleaning up requirements.txt file

* fix: revert to 1.21

* feat: add call to pull K8 server information

* fix: clean up kubectl update section

* fix: adjust to work around #134

Author: Jason Schmidt

bin/setup_venv.sh

@@ -235,16 +235,34 @@ else
   exit 1
 fi
 
-# Add local kubectl to the virtual environment
+#
+# This section originally pulled the most recent version of Kubectl down; however it turned out that
+# was causing isues with our AWS deploy (see the issues in the repo). Addtionally, this was only
+# downloading the kubectl if it did not exist; this could result in versions not being updated if the
+# MARA project was run in the same environment w/o a refresh.
+#
+# The two fixes here are to hardcode (For now) to a known good version (1.23.6) and force the script to
+# always download this version.
+#
+# TODO: Figure out a way to not hardocde the kubectl version
+# TODO: Should not always download if the versions match; need a version check
+#
+#
 if [ ! -x "${VIRTUAL_ENV}/bin/kubectl" ]; then
   echo "Downloading kubectl into virtual environment"
-  KUBECTL_VERSION="$(${download_cmd} https://dl.k8s.io/release/stable.txt)"
+  KUBECTL_VERSION="v1.23.6"
   ${download_cmd} "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl" >"${VIRTUAL_ENV}/bin/kubectl"
   KUBECTL_CHECKSUM="$(${download_cmd} "https://dl.k8s.io/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl.sha256")"
   echo "${KUBECTL_CHECKSUM}  ${VIRTUAL_ENV}/bin/kubectl" | ${sha256sum_cmd}
   chmod +x "${VIRTUAL_ENV}/bin/kubectl"
 else
-  echo "kubectl is already installed"
+  echo "kubectl is already installed, but will overwrite to ensure correct version"
+  echo "Downloading kubectl into virtual environment"
+  KUBECTL_VERSION="v1.23.6"
+  ${download_cmd} "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl" >"${VIRTUAL_ENV}/bin/kubectl"
+  KUBECTL_CHECKSUM="$(${download_cmd} "https://dl.k8s.io/${KUBECTL_VERSION}/bin/${OS}/${ARCH}/kubectl.sha256")"
+  echo "${KUBECTL_CHECKSUM}  ${VIRTUAL_ENV}/bin/kubectl" | ${sha256sum_cmd}
+  chmod +x "${VIRTUAL_ENV}/bin/kubectl"
 fi
 
 #

bin/start.sh

@@ -17,7 +17,7 @@ script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
 # Check to see if the venv has been installed, since this is only going to be used to start pulumi/python based
 # projects.
 #
-if ! command -v "${script_dir}/../pulumi/python/venv/bin/python" > /dev/null ; then
+if ! command -v "${script_dir}/../pulumi/python/venv/bin/python" >/dev/null; then
   echo "NOTICE! Unable to find the venv directory. This is required for the pulumi/python deployment process."
   echo "Please run ./setup_venv.sh from this directory to install the required virtual environment."
   echo " "
@@ -31,7 +31,6 @@ if ! command -v pulumi >/dev/null; then
   if [ -x "${script_dir}/../pulumi/python/venv/bin/pulumi" ]; then
     echo "Adding to [${script_dir}/venv/bin] to PATH"
     export PATH="${script_dir}/../pulumi/python/venv/bin:$PATH"
-
     if ! command -v pulumi >/dev/null; then
       echo >&2 "Pulumi must be installed to continue"
       exit 1
@@ -60,7 +59,7 @@ fi
 echo " "
 echo "NOTICE! This shell script will call the appropriate helper script depending on your answer to the next question."
 echo " "
-echo "This script currently supports standing up an AWS environment (including ECR, EKS, and VPC resources), provided "
+echo "This script currently supports standing up AWS, Linode, and Digital Ocean kubernetes deployments, provided "
 echo "the correct credentials are supplied. It also supports the user of a kubeconfig file with a defined cluster name"
 echo "and context, which must be provided by the user."
 echo " "
@@ -76,7 +75,7 @@ if [ -s "${script_dir}/../config/pulumi/environment" ] && grep --quiet '^PULUMI_
     read -r -e -p "Environment file exists and is not empty. Answer yes to use, no to delete. " yn
     case $yn in
     [Yy]*) # We have an environment file and they want to keep it....
-      if pulumi config get kubernetes:infra_type -C ${script_dir}/../pulumi/python/config>/dev/null 2>&1; then
+      if pulumi config get kubernetes:infra_type -C ${script_dir}/../pulumi/python/config >/dev/null 2>&1; then
         INFRA="$(pulumi config get kubernetes:infra_type -C ${script_dir}/../pulumi/python/config)"
         if [ $INFRA == 'AWS' ]; then
           exec ${script_dir}/start_aws.sh

bin/start_aws.sh

@@ -152,8 +152,8 @@ else
 fi
 
 function createpw() {
-  base64 /dev/random | tr -dc '[:alnum:]' | head -c${1:-16}
-  return 0
+  PWORD=$(dd if=/dev/urandom count=1 2>/dev/null | base64 | head -c16)
+  echo $PWORD
 }
 
 # The bank of sirius configuration file is stored in the ./sirius/config
@@ -208,13 +208,13 @@ function header() {
 }
 
 function add_kube_config() {
-  pulumi_region="$(pulumi config get aws:region -C ${script_dir}/../pulumi/python/config)"
+  pulumi_region="$(pulumi ${pulumi_args} config get aws:region -C ${script_dir}/../pulumi/python/config)"
   if [ "${pulumi_region}" != "" ]; then
     region_arg="--region ${pulumi_region}"
   else
     region_arg=""
   fi
-  pulumi_aws_profile="$(pulumi config get aws:profile -C ${script_dir}/../pulumi/python/config)"
+  pulumi_aws_profile="$(pulumi ${pulumi_args} config get aws:profile -C ${script_dir}/../pulumi/python/config)"
   if [ "${pulumi_aws_profile}" != "" ]; then
     echo "Using AWS profile [${pulumi_aws_profile}] from Pulumi configuration"
     profile_arg="--profile ${pulumi_aws_profile}"
@@ -225,7 +225,7 @@ function add_kube_config() {
     profile_arg=""
   fi
 
-  cluster_name="$(pulumi stack output cluster_name -C ${script_dir}/../pulumi/python/infrastructure/aws/eks)"
+  cluster_name="$(pulumi ${pulumi_args} stack output cluster_name -C ${script_dir}/../pulumi/python/infrastructure/aws/eks)"
 
   echo "adding ${cluster_name} cluster to local kubeconfig"
   "${script_dir}"/../pulumi/python/venv/bin/aws ${profile_arg} ${region_arg} eks update-kubeconfig --name ${cluster_name}
@@ -284,6 +284,19 @@ pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/config
 # configuration because of the encryption needed for the passwords.
 pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/kubernetes/applications/sirius AWS
 
+header "Version Info"
+echo "Version and Account Information"
+echo "====================================================================="
+echo "Pulumi version is: $(pulumi version)"
+echo "Pulumi user is: $(pulumi whoami)"
+echo "Python version is: $(python --version)"
+echo "Kubectl version information: "
+echo "$(kubectl version -o json)"
+echo "Python module information: "
+echo "$(pip list)"
+echo "====================================================================="
+echo " "
+
 header "AWS VPC"
 cd "${script_dir}/../pulumi/python/infrastructure/aws/vpc"
 pulumi $pulumi_args up
@@ -300,6 +313,11 @@ if command -v kubectl >/dev/null; then
   retry 30 kubectl version >/dev/null
 fi
 
+# Display the server information
+echo "Kubernetes client/server version information:"
+kubectl version -o json
+echo " "
+
 #
 # This is used to streamline the pieces that follow. Moving forward we can add new logic behind this and this
 # should abstract away for us. This way we just call the kubeconfig project to get the needed information and
@@ -357,7 +375,7 @@ header "Bank of Sirius"
 cd "${script_dir}/../pulumi/python/kubernetes/applications/sirius"
 
 pulumi $pulumi_args up
-app_url="$(pulumi stack output --json | python3 "${script_dir}"/../pulumi/python/kubernetes/applications/sirius/verify.py)"
+app_url="$(pulumi ${pulumi_args} stack output --json | python3 "${script_dir}"/../pulumi/python/kubernetes/applications/sirius/verify.py)"
 
 header "Finished!"
 echo "The startup process has finished successfully"
@@ -376,4 +394,3 @@ echo "Bank of Sirius (Example Application) Configuration: pulumi config -C ${scr
 echo "K8 Loadbalancer IP: kubectl get services --namespace nginx-ingress"
 echo " "
 echo "Please see the documentation in the github repository for more information"
-

bin/start_do.sh

@@ -124,8 +124,8 @@ else
 fi
 
 function createpw() {
-  base64 /dev/random | tr -dc '[:alnum:]' | head -c${1:-16}
-  return 0
+  PWORD=$(dd if=/dev/urandom count=1 2>/dev/null | base64 | head -c16)
+  echo $PWORD
 }
 
 # The bank of sirius configuration file is stored in the ./sirius/config
@@ -287,6 +287,19 @@ pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/config
 # configuration because of the encryption needed for the passwords.
 pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/kubernetes/applications/sirius DO
 
+header "Version Info"
+echo "Version and Account Information"
+echo "====================================================================="
+echo "Pulumi version is: $(pulumi version)"
+echo "Pulumi user is: $(pulumi whoami)"
+echo "Python version is: $(python --version)"
+echo "Kubectl version information: "
+echo "$(kubectl version -o json)"
+echo "Python module information: "
+echo "$(pip list)"
+echo "====================================================================="
+echo " "
+
 header "DO Kubernetes"
 cd "${script_dir}/../pulumi/python/infrastructure/digitalocean/domk8s"
 pulumi $pulumi_args up
@@ -300,6 +313,12 @@ if command -v kubectl >/dev/null; then
   retry 30 kubectl version >/dev/null
 fi
 
+# Display the server information
+echo "Kubernetes client/server version information:"
+kubectl version -o json
+echo " "
+
+
 #
 # This is used to streamline the pieces that follow. Moving forward we can add new logic behind this and this
 # should abstract away for us. This way we just call the kubeconfig project to get the needed information and
@@ -339,7 +358,7 @@ pulumi $pulumi_args up
 
 header "Finished!"
 THE_FQDN=$(pulumi config get kic-helm:fqdn -C ${script_dir}/../pulumi/python/config || echo "Cannot Retrieve")
-THE_IP=$(kubectl get service kic-nginx-ingress  --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
+THE_IP=$(kubectl get service kic-nginx-ingress --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
 
 echo " "
 echo "The startup process has finished successfully"
@@ -358,4 +377,4 @@ echo "Main Configuration: pulumi config -C ${script_dir}/../pulumi/python/config
 echo "Bank of Sirius (Example Application) Configuration: pulumi config -C ${script_dir}/../pulumi/python/kubernetes/applications/sirius"
 echo "K8 Loadbalancer IP: kubectl get services --namespace nginx-ingress"
 echo " "
-echo "Please see the documentation in the github repository for more information"
+echo "Please see the documentation in the github repository for more information"

bin/start_kube.sh

@@ -131,8 +131,8 @@ function retry() {
 }
 
 function createpw() {
-  base64 /dev/random | tr -dc '[:alnum:]' | head -c${1:-16}
-  return 0
+  PWORD=$(dd if=/dev/urandom count=1 2>/dev/null | base64 | head -c16)
+  echo $PWORD
 }
 
 #
@@ -288,6 +288,24 @@ else
   pulumi_args="--color never --stack ${PULUMI_STACK}"
 fi
 
+#
+# Note that this is somewhat different than the other startup scripts, because at the point we run this
+# here we know that we have a server so we can get the version. The other builds do not have server info
+# at this point in time.
+#
+header "Version Info"
+echo "Version and Account Information"
+echo "====================================================================="
+echo "Pulumi version is: $(pulumi version)"
+echo "Pulumi user is: $(pulumi whoami)"
+echo "Python version is: $(python --version)"
+echo "Kubectl version information: "
+echo "$(kubectl version -o json)"
+echo "Python module information: "
+echo "$(pip list)"
+echo "====================================================================="
+echo " "
+
 header "Kubeconfig"
 cd "${script_dir}/../pulumi/python/infrastructure/kubeconfig"
 pulumi $pulumi_args up
@@ -323,7 +341,7 @@ pulumi $pulumi_args up
 
 header "Finished!!"
 THE_FQDN=$(pulumi config get kic-helm:fqdn -C ${script_dir}/../pulumi/python/config || echo "Cannot Retrieve")
-THE_IP=$(kubectl get service kic-nginx-ingress  --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
+THE_IP=$(kubectl get service kic-nginx-ingress --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
 
 echo " "
 echo "The startup process has finished successfully"

bin/start_lke.sh

@@ -1,4 +1,4 @@
--#!/usr/bin/env bash
+#!/usr/bin/env bash
 
 set -o errexit  # abort on nonzero exit status
 set -o nounset  # abort on unbound variable
@@ -99,8 +99,8 @@ else
 fi
 
 function createpw() {
-  base64 /dev/random | tr -dc '[:alnum:]' | head -c${1:-16}
-  return 0
+  PWORD=$(dd if=/dev/urandom count=1 2>/dev/null | base64 | head -c16)
+  echo $PWORD
 }
 
 source "${script_dir}/../config/pulumi/environment"
@@ -296,6 +296,19 @@ pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/config
 # configuration because of the encryption needed for the passwords.
 pulumi config set kubernetes:infra_type -C ${script_dir}/../pulumi/python/kubernetes/applications/sirius LKE
 
+header "Version Info"
+echo "Version and Account Information"
+echo "====================================================================="
+echo "Pulumi version is: $(pulumi version)"
+echo "Pulumi user is: $(pulumi whoami)"
+echo "Python version is: $(python --version)"
+echo "Kubectl version information: "
+echo "$(kubectl version -o json)"
+echo "Python module information: "
+echo "$(pip list)"
+echo "====================================================================="
+echo " "
+
 header "Linode LKE"
 cd "${script_dir}/../pulumi/python/infrastructure/linode/lke"
 pulumi $pulumi_args up
@@ -313,6 +326,11 @@ pulumi $pulumi_args up
 cluster_name=$(pulumi stack output cluster_id -s "${PULUMI_STACK}" -C ${script_dir}/../pulumi/python/infrastructure/linode/lke)
 add_kube_config
 
+# Display the server information
+echo "Kubernetes client/server version information:"
+kubectl version -o json
+echo " "
+
 if command -v kubectl >/dev/null; then
   echo "Attempting to connect to newly create kubernetes cluster"
   retry 30 kubectl version >/dev/null
@@ -348,7 +366,7 @@ pulumi $pulumi_args up
 
 header "Finished!"
 THE_FQDN=$(pulumi config get kic-helm:fqdn -C ${script_dir}/../pulumi/python/config || echo "Cannot Retrieve")
-THE_IP=$(kubectl get service kic-nginx-ingress  --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
+THE_IP=$(kubectl get service kic-nginx-ingress --namespace nginx-ingress --output=jsonpath='{.status.loadBalancer.ingress[*].ip}' || echo "Cannot Retrieve")
 
 echo " "
 echo "The startup process has finished successfully"

config/pulumi/Pulumi.stackname.yaml.example

@@ -62,7 +62,7 @@ config:
   ############################################################################
 
   # This is the Kubernetes version to install using EKS.
-  eks:k8s_version: 1.22
+  eks:k8s_version: 1.21
   # This is the default instance type used by EKS.
   eks:instance_type: t2.large
   # The minimum number of compute instances to provision for the EKS cluster.
@@ -94,7 +94,7 @@ config:
   # Chart name for the helm chart for kic
   kic-helm:chart_name: nginx-ingress
   # Chart version for the helm chart for kic
-  kic-helm:chart_version: 0.12.0
+  kic-helm:chart_version: 0.13.1
   # Name of the repo to pull the kic chart from
   kic-helm:helm_repo_name: nginx-stable
   # URL of the chart repo to pull kic from
@@ -118,12 +118,12 @@ config:
   # https://docs.nginx.com/nginx-ingress-controller/installation/pulling-ingress-controller-image/
   #
   # The following are all valid image names:
-  # kic:image_name: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:2.1.0
-  # kic:image_name: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:2.1.0-ot
-  # kic:image_name: docker.io/nginx/nginx-ingress:2.1.0
-  # kic:image_name: nginx/nginx-ingress:2.1.0
-  # kic:image_name: nginx/nginx-ingress:2.1.0-alpine
-  kic:image_name: nginx/nginx-ingress:2.1.0
+  # kic:image_name: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:2.2.0
+  # kic:image_name: private-registry.nginx.com/nginx-ic/nginx-plus-ingress:2.2.0-ot
+  # kic:image_name: docker.io/nginx/nginx-ingress:2.2.0
+  # kic:image_name: nginx/nginx-ingress:2.2.0
+  # kic:image_name: nginx/nginx-ingress:2.2.0-alpine
+  kic:image_name: nginx/nginx-ingress:2.2.0
 
 
   ############################################################################
@@ -174,10 +174,10 @@ config:
   # ssh://[email protected]:nginxinc/kubernetes-ingress.git
   #
   # Github URL with a tag specified:
-  # https://github.com/nginxinc/kubernetes-ingress.git#v2.1.0
-  # [email protected]:nginxinc/kubernetes-ingress.git#v2.1.0
-  # ssh://[email protected]:nginxinc/kubernetes-ingress.git#v2.1.0
-  kic:src_url: https://github.com/nginxinc/kubernetes-ingress.git#v2.1.0
+  # https://github.com/nginxinc/kubernetes-ingress.git#v2.2.0
+  # [email protected]:nginxinc/kubernetes-ingress.git#v2.2.0
+  # ssh://[email protected]:nginxinc/kubernetes-ingress.git#v2.2.0
+  kic:src_url: https://github.com/nginxinc/kubernetes-ingress.git#v2.2.0
 
   # When set to true, Pulumi's diff logic is circumvented and the image will always be
   # rebuilt regardless of the input variables to Pulumi being the same or not.