Development tooling improvements (#3115)

This commit  changes the way we run tests in evergreen and locally via context switches. More can be read here:  REDACTED and here: REDACTED

Author: nammn

.evergreen-periodic-builds.yaml

@@ -3,6 +3,8 @@
 set -Eeou pipefail
 set -x
 
+source scripts/dev/set_env_context.sh
+
 if [[ -z "${EVERGREEN_MODE:-}" ]]; then
   git_last_changed=$(git diff --cached --name-only --diff-filter=ACM)
 else
@@ -34,13 +36,13 @@ function generate_standalone_yaml() {
 
 function black_formatting() {
   # installing Black
-  if ! command -v "black" >/dev/null; then
-    pip3 install -r docker/mongodb-enterprise-tests/requirements-dev.txt
+  if [ -f "${workdir}"/venv/bin/black ]; then
+    "${workdir}"/venv/bin/pip install -r docker/mongodb-enterprise-tests/requirements-dev.txt
   fi
 
   # Black formatting of every python file that was changed
   for file in $(echo "$git_last_changed" | grep '\.py$'); do
-    black -q "$file"
+    "${workdir}"/venv/bin/black -q "$file"
     git add "$file"
   done
 }

.evergreen.yml

@@ -25,6 +25,8 @@ helm_out
 .redo-last-namespace
 exports.do
 __debug_bin
+.generated/
+scripts/dev/contexts/private-context
 
 # These files get generated by emacs and sometimes they are still present when committing
 **/flycheck_*

.githooks/pre-commit

@@ -0,0 +1,2 @@
+public/
+.git

.gitignore

@@ -0,0 +1 @@
+/scripts/dev/contexts/private-context

.rsyncignore

@@ -13,11 +13,9 @@ usage:
 	@ echo
 	@ echo "Usage:"
 	@ echo "  prerequisites:                  installs the command line applications necessary for working with this tool and adds git pre-commit hook."
-	@ echo "  init:                           prepares operator environment."
 	@ echo "  switch:                         switch current dev context, e.g 'make switch context=kops'. Note, that it switches"
 	@ echo "                                  kubectl context as well and sets the current namespace to the one configured as the default"
 	@ echo "                                  one"
-	@ echo "  contexts:                       list all available contexts"
 	@ echo "  operator:                       build and push Operator image, deploy it to the Kubernetes cluster"
 	@ echo "                                  Use the 'debug' flag to build and deploy the Operator in debug mode - you need"
 	@ echo "                                  to ensure the 30042 port on the K8s node is open"
@@ -39,7 +37,7 @@ usage:
 	@ echo "                                  Use a 'local=true' to run the test locally using 'pytest'."
 	@ echo "                                  Use a 'skip=true' to skip cleaning resources (this may help developing long-running tests like for Ops Manager)"
 	@ echo "                                  Sometimes you may need to pass some custom configuration, this can be done this way:"
-	@ echo "                                  make e2e test=e2e_om_ops_manager_upgrade custom_om_version=4.2.8"
+	@ echo "                                  make e2e test=e2e_om_ops_manager_upgrade CUSTOM_OM_VERSION=4.2.8"
 	@ echo "  recreate-e2e-kops:              deletes and creates a specified e2e cluster 'cluster' using kops (note, that you don't need to switch to the correct"
 	@ echo "                                  kubectl context - the script will handle everything). Pass the flag 'imsure=yes' to make it work."
 	@ echo "                                  Pass 'cluster' parameter for a cluster name if it's different from default ('e2e.mongokubernetes.com')"
@@ -58,42 +56,26 @@ usage:
 prerequisites:
 	@ scripts/dev/install.sh
 
-# prepare default configuration context files
-init:
-	@ mkdir -p ~/.operator-dev/contexts
-	@ cp -n scripts/dev/samples/* ~/.operator-dev/contexts || true
-	@ echo "Initialized dev environment (~/.operator-dev)"
-	@ make switch context=dev
-
 switch:
 	@ scripts/dev/switch_context.sh $(context)
 
-# prints all current contexts
-contexts:
-	@ scripts/dev/print_contexts
-
 # builds the Operator binary file and docker image and pushes it to the remote registry if using a remote registry. Deploys it to
 # k8s cluster
 operator: configure-operator build-and-push-operator-image
 	@ $(MAKE) deploy-operator
 
 # build-push, (todo) restart database
 database: aws_login
-	@ ./pipeline.py --include database
+	@ scripts/evergreen/run_pipelinepy.sh --include database
 
 # ensures cluster is up, cleans Kubernetes + OM, build-push-deploy operator,
 # push-deploy database, create secrets, config map, resources etc
-full: ensure-k8s-and-reset build-and-push-images
+full: build-and-push-images
 	@ $(MAKE) deploy-and-configure-operator
-	@ scripts/dev/apply_resources
 
 # build-push appdb image
 appdb: aws_login
-	@ ./pipeline.py --include appdb
-
-log:
-	@ . scripts/dev/read_context.sh
-	@ kubectl logs -f deployment/mongodb-enterprise-operator --tail=1000
+	@ scripts/evergreen/run_pipelinepy.sh --include appdb
 
 # runs the e2e test: make e2e test=e2e_sharded_cluster_pv. The Operator is redeployed before the test, the namespace is cleaned.
 # The e2e test image is built and pushed together with all main ones (operator, database, init containers)
@@ -114,11 +96,6 @@ e2e-telepresence: build-and-push-test-image
 recreate-e2e-kops:
 	@ scripts/dev/recreate_e2e_kops.sh $(imsure) $(cluster)
 
-# TODO: Automate this process
-# deletes and creates a openshift e2e cluster
-recreate-e2e-openshift:
-	@ echo "Please follow instructions in docs/openshift4.md to install the Openshift4 cluster."
-
 # clean all kubernetes cluster resources and OM state. "light=true" to clean only Mongodb resources
 reset:
 	@ scripts/dev/reset.sh $(light)
@@ -142,21 +119,21 @@ ac:
 # in parallel and both call 'aws_login') then Docker login may return an error "Error saving credentials:..The
 # specified item already exists in the keychain". Seems this allows to ignore the error
 aws_login:
-	@ . scripts/dev/set_env_context.sh; scripts/dev/configure_docker_auth.sh
+	@ scripts/dev/configure_docker_auth.sh
 
 # cleans up aws resources, including s3 buckets which are older than 5 hours
 aws_cleanup:
 	@ scripts/evergreen/prepare_aws.sh
 
 build-and-push-operator-image: aws_login
-	@ ./pipeline.py --include operator-quick
+	@ scripts/evergreen/run_pipelinepy.sh --include operator-quick
 
 build-and-push-database-image: aws_login
 	@ scripts/dev/build_push_database_image
 
 build-and-push-test-image: aws_login build-multi-cluster-binary
 	@ if [[ -z "$(local)" ]]; then \
-		./pipeline.py --include test; \
+		scripts/evergreen/run_pipelinepy.sh --include test; \
 	fi
 
 build-multi-cluster-binary:
@@ -168,13 +145,13 @@ build-and-push-images: build-and-push-operator-image appdb-init-image om-init-im
 	@ $(MAKE) database-init-image
 
 database-init-image:
-	@ ./pipeline.py --include init-database
+	@ scripts/evergreen/run_pipelinepy.sh --include init-database
 
 appdb-init-image:
-	@ ./pipeline.py --include init-appdb
+	@ scripts/evergreen/run_pipelinepy.sh --include init-appdb
 
 om-init-image:
-	@ ./pipeline.py --include init-ops-manager
+	@ scripts/evergreen/run_pipelinepy.sh --include init-ops-manager
 
 deploy-operator:
 	@ scripts/dev/deploy_operator.sh $(debug)
@@ -184,19 +161,13 @@ configure-operator:
 
 deploy-and-configure-operator: deploy-operator configure-operator
 
-ensure-k8s:
-	@ scripts/dev/ensure_k8s.sh
-
 cert:
 	@ openssl req  -nodes -new -x509  -keyout ca-tls.key -out ca-tls.crt -extensions v3_ca -days 3650
 	@ mv ca-tls.key ca-tls.crt docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/
 	@ cat docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls.crt \
 	docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/mongodb-download.crt \
 	> docker/mongodb-enterprise-tests/tests/opsmanager/fixtures/ca-tls-full-chain.crt
 
-ensure-k8s-and-reset: ensure-k8s
-	@ $(MAKE) reset
-
 .PHONY: recreate-e2e-multicluster-kind
 recreate-e2e-multicluster-kind:
 	scripts/dev/recreate_kind_clusters.sh
@@ -288,7 +259,7 @@ undeploy:
 
 # Generate manifests e.g. CRD, RBAC etc.
 manifests: controller-gen
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths=./... output:crd:artifacts:config=config/crd/bases
+	export PATH=$(PATH); export GOROOT=$(GOROOT); $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths=./... output:crd:artifacts:config=config/crd/bases
 	# copy the CRDs to the public folder
 	cp config/crd/bases/* helm_chart/crds/
 	cat "helm_chart/crds/"* > public/crds.yaml

.rsyncinclude

@@ -167,7 +167,7 @@ def evergreen_task_id() -> str:
 
 
 def get_evergreen_task_id():
-    return os.environ.get("TASK_ID", "")
+    return os.environ.get("task_id", "")
 
 
 @fixture(scope="module")

Makefile

@@ -75,27 +75,6 @@ def make_list_of_str(value: Union[None, str, List[str]]) -> List[str]:
     return value
 
 
-def build_configuration_from_context_file(filename: str) -> Dict[str, str]:
-    config = {}
-    with open(filename) as fd:
-        for line in fd.readlines():
-            if line.startswith("#") or line.strip() == "":
-                continue
-
-            key, value = line.split("=")
-            key = key.replace("export ", "").lower()
-            value = value.strip().replace('"', "")
-            config[key] = value
-
-    # calculates skip_tags from image_type in local mode
-    config["skip_tags"] = list(skippable_tags - {config["image_type"]})
-
-    # explicitly skipping release tags locally
-    config["skip_tags"].append("release")
-
-    return config
-
-
 def build_configuration_from_env() -> Dict[str, str]:
     """Builds a running configuration by reading values from environment.
     This is to be used in Evergreen environment.
@@ -105,7 +84,7 @@ def build_configuration_from_env() -> Dict[str, str]:
     # would replace the `username` we use locally.
     return {
         "image_type": os.environ.get("distro"),
-        "base_repo_url": os.environ["registry"] + "/dev",
+        "base_repo_url": os.environ["BASE_REPO_URL"],
         "include_tags": os.environ.get("include_tags"),
         "skip_tags": os.environ.get("skip_tags"),
     }
@@ -114,16 +93,7 @@ def build_configuration_from_env() -> Dict[str, str]:
 def operator_build_configuration(
     builder: str, parallel: bool, debug: bool
 ) -> BuildConfiguration:
-    # TODO: This will be fixed/changed once we update the local dev tooling
-    default_config_location = os.path.expanduser("~/.operator-dev/context.env")
-    context_file = os.environ.get(
-        "OPERATOR_BUILD_CONFIGURATION", default_config_location
-    )
-
-    if os.path.exists(context_file):
-        context = build_configuration_from_context_file(context_file)
-    else:
-        context = build_configuration_from_env()
+    context = build_configuration_from_env()
 
     print(f"Context: {context}")
 

docker/mongodb-enterprise-tests/tests/conftest.py

@@ -195,17 +195,17 @@ mkdir -p "${log_dir}" &>/dev/null
 
 if [ -n "${CENTRAL_CLUSTER}" ]; then
   if [ -z "${MEMBER_CLUSTERS}" ]; then
-    echo "central_cluster is set but no member_clusters"
+    echo "CENTRAL_CLUSTER is set but no MEMBER_CLUSTERS"
     exit 1
   else
-    echo "starting with the central_cluster!"
+    echo "starting with the CENTRAL_CLUSTER!"
     kubectl config use-context "${CENTRAL_CLUSTER}"
   fi
 fi
 
 if [ -n "${MEMBER_CLUSTERS}" ]; then
   if [ -z "${CENTRAL_CLUSTER}" ]; then
-    echo "member_clusters is set but no central_cluster"
+    echo "MEMBER_CLUSTERS is set but no CENTRAL_CLUSTER"
     exit 1
   fi
 fi

pipeline.py

@@ -2,6 +2,7 @@
 
 set -Eeou pipefail
 
+source scripts/dev/set_env_context.sh
 source scripts/funcs/checks
 source scripts/funcs/printing
 source scripts/funcs/kubernetes

public/support/mdb_operator_diagnostic_data.sh

@@ -42,3 +42,4 @@ kubectl delete secret ${om_admin_secret} -n "${NAMESPACE}" 2>/dev/null || true
 kubectl create secret generic ${om_admin_secret}  --from-literal=Username="[email protected]" --from-literal=Password="Passw0rd."  --from-literal=FirstName="Jane" --from-literal=LastName="Doe" -n "${NAMESPACE}"
 
 title "All necessary ConfigMaps and Secrets for the Operator are configured"
+

scripts/dev/apply_resources

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+# shellcheck disable=SC1091
+source "$script_dir/root-context"
+# shellcheck disable=SC1091
+source "$script_dir/variables/om50_image"

scripts/dev/configure_docker_auth.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+# shellcheck disable=SC1091
+source "$script_dir/root-context"
+# shellcheck disable=SC1091
+source "$script_dir/variables/om60_image"

scripts/dev/configure_operator.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+# shellcheck disable=SC1091
+source "$script_dir/root-context"
+# shellcheck disable=SC1091
+source "$script_dir/variables/om50"
+
+export KUBE_ENVIRONMENT_NAME=kind
+export CUSTOM_MDB_VERSION=5.0.5
+export CUSTOM_MDB_PREV_VERSION=4.4.20

scripts/dev/contexts/build_om50_images

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+# shellcheck disable=SC1091
+source "$script_dir/root-context"
+
+export ops_manager_version="cloud_qa"

scripts/dev/contexts/build_om60_images

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -Eeou pipefail
+
+script_name=$(readlink -f "${BASH_SOURCE[0]}")
+script_dir=$(dirname "${script_name}")
+
+# shellcheck disable=SC1091
+source "$script_dir/root-context"
+
+export KUBE_ENVIRONMENT_NAME=openshift_4
+export CLUSTER_TYPE="openshift"
+export ecr_registry_needs_auth=ecr-registry
+export MANAGED_SECURITY_CONTEXT="true"
+export ALWAYS_REMOVE_TESTING_NAMESPACE="true"
+export ops_manager_version="cloud_qa"
+
+export CUSTOM_MDB_VERSION=4.4.20
+
+
+# shellcheck disable=SC2154
+export OPENSHIFT_URL="$openshift_url"
+# shellcheck disable=SC2154
+export OPENSHIFT_TOKEN="$openshift_token"