Merge pull request #5489 from nawazkh/enable_local_testing

Add documentation and scripts on running e2e test locally

Author: k8s-ci-robot

Makefile

@@ -186,6 +186,7 @@ E2E_CONF_FILE_ENVSUBST := $(ROOT_DIR)/test/e2e/config/azure-dev-envsubst.yaml
 SKIP_CLEANUP ?= false
 AZWI_SKIP_CLEANUP ?= false
 SKIP_LOG_COLLECTION ?= false
+MGMT_CLUSTER_TYPE ?= kind
 # @sonasingh46: Skip creating mgmt cluster for ci as workload identity needs kind cluster
 # to be created with extra mounts for key pairs which is not yet supported
 # by existing e2e framework. A mgmt cluster(kind) is created as part of e2e suite
@@ -317,8 +318,12 @@ install-tools: $(ENVSUBST) $(KUSTOMIZE) $(KUBECTL) $(HELM) $(GINKGO) $(KIND) $(A
 
 .PHONY: create-management-cluster
 create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create a management cluster.
-	# Create kind management cluster.
-	$(MAKE) kind-create
+	# Create management cluster based on type
+	@if [ "$(MGMT_CLUSTER_TYPE)" = "aks" ]; then \
+		$(MAKE) aks-create; \
+	else \
+		$(MAKE) kind-create; \
+	fi
 
 	# Install cert manager and wait for availability
 	./hack/install-cert-manager.sh
@@ -334,7 +339,9 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 	timeout --foreground 300 bash -c "until curl --retry $(CURL_RETRIES) -sSL https://github.com/kubernetes-sigs/cluster-api-addon-provider-helm/releases/download/v0.2.5/addon-components.yaml | $(ENVSUBST) | $(KUBECTL) apply -f -; do sleep 5; done"
 
 	# Deploy CAPZ
-	$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME)
+	if [ "$(MGMT_CLUSTER_TYPE)" != "aks" ]; then \
+		$(KIND) load docker-image $(CONTROLLER_IMG)-$(ARCH):$(TAG) --name=$(KIND_CLUSTER_NAME); \
+	fi
 	timeout --foreground 300 bash -c "until $(KUSTOMIZE) build config/default | $(ENVSUBST) | $(KUBECTL) apply -f - --server-side=true; do sleep 5; done"
 
 	# Wait for CAPI deployments
@@ -360,7 +367,10 @@ create-management-cluster: $(KUSTOMIZE) $(ENVSUBST) $(KUBECTL) $(KIND) ## Create
 	timeout --foreground 300 bash -c "until $(KUBECTL) get clusters -A; do sleep 3; done"
 	timeout --foreground 300 bash -c "until $(KUBECTL) get azureclusters -A; do sleep 3; done"
 	timeout --foreground 300 bash -c "until $(KUBECTL) get kubeadmcontrolplanes -A; do sleep 3; done"
-	@echo 'Set kubectl context to the kind management cluster by running "$(KUBECTL) config set-context kind-$(KIND_CLUSTER_NAME)"'
+
+	@if [ "$(MGMT_CLUSTER_TYPE)" != "aks" ]; then \
+		echo 'Set kubectl context to the kind management cluster by running "$(KUBECTL) config set-context kind-$(KIND_CLUSTER_NAME)"'; \
+	fi
 
 .PHONY: create-workload-cluster
 create-workload-cluster: $(ENVSUBST) $(KUBECTL) ## Create a workload cluster.
@@ -726,6 +736,17 @@ test-cover: test ## Run tests with code coverage and generate reports.
 kind-create-bootstrap: $(KUBECTL) ## Create capz kind bootstrap cluster.
 	KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh
 
+.PHONY: create-bootstrap
+create-bootstrap: $(KUBECTL) ## Create bootstrap cluster (AKS or KIND) for CAPZ testing. Default is KIND.
+	@echo "Creating bootstrap cluster with type: $(MGMT_CLUSTER_TYPE)"
+	@if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
+		MGMT_CLUSTER_NAME="$${MGMT_CLUSTER_NAME:-capz-e2e-$(shell date +%s)}" \
+		./scripts/aks-as-mgmt.sh || { echo "Failed to create AKS bootstrap cluster" >&2; exit 1; }; \
+	else \
+		KIND_CLUSTER_NAME=capz-e2e ./scripts/kind-with-registry.sh || { echo "Failed to create KIND bootstrap cluster" >&2; exit 1; }; \
+	fi
+	@echo "Bootstrap cluster created successfully"
+
 .PHONY: cleanup-workload-identity
 cleanup-workload-identity: ## Cleanup CI workload-identity infra
 	@if ! [ "$(AZWI_SKIP_CLEANUP)" == "true" ]; then \

docs/book/src/developers/development.md

@@ -507,6 +507,8 @@ To run E2E locally, set `AZURE_CLIENT_ID`, `AZURE_CLIENT_SECRET`, `AZURE_SUBSCRI
 ./scripts/ci-e2e.sh
 ```
 
+Note: Users that have a restrictive environment and want to leverage API Server ILB in their flavors and to run e2e tests locally should refer to the detailed explanation in [running e2e tests locally leveraging apiserver ilb solution](./tilt-with-aks-as-mgmt-ilb.md#running-e2e-tests-locally-using-api-server-ilbs-networking-solution)
+
 You can optionally set the following variables:
 
 | Variable                   | Description                                                                                                                                                                                                                                                                                                                                                                | Default                                                                               |

docs/book/src/developers/tilt-with-aks-as-mgmt-ilb.md

@@ -38,6 +38,7 @@ While the default Tilt setup recommends using a KIND cluster as the management c
    export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY="<user-assigned-managed-identity-client-id>"
    export AZURE_CLIENT_ID="${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}"
    export AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY="<user-assigned-managed-identity--object-id>"
+   export AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID="<resource-id-of-user-assigned-managed-id-from-json-view>"
    export AZURE_LOCATION="<azure-location-having-quota-for-B2s-and-D4s_v3-SKU>"
    export REGISTRY=<your-container-registry>
    ```
@@ -76,6 +77,158 @@ While the default Tilt setup recommends using a KIND cluster as the management c
          - [apiserver-ilb](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/cluster-template-apiserver-ilb.yaml): VM-based default flavor that brings up native K8s clusters with Linux nodes.
          - [apiserver-ilb-windows](https://github.com/kubernetes-sigs/cluster-api-provider-azure/blob/main/templates/cluster-template-windows-apiserver-ilb.yaml): VM-based flavor that brings up native K8s clusters with Linux and Windows nodes.
 
+## Running e2e tests locally using API Server ILB's networking solution
+
+Running an e2e test locally in a restricted environment calls for some workarounds in the prow templates and the e2e test itself.
+
+1. We need to add the apiserver ILB with private endpoints and predeterimined CIDRs to the workload cluster's VNet & Subnets, and pre-kubeadm commands updating the `/etc/hosts` file of the nodes of the workload cluster.
+
+2. Once the template has been modified to be run in local environment using AKS as management cluster, we need to be able to peer the vnets, create private DNS zone for the FQDN of the workload cluster and re-enable blocked NSG ports.
+
+**Note:**
+
+- The following guidance is only for debugging, and is not a recommendation for any production environment.
+
+- The below steps are for self-managed templates only and do not apply to AKS workload clusters.
+
+- If you are going to run the local tests from a dev machine in Azure, you will have to use user-assigned managed identity and assign it to the management cluster. Follow the below steps before proceeding.
+  1. Create a user-assigned managed identity
+  2. Assign that managed identity a contributor role to your subscription
+  3. Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY`, and `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to the user-assigned managed identity.
+
+#### Update prow template with apiserver ILB networking solution
+
+There are three sections of a prow template that need an update.
+
+1. AzureCluster
+   - `/spec/networkSpec/apiServerLB`
+      - Add FrontendIP
+      - Add an associated private IP to be leveraged by an internal ILB
+   - `/spec/networkSpec/vnet/cidrBlocks`
+      - Add VNet CIDR
+   - `/spec/networkSpec/subnets/0/cidrBlocks`
+      - Add Subnet CIDR for the control plane
+   - `/spec/networkSpec/subnets/1/cidrBlocks`
+      - Add Subnet CIDR for the worker node
+2. `KubeadmConfigTemplate` - linux node; Identifiable by `name: .*-md-0`
+   - `/spec/template/spec/preKubeadmCommands/0`
+      - Add a prekubeadm command updating the `/etc/hosts` of worker nodes of type "linux".
+3. `KubeadmConfigTemplate` - windows node; Identifiable by `name: .*-md-win`
+   - `/spec/template/spec/preKubeadmCommands/0`
+      - Add a prekubeadm command updating the `/etc/hosts` of worker nodes of type "windows".
+
+A sample kustomize command for updating a prow template via its kustomization.yaml is pasted below.
+
+```yaml
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/apiServerLB
+      value:
+        frontendIPs:
+        - name: ${CLUSTER_NAME}-api-lb
+          publicIP:
+            dnsName: ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com
+            name: ${CLUSTER_NAME}-api-lb
+        - name: ${CLUSTER_NAME}-internal-lb-private-ip
+          privateIP: ${AZURE_INTERNAL_LB_PRIVATE_IP}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/vnet/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/vnet/cidrBlocks/-
+      value: ${AZURE_VNET_CIDR}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/subnets/0/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/subnets/0/cidrBlocks/-
+      value: ${AZURE_CP_SUBNET_CIDR}
+- target:
+    kind: AzureCluster
+  patch: |-
+    - op: add
+      path: /spec/networkSpec/subnets/1/cidrBlocks
+      value: []
+    - op: add
+      path: /spec/networkSpec/subnets/1/cidrBlocks/-
+      value: ${AZURE_NODE_SUBNET_CIDR}
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-0
+  patch: |-
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands
+      value: []
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value: echo '${AZURE_INTERNAL_LB_PRIVATE_IP}   ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com' >> /etc/hosts
+- target:
+    kind: KubeadmConfigTemplate
+    name: .*-md-win
+  patch: |-
+    - op: add
+      path: /spec/template/spec/preKubeadmCommands/-
+      value:
+        powershell -Command "Add-Content -Path 'C:\\Windows\\System32\\drivers\\etc\\hosts' -Value '${AZURE_INTERNAL_LB_PRIVATE_IP} ${CLUSTER_NAME}-${APISERVER_LB_DNS_SUFFIX}.${AZURE_LOCATION}.cloudapp.azure.com'"
+```
+
+#### Peer Vnets of the management cluster and the workload cluster
+
+Peering VNets, creating a private DNS zone with the FQDN of the workload cluster, and updating NSGs of the management and workload clusters can be achieved by running `scripts/peer-vnets.sh`.
+
+This script, `scripts/peer-vnets.sh`, should be run after triggering the test run locally and from a separate terminal.
+
+#### Running the test locally
+
+We recommend running each test individually while debugging the test failure. This implies that `GINKGO_FOCUS` is as unique as possible. So for instance if you want to run `periodic-cluster-api-provider-azure-e2e-main`'s "With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes" test,
+
+1. We first need to add the following environment variables to the test itself. For example:
+
+   ```go
+   Expect(os.Setenv("EXP_APISERVER_ILB", "true")).To(Succeed())
+   Expect(os.Setenv("AZURE_INTERNAL_LB_PRIVATE_IP", "10.0.0.101")).To(Succeed())
+   Expect(os.Setenv("AZURE_VNET_CIDR", "10.0.0.0/8")).To(Succeed())
+   Expect(os.Setenv("AZURE_CP_SUBNET_CIDR", "10.0.0.0/16")).To(Succeed())
+   Expect(os.Setenv("AZURE_NODE_SUBNET_CIDR", "10.1.0.0/16")).To(Succeed())
+   ```
+
+   The above lines should be added before the `clusterctl.ApplyClusterTemplateAndWait()` is invoked.
+
+
+2. Open the terminal and run the below command:
+
+   ```bash
+   GINKGO_FOCUS="With 3 control-plane nodes and 2 Linux and 2 Windows worker nodes" USE_LOCAL_KIND_REGISTRY=false SKIP_CLEANUP="true" SKIP_LOG_COLLECTION="true" REGISTRY="<>" MGMT_CLUSTER_TYPE="aks" EXP_APISERVER_ILB=true AZURE_LOCATION="<>" ARCH="amd64" scripts/ci-e2e.sh
+   ```
+
+   **Note:**
+
+   - Set `MGMT_CLUSTER_TYPE` to `"aks"` to leverage `AKS` as the management cluster.
+   - Set `EXP_APISERVER_ILB` to `true` to enable the API Server ILB feature gate.
+   - Set `AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY`, `AZURE_OBJECT_ID_USER_ASSIGNED_IDENTITY` and `AZURE_USER_ASSIGNED_IDENTITY_RESOURCE_ID` to use the user-assigned managed identity instead of the AKS-created managed identity.
+
+3. In a new terminal, wait for AzureClusters to be created by the above command. Check it using `kubectl get AzureClusters -A`. Note that this command will fail or will not output anything unless the above command, `GINKGO_FOCUS...`, has deployed the worker template and initiated workload cluster creation.
+
+   Once the worker cluster has been created, `export` the `CLUSTER_NAME` and `CLUSTER_NAMESPACE`.
+   It is recommended that `AZURE_INTERNAL_LB_PRIVATE_IP` is set an IP of `10.0.0.x`, say `10.0.0.101`, to avoid any test updates.
+
+   Then open a new terminal at the root of the cluster api provider azure repo and run the below command.
+
+   ```bash
+   AZURE_INTERNAL_LB_PRIVATE_IP="<Internal-IP-from-the-e2e-test>" CLUSTER_NAME="<e2e workload-cluster-name>" CLUSTER_NAMESPACE="<e2e-cluster-namespace>" ./scripts/peer-vnets.sh ./tilt-settings.yaml
+   ```
+
+You will see that the test progresses in the first terminal window that invoked `GINKGO_FOCUS=....`
+
+
 ## Leveraging internal load balancer
 
 By default using Tilt with Cluster API Provider Azure (CAPZ), the management cluster is exposed via a public endpoint. This works well for many development scenarios but presents challenges in environments with strict network security requirements.
@@ -127,4 +280,3 @@ By default using Tilt with Cluster API Provider Azure (CAPZ), the management clu
    **Solution:**
    - Use 3 control plane nodes in a stacked etcd setup.
       - Using aks as management cluster sets `CONTROL_PLANE_MACHINE_COUNT` to 3 by default.
-

e2e.mk

@@ -5,8 +5,11 @@
 
 ##@ E2E Testing:
 .PHONY: test-e2e-run
-test-e2e-run: generate-e2e-templates install-tools kind-create-bootstrap ## Run e2e tests.
-	@$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
+test-e2e-run: generate-e2e-templates install-tools create-bootstrap ## Run e2e tests.
+	if [ "$(MGMT_CLUSTER_TYPE)" == "aks" ]; then \
+		source ./scripts/peer-vnets.sh && source_tilt_settings tilt-settings.yaml; \
+	fi; \
+	$(ENVSUBST) < $(E2E_CONF_FILE) > $(E2E_CONF_FILE_ENVSUBST) && \
 	if [ -z "${AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY}" ]; then \
 		export AZURE_CLIENT_ID_USER_ASSIGNED_IDENTITY=$(shell cat $(AZURE_IDENTITY_ID_FILEPATH)); \
 	fi; \