Skip to content

Commit 1d2c092

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #5689 from nojnhuh/e2e-addons
Don't install addons twice in e2e
2 parents 8f27a37 + a031498 commit 1d2c092

File tree

14 files changed

+210
-247
lines changed

14 files changed

+210
-247
lines changed

‎templates/addons/cluster-api-helm/gpu-operator.yaml

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
apiVersion: addons.cluster.x-k8s.io/v1alpha1
2+
kind: HelmChartProxy
3+
metadata:
4+
name: gpu-operator
5+
spec:
6+
clusterSelector:
7+
matchLabels:
8+
gpu-operator: "true"
9+
repoURL: https://helm.ngc.nvidia.com/nvidia
10+
chartName: gpu-operator
11+
releaseName: nvidia-gpu-operator
12+
namespace: default

‎templates/test/ci/cluster-template-prow-nvidia-gpu.yaml

Lines changed: 15 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎templates/test/ci/patches/cluster-label-gpu-operator.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
apiVersion: cluster.x-k8s.io/v1beta1
2+
kind: Cluster
3+
metadata:
4+
name: ${CLUSTER_NAME}
5+
labels:
6+
gpu-operator: "true"

‎templates/test/ci/prow-nvidia-gpu/kustomization.yaml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ resources:
77
- ../../../addons/cluster-api-helm/azuredisk-csi-driver.yaml
88
- ../../../addons/cluster-api-helm/cloud-provider-azure.yaml
99
- ../../../addons/cluster-api-helm/cloud-provider-azure-ci.yaml
10+
- ../../../addons/cluster-api-helm/gpu-operator.yaml
1011
patches:
1112
- path: patches/node-storage-type.yaml
1213
target:
@@ -19,6 +20,7 @@ patches:
1920
- path: ../patches/azurecluster-gpu.yaml
2021
- path: ../patches/cluster-label-calico.yaml
2122
- path: ../patches/cluster-label-cloud-provider-azure.yaml
23+
- path: ../patches/cluster-label-gpu-operator.yaml
2224

2325
sortOptions:
2426
order: fifo

‎test/e2e/azure_test.go

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -161,7 +161,7 @@ var _ = Describe("Workload cluster creation", func() {
161161
withControlPlaneMachineCount(1),
162162
withWorkerMachineCount(1),
163163
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
164-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
164+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
165165
}),
166166
withPostMachinesProvisioned(func() {
167167
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -211,7 +211,7 @@ var _ = Describe("Workload cluster creation", func() {
211211
withWorkerMachineCount(2),
212212
withControlPlaneInterval(specName, "wait-control-plane-ha"),
213213
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
214-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
214+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
215215
}),
216216
withPostMachinesProvisioned(func() {
217217
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -309,7 +309,7 @@ var _ = Describe("Workload cluster creation", func() {
309309
withWorkerMachineCount(2),
310310
withControlPlaneInterval(specName, "wait-control-plane-ha"),
311311
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
312-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
312+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
313313
}),
314314
withPostMachinesProvisioned(func() {
315315
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -368,7 +368,7 @@ var _ = Describe("Workload cluster creation", func() {
368368
withControlPlaneMachineCount(1),
369369
withWorkerMachineCount(1),
370370
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
371-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
371+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
372372
}),
373373
withPostMachinesProvisioned(func() {
374374
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -406,7 +406,7 @@ var _ = Describe("Workload cluster creation", func() {
406406
withControlPlaneMachineCount(1),
407407
withWorkerMachineCount(1),
408408
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
409-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
409+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
410410
}),
411411
withPostMachinesProvisioned(func() {
412412
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -443,7 +443,7 @@ var _ = Describe("Workload cluster creation", func() {
443443
withControlPlaneMachineCount(1),
444444
withWorkerMachineCount(1),
445445
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
446-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
446+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
447447
}),
448448
withPostMachinesProvisioned(func() {
449449
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -481,7 +481,7 @@ var _ = Describe("Workload cluster creation", func() {
481481
withWorkerMachineCount(1),
482482
withControlPlaneInterval(specName, "wait-control-plane-ha"),
483483
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
484-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
484+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
485485
}),
486486
withPostMachinesProvisioned(func() {
487487
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -541,7 +541,7 @@ var _ = Describe("Workload cluster creation", func() {
541541
withControlPlaneInterval(specName, "wait-control-plane"),
542542
withMachinePoolInterval(specName, "wait-machine-pool-nodes"),
543543
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
544-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
544+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
545545
}),
546546
withPostMachinesProvisioned(func() {
547547
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -610,7 +610,7 @@ var _ = Describe("Workload cluster creation", func() {
610610
withWorkerMachineCount(1),
611611
withMachineDeploymentInterval(specName, "wait-gpu-nodes"),
612612
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
613-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
613+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
614614
}),
615615
withPostMachinesProvisioned(func() {
616616
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -620,8 +620,8 @@ var _ = Describe("Workload cluster creation", func() {
620620
ClusterName: clusterName,
621621
}
622622
})
623-
InstallGPUOperator(ctx, func() GPUOperatorSpecInput {
624-
return GPUOperatorSpecInput{
623+
EnsureGPUOperator(ctx, func() EnsureGPUOperatorInput {
624+
return EnsureGPUOperatorInput{
625625
BootstrapClusterProxy: bootstrapClusterProxy,
626626
Namespace: namespace,
627627
ClusterName: clusterName,
@@ -668,7 +668,7 @@ var _ = Describe("Workload cluster creation", func() {
668668
withWorkerMachineCount(1),
669669
withMachineDeploymentInterval(specName, ""),
670670
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
671-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
671+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
672672
}),
673673
withMachinePoolInterval(specName, "wait-machine-pool-nodes"),
674674
withControlPlaneInterval(specName, "wait-control-plane"),
@@ -1010,7 +1010,7 @@ var _ = Describe("Workload cluster creation", func() {
10101010
withWorkerMachineCount(1),
10111011
withControlPlaneInterval(specName, "wait-control-plane-ha"),
10121012
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1013-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
1013+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
10141014
}),
10151015
withPostMachinesProvisioned(func() {
10161016
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1084,7 +1084,7 @@ var _ = Describe("Workload cluster creation", func() {
10841084
withControlPlaneMachineCount(1),
10851085
withWorkerMachineCount(1),
10861086
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1087-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
1087+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
10881088
}),
10891089
withPostMachinesProvisioned(func() {
10901090
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1182,7 +1182,7 @@ var _ = Describe("Workload cluster creation", func() {
11821182
withControlPlaneMachineCount(1),
11831183
withWorkerMachineCount(1),
11841184
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1185-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
1185+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
11861186
}),
11871187
withPostMachinesProvisioned(func() {
11881188
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1225,7 +1225,7 @@ var _ = Describe("Workload cluster creation", func() {
12251225
withWorkerMachineCount(2),
12261226
withControlPlaneInterval(specName, "wait-control-plane-ha"),
12271227
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1228-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
1228+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
12291229
}),
12301230
withPostMachinesProvisioned(func() {
12311231
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {
@@ -1276,7 +1276,7 @@ var _ = Describe("Workload cluster creation", func() {
12761276
withWorkerMachineCount(2),
12771277
withControlPlaneInterval(specName, "wait-control-plane-ha"),
12781278
withControlPlaneWaiters(clusterctl.ControlPlaneWaiters{
1279-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
1279+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
12801280
}),
12811281
withPostMachinesProvisioned(func() {
12821282
EnsureDaemonsets(ctx, func() DaemonsetsSpecInput {

‎test/e2e/capi_test.go

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
8181
ArtifactFolder: artifactFolder,
8282
SkipCleanup: skipCleanup,
8383
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
84-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
84+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
8585
},
8686
}
8787
})
@@ -96,7 +96,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
9696
ArtifactFolder: artifactFolder,
9797
SkipCleanup: skipCleanup,
9898
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
99-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
99+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
100100
},
101101
}
102102
})
@@ -112,7 +112,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
112112
ArtifactFolder: artifactFolder,
113113
SkipCleanup: skipCleanup,
114114
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
115-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
115+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
116116
},
117117
}
118118
})
@@ -129,7 +129,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
129129
ArtifactFolder: artifactFolder,
130130
SkipCleanup: skipCleanup,
131131
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
132-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
132+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
133133
},
134134
}
135135
})
@@ -144,7 +144,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
144144
ArtifactFolder: artifactFolder,
145145
SkipCleanup: skipCleanup,
146146
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
147-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
147+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
148148
},
149149
}
150150
})
@@ -159,7 +159,7 @@ var _ = Describe("Running the Cluster API E2E tests", func() {
159159
ArtifactFolder: artifactFolder,
160160
SkipCleanup: skipCleanup,
161161
ControlPlaneWaiters: clusterctl.ControlPlaneWaiters{
162-
WaitForControlPlaneInitialized: EnsureControlPlaneInitializedNoAddons,
162+
WaitForControlPlaneInitialized: EnsureControlPlaneInitialized,
163163
},
164164
}
165165
})

‎test/e2e/cloud-provider-azure.go

Lines changed: 20 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,6 @@ package e2e
2121

2222
import (
2323
"context"
24-
"fmt"
25-
"os"
26-
"strings"
2724

2825
. "github.com/onsi/ginkgo/v2"
2926
. "github.com/onsi/gomega"
@@ -44,40 +41,14 @@ const (
4441

4542
// EnsureCNIAndCloudProviderAzureHelmChart installs the official cloud-provider-azure helm chart
4643
// and a CNI and validates that expected pods exist and are Ready.
47-
func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, cidrBlocks []string, hasWindows bool) {
44+
func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, hasWindows bool) {
4845
specName := "ensure-cloud-provider-azure"
4946
clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName)
5047

51-
if installHelmChart {
52-
By("Installing cloud-provider-azure components via helm")
53-
options := &HelmOptions{
54-
Values: []string{
55-
fmt.Sprintf("infra.clusterName=%s", input.ClusterName),
56-
"cloudControllerManager.logVerbosity=4",
57-
},
58-
StringValues: []string{fmt.Sprintf("cloudControllerManager.clusterCIDR=%s", strings.Join(cidrBlocks, `\,`))},
59-
}
60-
// If testing a CI version of Kubernetes, use CCM and CNM images built from source.
61-
if useCIArtifacts || usePRArtifacts {
62-
options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageName=%s", os.Getenv("CCM_IMAGE_NAME")))
63-
options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageName=%s", os.Getenv("CNM_IMAGE_NAME")))
64-
options.Values = append(options.Values, fmt.Sprintf("cloudControllerManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY")))
65-
options.Values = append(options.Values, fmt.Sprintf("cloudNodeManager.imageRepository=%s", os.Getenv("IMAGE_REGISTRY")))
66-
options.StringValues = append(options.StringValues, fmt.Sprintf("cloudControllerManager.imageTag=%s", os.Getenv("IMAGE_TAG_CCM")))
67-
options.StringValues = append(options.StringValues, fmt.Sprintf("cloudNodeManager.imageTag=%s", os.Getenv("IMAGE_TAG_CNM")))
68-
}
69-
70-
if strings.Contains(input.ClusterName, "flatcar") {
71-
options.StringValues = append(options.StringValues, "cloudControllerManager.caCertDir=/usr/share/ca-certificates")
72-
}
73-
74-
InstallHelmChart(ctx, clusterProxy, defaultNamespace, cloudProviderAzureHelmRepoURL, cloudProviderAzureChartName, cloudProviderAzureHelmReleaseName, options, "")
75-
} else {
76-
By("Ensuring cloud-provider-azure is installed via CAAPH")
77-
}
48+
By("Ensuring cloud-provider-azure is installed via CAAPH")
7849

7950
// We do this before waiting for the pods to be ready because there is a co-dependency between CNI (nodes ready) and cloud-provider being initialized.
80-
EnsureCNI(ctx, input, installHelmChart, cidrBlocks, hasWindows)
51+
EnsureCNI(ctx, input, hasWindows)
8152

8253
By("Waiting for Ready cloud-controller-manager deployment pods")
8354
for _, d := range []string{"cloud-controller-manager"} {
@@ -87,40 +58,28 @@ func EnsureCNIAndCloudProviderAzureHelmChart(ctx context.Context, input clusterc
8758
}
8859

8960
// EnsureAzureDiskCSIDriverHelmChart installs the official azure-disk CSI driver helm chart
90-
func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput, installHelmChart bool, hasWindows bool) {
61+
func EnsureAzureDiskCSIDriverHelmChart(ctx context.Context, input clusterctl.ApplyCustomClusterTemplateAndWaitInput) {
9162
specName := "ensure-azuredisk-csi-drivers"
9263
clusterProxy := input.ClusterProxy.GetWorkloadCluster(ctx, input.Namespace, input.ClusterName)
9364
mgmtClient := input.ClusterProxy.GetClient()
9465

95-
if installHelmChart {
96-
By("Installing azure-disk CSI driver components via helm")
97-
options := &HelmOptions{
98-
Values: []string{"controller.replicas=1", "controller.runOnControlPlane=true"},
99-
}
100-
// TODO: make this always true once HostProcessContainers are on for all supported k8s versions.
101-
if hasWindows {
102-
options.Values = append(options.Values, "windows.useHostProcessContainers=true")
103-
}
104-
InstallHelmChart(ctx, clusterProxy, kubesystem, azureDiskCSIDriverHelmRepoURL, azureDiskCSIDriverChartName, azureDiskCSIDriverHelmReleaseName, options, "")
105-
} else {
106-
By("Ensuring azure-disk CSI driver is installed via CAAPH")
107-
cluster := &clusterv1.Cluster{}
108-
Eventually(func(g Gomega) {
109-
g.Expect(mgmtClient.Get(ctx, types.NamespacedName{
110-
Namespace: input.Namespace,
111-
Name: input.ClusterName,
112-
}, cluster)).To(Succeed())
113-
// Label the cluster so that CAAPH installs the azuredisk-csi helm chart via existing HelmChartProxy resource
114-
if cluster.Labels != nil {
115-
cluster.Labels[azureDiskCSIDriverCAAPHLabelName] = "true"
116-
} else {
117-
cluster.Labels = map[string]string{
118-
azureDiskCSIDriverCAAPHLabelName: "true",
119-
}
66+
By("Ensuring azure-disk CSI driver is installed via CAAPH")
67+
cluster := &clusterv1.Cluster{}
68+
Eventually(func(g Gomega) {
69+
g.Expect(mgmtClient.Get(ctx, types.NamespacedName{
70+
Namespace: input.Namespace,
71+
Name: input.ClusterName,
72+
}, cluster)).To(Succeed())
73+
// Label the cluster so that CAAPH installs the azuredisk-csi helm chart via existing HelmChartProxy resource
74+
if cluster.Labels != nil {
75+
cluster.Labels[azureDiskCSIDriverCAAPHLabelName] = "true"
76+
} else {
77+
cluster.Labels = map[string]string{
78+
azureDiskCSIDriverCAAPHLabelName: "true",
12079
}
121-
g.Expect(mgmtClient.Update(ctx, cluster)).To(Succeed())
122-
}, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed())
123-
}
80+
}
81+
g.Expect(mgmtClient.Update(ctx, cluster)).To(Succeed())
82+
}, e2eConfig.GetIntervals(specName, "wait-deployment")...).Should(Succeed())
12483

12584
By("Waiting for Ready csi-azuredisk-controller deployment pods")
12685
for _, d := range []string{"csi-azuredisk-controller"} {

0 commit comments

Comments
 (0)