Skip to content

Commit 5100cfc

Browse files
cbandycbandy
committed
Execute Kubernetes tests in parallel
Issue: [sc-12208]
1 parent 76ee3c2 commit 5100cfc

20 files changed

+277
-226
lines changed

‎.golangci.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ issues:
6464
path: _test[.]go$
6565
text: \`(net/http/httptest|[^`]*testing[^`]*)`
6666

67+
# Allow setupTestEnv and teardownTestEnv to remain unused for a while.
68+
# TODO(cbandy): Remove these functions after setupKubernetes has landed.
69+
- linters: [deadcode, unused]
70+
path: internal/controller/postgrescluster/helpers_test.go
71+
text: \`(setupTestEnv|teardownTestEnv)`
72+
6773
run:
6874
build-tags:
6975
- envtest

‎internal/controller/postgrescluster/apply_test.go

Lines changed: 5 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,26 +37,18 @@ import (
3737
"k8s.io/client-go/discovery"
3838
"sigs.k8s.io/controller-runtime/pkg/client"
3939
"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
40-
"sigs.k8s.io/controller-runtime/pkg/envtest"
40+
41+
"github.com/crunchydata/postgres-operator/internal/testing/require"
4142
)
4243

4344
func TestServerSideApply(t *testing.T) {
44-
// TODO: Update tests that include envtest package to better handle
45-
// running in parallel
46-
// t.Parallel()
47-
4845
ctx := context.Background()
49-
env := &envtest.Environment{}
50-
config, err := env.Start()
51-
assert.NilError(t, err)
52-
t.Cleanup(func() { assert.Check(t, env.Stop()) })
53-
54-
cc, err := client.New(config, client.Options{})
55-
assert.NilError(t, err)
46+
env, cc := setupKubernetes(t)
47+
require.ParallelCapacity(t, 0)
5648

5749
ns := setupNamespace(t, cc)
5850

59-
dc, err := discovery.NewDiscoveryClientForConfig(config)
51+
dc, err := discovery.NewDiscoveryClientForConfig(env.Config)
6052
assert.NilError(t, err)
6153

6254
server, err := dc.ServerVersion()

‎internal/controller/postgrescluster/cluster_test.go

Lines changed: 17 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import (
4343

4444
"github.com/crunchydata/postgres-operator/internal/initialize"
4545
"github.com/crunchydata/postgres-operator/internal/naming"
46+
"github.com/crunchydata/postgres-operator/internal/testing/require"
4647
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
4748
)
4849

@@ -93,13 +94,11 @@ var gvks = []schema.GroupVersionKind{{
9394
}}
9495

9596
func TestCustomLabels(t *testing.T) {
96-
t.Parallel()
97-
98-
env, cc, config := setupTestEnv(t, ControllerName)
99-
t.Cleanup(func() { teardownTestEnv(t, env) })
97+
env, cc := setupKubernetes(t)
98+
require.ParallelCapacity(t, 2)
10099

101100
reconciler := &Reconciler{}
102-
ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
101+
ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
103102
reconciler = &Reconciler{
104103
Client: cc,
105104
Owner: client.FieldOwner(t.Name()),
@@ -351,13 +350,11 @@ func TestCustomLabels(t *testing.T) {
351350
}
352351

353352
func TestCustomAnnotations(t *testing.T) {
354-
t.Parallel()
355-
356-
env, cc, config := setupTestEnv(t, ControllerName)
357-
t.Cleanup(func() { teardownTestEnv(t, env) })
353+
env, cc := setupKubernetes(t)
354+
require.ParallelCapacity(t, 2)
358355

359356
reconciler := &Reconciler{}
360-
ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
357+
ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
361358
reconciler = &Reconciler{
362359
Client: cc,
363360
Owner: client.FieldOwner(t.Name()),
@@ -614,20 +611,18 @@ func TestContainerSecurityContext(t *testing.T) {
614611
t.Skip("Test requires pods to be created")
615612
}
616613

617-
t.Parallel()
618-
619-
env, cc, config := setupTestEnv(t, ControllerName)
620-
t.Cleanup(func() { teardownTestEnv(t, env) })
614+
env, cc := setupKubernetes(t)
615+
require.ParallelCapacity(t, 1)
621616

622617
reconciler := &Reconciler{}
623-
ctx, cancel := setupManager(t, config, func(mgr manager.Manager) {
618+
ctx, cancel := setupManager(t, env.Config, func(mgr manager.Manager) {
624619
reconciler = &Reconciler{
625620
Client: cc,
626621
Owner: client.FieldOwner(t.Name()),
627622
Recorder: mgr.GetEventRecorderFor(ControllerName),
628623
Tracer: otel.Tracer(t.Name()),
629624
}
630-
podExec, err := newPodExecutor(config)
625+
podExec, err := newPodExecutor(env.Config)
631626
assert.NilError(t, err)
632627
reconciler.PodExec = podExec
633628
})
@@ -691,8 +686,8 @@ func TestContainerSecurityContext(t *testing.T) {
691686
}
692687

693688
func TestGenerateClusterPrimaryService(t *testing.T) {
694-
env, cc, _ := setupTestEnv(t, ControllerName)
695-
t.Cleanup(func() { teardownTestEnv(t, env) })
689+
_, cc := setupKubernetes(t)
690+
require.ParallelCapacity(t, 0)
696691

697692
reconciler := &Reconciler{Client: cc}
698693

@@ -792,8 +787,8 @@ subsets:
792787

793788
func TestReconcileClusterPrimaryService(t *testing.T) {
794789
ctx := context.Background()
795-
env, cc, _ := setupTestEnv(t, ControllerName)
796-
t.Cleanup(func() { teardownTestEnv(t, env) })
790+
_, cc := setupKubernetes(t)
791+
require.ParallelCapacity(t, 1)
797792

798793
reconciler := &Reconciler{Client: cc, Owner: client.FieldOwner(t.Name())}
799794

@@ -813,8 +808,8 @@ func TestReconcileClusterPrimaryService(t *testing.T) {
813808
}
814809

815810
func TestGenerateClusterReplicaServiceIntent(t *testing.T) {
816-
env, cc, _ := setupTestEnv(t, ControllerName)
817-
t.Cleanup(func() { teardownTestEnv(t, env) })
811+
_, cc := setupKubernetes(t)
812+
require.ParallelCapacity(t, 0)
818813

819814
reconciler := &Reconciler{Client: cc}
820815

‎internal/controller/postgrescluster/controller_ref_manager_test.go

Lines changed: 5 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -29,25 +29,23 @@ import (
2929
"sigs.k8s.io/controller-runtime/pkg/manager"
3030

3131
"github.com/crunchydata/postgres-operator/internal/naming"
32+
"github.com/crunchydata/postgres-operator/internal/testing/require"
3233
)
3334

3435
func TestManageControllerRefs(t *testing.T) {
36+
tEnv, tClient := setupKubernetes(t)
37+
require.ParallelCapacity(t, 1)
3538

36-
// setup the test environment and ensure a clean teardown
37-
tEnv, tClient, cfg := setupTestEnv(t, ControllerName)
3839
r := &Reconciler{}
39-
ctx, cancel := setupManager(t, cfg, func(mgr manager.Manager) {
40+
ctx, cancel := setupManager(t, tEnv.Config, func(mgr manager.Manager) {
4041
r = &Reconciler{
4142
Client: mgr.GetClient(),
4243
Recorder: mgr.GetEventRecorderFor(ControllerName),
4344
Tracer: otel.Tracer(ControllerName),
4445
Owner: ControllerName,
4546
}
4647
})
47-
t.Cleanup(func() {
48-
teardownManager(cancel, t)
49-
teardownTestEnv(t, tEnv)
50-
})
48+
t.Cleanup(func() { teardownManager(cancel, t) })
5149

5250
clusterName := "hippo"
5351

‎internal/controller/postgrescluster/controller_test.go

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,13 +42,14 @@ import (
4242
"sigs.k8s.io/yaml"
4343

4444
"github.com/crunchydata/postgres-operator/internal/naming"
45+
"github.com/crunchydata/postgres-operator/internal/testing/require"
4546
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
4647
)
4748

4849
func TestDeleteControlled(t *testing.T) {
4950
ctx := context.Background()
50-
tEnv, cc, _ := setupTestEnv(t, t.Name())
51-
t.Cleanup(func() { teardownTestEnv(t, tEnv) })
51+
_, cc := setupKubernetes(t)
52+
require.ParallelCapacity(t, 1)
5253

5354
ns := setupNamespace(t, cc)
5455
reconciler := Reconciler{Client: cc}

‎internal/controller/postgrescluster/delete_test.go

Lines changed: 10 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ import (
2222
"context"
2323
"io"
2424
"os"
25-
"path/filepath"
2625
"strings"
2726
"testing"
2827
"time"
@@ -34,47 +33,27 @@ import (
3433
apierrors "k8s.io/apimachinery/pkg/api/errors"
3534
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3635
"k8s.io/apimachinery/pkg/labels"
37-
"k8s.io/apimachinery/pkg/runtime"
3836
"k8s.io/apimachinery/pkg/util/sets"
3937
"k8s.io/apimachinery/pkg/util/wait"
40-
"k8s.io/client-go/kubernetes/scheme"
4138
"k8s.io/client-go/tools/record"
4239
"sigs.k8s.io/controller-runtime/pkg/client"
43-
"sigs.k8s.io/controller-runtime/pkg/envtest"
4440
"sigs.k8s.io/controller-runtime/pkg/manager"
4541
"sigs.k8s.io/controller-runtime/pkg/reconcile"
4642
"sigs.k8s.io/yaml"
4743

4844
"github.com/crunchydata/postgres-operator/internal/patroni"
45+
"github.com/crunchydata/postgres-operator/internal/testing/require"
4946
"github.com/crunchydata/postgres-operator/pkg/apis/postgres-operator.crunchydata.com/v1beta1"
5047
)
5148

5249
func TestReconcilerHandleDelete(t *testing.T) {
5350
if !strings.EqualFold(os.Getenv("USE_EXISTING_CLUSTER"), "true") {
5451
t.Skip("requires a running garbage collection controller")
5552
}
56-
// TODO: Update tests that include envtest package to better handle
57-
// running in parallel
58-
// t.Parallel()
5953

6054
ctx := context.Background()
61-
env := &envtest.Environment{
62-
CRDDirectoryPaths: []string{
63-
filepath.Join("..", "..", "..", "config", "crd", "bases"),
64-
},
65-
}
66-
67-
options := client.Options{}
68-
options.Scheme = runtime.NewScheme()
69-
assert.NilError(t, scheme.AddToScheme(options.Scheme))
70-
assert.NilError(t, v1beta1.AddToScheme(options.Scheme))
71-
72-
config, err := env.Start()
73-
assert.NilError(t, err)
74-
t.Cleanup(func() { assert.Check(t, env.Stop()) })
75-
76-
cc, err := client.New(config, options)
77-
assert.NilError(t, err)
55+
env, cc := setupKubernetes(t)
56+
require.ParallelCapacity(t, 2)
7857

7958
ns := setupNamespace(t, cc)
8059
reconciler := Reconciler{
@@ -84,7 +63,8 @@ func TestReconcilerHandleDelete(t *testing.T) {
8463
Tracer: otel.Tracer(t.Name()),
8564
}
8665

87-
reconciler.PodExec, err = newPodExecutor(config)
66+
var err error
67+
reconciler.PodExec, err = newPodExecutor(env.Config)
8868
assert.NilError(t, err)
8969

9070
mustReconcile := func(t *testing.T, cluster *v1beta1.PostgresCluster) reconcile.Result {
@@ -353,28 +333,9 @@ func TestReconcilerHandleDeleteNamespace(t *testing.T) {
353333
t.Skip("requires a running garbage collection controller")
354334
}
355335

356-
// TODO: Update tests that include envtest package to better handle
357-
// running in parallel
358-
// t.Parallel()
359-
360336
ctx := context.Background()
361-
env := &envtest.Environment{
362-
CRDDirectoryPaths: []string{
363-
filepath.Join("..", "..", "..", "config", "crd", "bases"),
364-
},
365-
}
366-
367-
options := client.Options{}
368-
options.Scheme = runtime.NewScheme()
369-
assert.NilError(t, scheme.AddToScheme(options.Scheme))
370-
assert.NilError(t, v1beta1.AddToScheme(options.Scheme))
371-
372-
config, err := env.Start()
373-
assert.NilError(t, err)
374-
t.Cleanup(func() { assert.Check(t, env.Stop()) })
375-
376-
cc, err := client.New(config, options)
377-
assert.NilError(t, err)
337+
env, cc := setupKubernetes(t)
338+
require.ParallelCapacity(t, 2)
378339

379340
ns := setupNamespace(t, cc)
380341

@@ -385,11 +346,12 @@ func TestReconcilerHandleDeleteNamespace(t *testing.T) {
385346
Stop context.CancelFunc
386347
}
387348

349+
var err error
388350
mm.Context, mm.Stop = context.WithCancel(context.Background())
389351
mm.Error = make(chan error, 1)
390-
mm.Manager, err = manager.New(config, manager.Options{
352+
mm.Manager, err = manager.New(env.Config, manager.Options{
391353
Namespace: ns.Name,
392-
Scheme: options.Scheme,
354+
Scheme: cc.Scheme(),
393355

394356
HealthProbeBindAddress: "0", // disable
395357
MetricsBindAddress: "0", // disable

‎internal/controller/postgrescluster/helpers_test.go

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"os"
2121
"path/filepath"
2222
"strconv"
23+
"sync"
2324
"testing"
2425
"time"
2526

@@ -69,6 +70,60 @@ func marshalMatches(actual interface{}, expected string) cmp.Comparison {
6970
return cmp.MarshalMatches(actual, expected)
7071
}
7172

73+
var kubernetes struct {
74+
sync.Mutex
75+
76+
env *envtest.Environment
77+
count int
78+
}
79+
80+
// setupKubernetes starts or connects to a Kubernetes API and returns a client
81+
// that uses it. When starting a local API, the client is a member of the
82+
// "system:masters" group. It also creates any CRDs present in the
83+
// "/config/crd/bases" directory. When any of these fail, it calls t.Fatal.
84+
// It deletes CRDs and stops the local API using t.Cleanup.
85+
func setupKubernetes(t testing.TB) (*envtest.Environment, client.Client) {
86+
t.Helper()
87+
88+
kubernetes.Lock()
89+
defer kubernetes.Unlock()
90+
91+
if kubernetes.env == nil {
92+
env := &envtest.Environment{
93+
CRDDirectoryPaths: []string{
94+
filepath.Join("..", "..", "..", "config", "crd", "bases"),
95+
},
96+
}
97+
98+
_, err := env.Start()
99+
assert.NilError(t, err)
100+
101+
kubernetes.env = env
102+
}
103+
104+
kubernetes.count++
105+
106+
t.Cleanup(func() {
107+
kubernetes.Lock()
108+
defer kubernetes.Unlock()
109+
110+
kubernetes.count--
111+
112+
if kubernetes.count == 0 {
113+
assert.Check(t, kubernetes.env.Stop())
114+
kubernetes.env = nil
115+
}
116+
})
117+
118+
scheme, err := runtime.CreatePostgresOperatorScheme()
119+
assert.NilError(t, err)
120+
121+
client, err := client.New(kubernetes.env.Config, client.Options{Scheme: scheme})
122+
assert.NilError(t, err)
123+
124+
return kubernetes.env, client
125+
}
126+
72127
// setupNamespace creates a random namespace that will be deleted by t.Cleanup.
73128
// When creation fails, it calls t.Fatal. The caller may delete the namespace
74129
// at any time.
@@ -137,6 +192,8 @@ func testCluster() *v1beta1.PostgresCluster {
137192

138193
// setupTestEnv configures and starts an EnvTest instance of etcd and the Kubernetes API server
139194
// for test usage, as well as creates a new client instance.
195+
//
196+
// Deprecated: use setupKubernetes instead.
140197
func setupTestEnv(t *testing.T,
141198
_ string) (*envtest.Environment, client.Client, *rest.Config) {
142199

@@ -147,6 +204,7 @@ func setupTestEnv(t *testing.T,
147204
if err != nil {
148205
t.Fatal(err)
149206
}
207+
t.Log("DEPRECATED: Use setupKubernetes instead of setupTestEnv.")
150208
t.Log("Test environment started")
151209

152210
pgoScheme, err := runtime.CreatePostgresOperatorScheme()

0 commit comments

Comments
 (0)