support (force) deleting buckets

Author: AshleyDumaine

api/v1alpha2/linodecluster_types.go

@@ -33,7 +33,8 @@ type LinodeClusterSpec struct {
 	// The Linode Region the LinodeCluster lives in.
 	Region string `json:"region"`
 
-	// ControlPlaneEndpoint represents the endpoint used to communicate with the LinodeCluster control plane.
+	// ControlPlaneEndpoint represents the endpoint used to communicate with
+	// the LinodeCluster control plane.
 	// If ControlPlaneEndpoint is unset then the Nodebalancer ip will be used.
 	// +optional
 	ControlPlaneEndpoint clusterv1.APIEndpoint `json:"controlPlaneEndpoint"`
@@ -47,22 +48,25 @@ type LinodeClusterSpec struct {
 	VPCRef *corev1.ObjectReference `json:"vpcRef,omitempty"`
 
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
-	// VPCID is the ID of an existing VPC in Linode. This allows using a VPC that is not managed by CAPL.
+	// VPCID is the ID of an existing VPC in Linode. This allows using a VPC
+	// that is not managed by CAPL.
 	// +optional
 	VPCID *int `json:"vpcID,omitempty"`
 
 	// +kubebuilder:validation:XValidation:rule="self == oldSelf",message="Value is immutable"
 	// +optional
-	// NodeBalancerFirewallRef is a reference to a NodeBalancer Firewall object. This makes the linode use the specified NodeBalancer Firewall.
+	// NodeBalancerFirewallRef is a reference to a NodeBalancer Firewall object.
+	// This makes the linode use the specified NodeBalancer Firewall.
 	NodeBalancerFirewallRef *corev1.ObjectReference `json:"nodeBalancerFirewallRef,omitempty"`
 
-	// ObjectStore defines a supporting Object Storage bucket for cluster operations. This is currently used for
-	// bootstrapping (e.g. Cloud-init).
+	// ObjectStore defines a supporting Object Storage bucket for cluster
+	// operations. This is currently used for bootstrapping (e.g. Cloud-init).
 	// +optional
 	ObjectStore *ObjectStore `json:"objectStore,omitempty"`
 
-	// CredentialsRef is a reference to a Secret that contains the credentials to use for provisioning this cluster. If not
-	// supplied then the credentials of the controller will be used.
+	// CredentialsRef is a reference to a Secret that contains the credentials
+	// to use for provisioning this cluster. If not supplied then the
+	// credentials of the controller will be used.
 	// +optional
 	CredentialsRef *corev1.SecretReference `json:"credentialsRef,omitempty"`
 }

api/v1alpha2/linodeobjectstoragebucket_types.go

@@ -29,6 +29,8 @@ const (
 	ACLPublicRead        ObjectStorageACL = "public-read"
 	ACLAuthenticatedRead ObjectStorageACL = "authenticated-read"
 	ACLPublicReadWrite   ObjectStorageACL = "public-read-write"
+
+	BucketFinalizer = "linodeobjectstoragebucket.infrastructure.cluster.x-k8s.io"
 )
 
 // LinodeObjectStorageBucketSpec defines the desired state of LinodeObjectStorageBucket
@@ -55,6 +57,14 @@ type LinodeObjectStorageBucketSpec struct {
 	// If not supplied then the credentials of the controller will be used.
 	// +optional
 	CredentialsRef *corev1.SecretReference `json:"credentialsRef"`
+
+	// AccessKeyRef is a reference to a LinodeObjectStorageBucketKey for the bucket.
+	// +optional
+	AccessKeyRef *corev1.ObjectReference `json:"accessKeyRef"`
+
+	// ForceDeleteBucket enables the object storage bucket used to be deleted even if it contains objects.
+	// +optional
+	ForceDeleteBucket bool `json:"forceDeleteBucket,omitempty"`
 }
 
 // LinodeObjectStorageBucketStatus defines the observed state of LinodeObjectStorageBucket

api/v1alpha2/zz_generated.deepcopy.go

@@ -8,6 +8,8 @@ import (
 
 	"github.com/go-logr/logr"
 	"sigs.k8s.io/cluster-api/util/patch"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	infrav1alpha2 "github.com/linode/cluster-api-provider-linode/api/v1alpha2"
 	"github.com/linode/cluster-api-provider-linode/clients"
@@ -77,6 +79,16 @@ func NewObjectStorageBucketScope(ctx context.Context, linodeClientConfig ClientC
 	}, nil
 }
 
+// AddFinalizer adds a finalizer if not present and immediately patches the
+// object to avoid any race conditions.
+func (s *ObjectStorageBucketScope) AddFinalizer(ctx context.Context) error {
+	if controllerutil.AddFinalizer(s.Bucket, infrav1alpha2.BucketFinalizer) {
+		return s.Close(ctx)
+	}
+
+	return nil
+}
+
 // PatchObject persists the object storage bucket configuration and status.
 func (s *ObjectStorageBucketScope) PatchObject(ctx context.Context) error {
 	return s.PatchHelper.Patch(ctx, s.Bucket)
@@ -86,3 +98,54 @@ func (s *ObjectStorageBucketScope) PatchObject(ctx context.Context) error {
 func (s *ObjectStorageBucketScope) Close(ctx context.Context) error {
 	return s.PatchObject(ctx)
 }
+
+// AddAccessKeyRefFinalizer adds a finalizer to the linodeobjectstoragekey referenced in spec.AccessKeyRef.
+func (s *ObjectStorageBucketScope) AddAccessKeyRefFinalizer(ctx context.Context, finalizer string) error {
+	obj, err := s.getAccessKey(ctx)
+	if err != nil {
+		return err
+	}
+
+	controllerutil.AddFinalizer(obj, finalizer)
+	if err := s.Client.Update(ctx, obj); err != nil {
+		return fmt.Errorf("add linodeobjectstoragekey finalizer %s/%s: %w", s.Bucket.Spec.AccessKeyRef.Namespace, s.Bucket.Spec.AccessKeyRef.Name, err)
+	}
+
+	return nil
+}
+
+// RemoveAccessKeyRefFinalizer removes a finalizer from the linodeobjectstoragekey referenced in spec.AccessKeyRef.
+func (s *ObjectStorageBucketScope) RemoveAccessKeyRefFinalizer(ctx context.Context, finalizer string) error {
+	obj, err := s.getAccessKey(ctx)
+	if err != nil {
+		return err
+	}
+
+	controllerutil.RemoveFinalizer(obj, finalizer)
+	if err := s.Client.Update(ctx, obj); err != nil {
+		return fmt.Errorf("remove linodeobjectstoragekey finalizer %s/%s: %w", s.Bucket.Spec.AccessKeyRef.Namespace, s.Bucket.Spec.AccessKeyRef.Name, err)
+	}
+
+	return nil
+}
+
+func (s *ObjectStorageBucketScope) getAccessKey(ctx context.Context) (*infrav1alpha2.LinodeObjectStorageKey, error) {
+	if s.Bucket.Spec.AccessKeyRef == nil {
+		return nil, fmt.Errorf("accessKeyRef is nil for bucket %s", s.Bucket.Name)
+	}
+
+	objKey := client.ObjectKey{
+		Name:      s.Bucket.Spec.AccessKeyRef.Name,
+		Namespace: s.Bucket.Spec.AccessKeyRef.Namespace,
+	}
+	if s.Bucket.Spec.AccessKeyRef.Namespace == "" {
+		s.Bucket.Spec.AccessKeyRef.Namespace = s.Bucket.GetNamespace()
+	}
+
+	objStorageKey := &infrav1alpha2.LinodeObjectStorageKey{}
+	if err := s.Client.Get(ctx, objKey, objStorageKey); err != nil {
+		return nil, fmt.Errorf("get linodeobjectstoragekey %s: %w", s.Bucket.Spec.AccessKeyRef.Name, err)
+	}
+
+	return objStorageKey, nil
+}

cloud/scope/object_storage_bucket.go

@@ -5,12 +5,20 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/aws/aws-sdk-go-v2/aws"
+	awsconfig "github.com/aws/aws-sdk-go-v2/config"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 	"github.com/linode/linodego"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
 
+	"github.com/linode/cluster-api-provider-linode/clients"
 	"github.com/linode/cluster-api-provider-linode/cloud/scope"
 	"github.com/linode/cluster-api-provider-linode/util"
 )
 
+// EnsureAndUpdateObjectStorageBucket ensures that the bucket exists and updates its access options if necessary.
 func EnsureAndUpdateObjectStorageBucket(ctx context.Context, bScope *scope.ObjectStorageBucketScope) (*linodego.ObjectStorageBucket, error) {
 	bucket, err := bScope.LinodeClient.GetObjectStorageBucket(
 		ctx,
@@ -62,3 +70,50 @@ func EnsureAndUpdateObjectStorageBucket(ctx context.Context, bScope *scope.Objec
 
 	return bucket, nil
 }
+
+// DeleteBucket deletes the bucket and all its objects.
+func DeleteBucket(ctx context.Context, bScope *scope.ObjectStorageBucketScope) error {
+	s3Client, err := createS3ClientWithAccessKey(ctx, bScope.Client, bScope.Bucket.Spec.AccessKeyRef, bScope.Bucket.Namespace)
+	if err != nil {
+		return fmt.Errorf("failed to create S3 client: %w", err)
+	}
+	if err := PurgeAllObjects(ctx, bScope.Bucket.Name, s3Client, true, true); err != nil {
+		return fmt.Errorf("failed to purge all objects: %w", err)
+	}
+
+	return nil
+}
+
+// createS3ClientWithAccessKey creates a connection to s3 given k8s client and an access key reference.
+func createS3ClientWithAccessKey(ctx context.Context, crClient clients.K8sClient, accessKeyRef *corev1.ObjectReference, defaultNamespace string) (*s3.Client, error) {
+	if accessKeyRef == nil {
+		return nil, fmt.Errorf("accessKeyRef is nil")
+	}
+	objSecret := &corev1.Secret{}
+	if accessKeyRef.Namespace == "" {
+		accessKeyRef.Namespace = defaultNamespace
+	}
+	if err := crClient.Get(ctx, types.NamespacedName{Name: accessKeyRef.Name + "-obj-key", Namespace: accessKeyRef.Namespace}, objSecret); err != nil {
+		return nil, fmt.Errorf("failed to get bucket secret: %w", err)
+	}
+	accessKey := string(objSecret.Data["access"])
+	secretKey := string(objSecret.Data["secret"])
+	endpoint := string(objSecret.Data["endpoint"])
+	awsConfig, err := awsconfig.LoadDefaultConfig(
+		ctx,
+		awsconfig.WithCredentialsProvider(
+			credentials.NewStaticCredentialsProvider(accessKey, secretKey, ""),
+		),
+		awsconfig.WithRegion("auto"),
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create aws config: %w", err)
+	}
+
+	s3Client := s3.NewFromConfig(awsConfig, func(opts *s3.Options) {
+		opts.BaseEndpoint = aws.String(endpoint)
+		opts.DisableLogOutputChecksumValidationSkipped = true
+	})
+
+	return s3Client, nil
+}

cloud/services/object_storage_buckets.go

@@ -9,7 +9,7 @@ import (
 	"github.com/aws/aws-sdk-go-v2/aws"
 	s3manager "github.com/aws/aws-sdk-go-v2/feature/s3/manager"
 	"github.com/aws/aws-sdk-go-v2/service/s3"
-	"github.com/aws/aws-sdk-go-v2/service/s3/types"
+	s3types "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	"github.com/aws/smithy-go"
 
 	"github.com/linode/cluster-api-provider-linode/cloud/scope"
@@ -104,9 +104,9 @@ func DeleteObject(ctx context.Context, mscope *scope.MachineScope) error {
 	if err != nil {
 		var (
 			ae  smithy.APIError
-			bne *types.NoSuchBucket
-			kne *types.NoSuchKey
-			nf  *types.NotFound
+			bne *s3types.NoSuchBucket
+			kne *s3types.NoSuchKey
+			nf  *s3types.NotFound
 		)
 		switch {
 		// In the case that the IAM policy does not have sufficient permissions to get the object, we will attempt to
@@ -137,3 +137,141 @@ func DeleteObject(ctx context.Context, mscope *scope.MachineScope) error {
 
 	return nil
 }
+
+// PurgeAllObjects wipes out all versions and delete markers for versioned objects.
+func PurgeAllObjects(
+	ctx context.Context,
+	bucket string,
+	s3client *s3.Client,
+	bypassRetention,
+	ignoreNotFound bool,
+) error {
+	versioning, err := s3client.GetBucketVersioning(ctx, &s3.GetBucketVersioningInput{
+		Bucket: aws.String(bucket),
+	})
+	if err != nil {
+		return err
+	}
+
+	if versioning.Status == s3types.BucketVersioningStatusEnabled {
+		err = DeleteAllObjectVersionsAndDeleteMarkers(
+			ctx,
+			s3client,
+			bucket,
+			"",
+			bypassRetention,
+			ignoreNotFound,
+		)
+	} else {
+		err = DeleteAllObjects(ctx, s3client, bucket, bypassRetention)
+	}
+	return err
+}
+
+// DeleteAllObjects sends delete requests for every object.
+// Versioned objects will get a deletion marker instead of being fully purged.
+func DeleteAllObjects(
+	ctx context.Context,
+	s3client *s3.Client,
+	bucketName string,
+	bypassRetention bool,
+) error {
+	objPaginator := s3.NewListObjectsV2Paginator(s3client, &s3.ListObjectsV2Input{
+		Bucket: aws.String(bucketName),
+	})
+
+	var objectsToDelete []s3types.ObjectIdentifier
+	for objPaginator.HasMorePages() {
+		page, err := objPaginator.NextPage(ctx)
+		if err != nil {
+			return err
+		}
+
+		for _, obj := range page.Contents {
+			objectsToDelete = append(objectsToDelete, s3types.ObjectIdentifier{
+				Key: obj.Key,
+			})
+		}
+	}
+
+	if len(objectsToDelete) == 0 {
+		return nil
+	}
+
+	_, err := s3client.DeleteObjects(ctx, &s3.DeleteObjectsInput{
+		Bucket:                    aws.String(bucketName),
+		Delete:                    &s3types.Delete{Objects: objectsToDelete},
+		BypassGovernanceRetention: &bypassRetention,
+	})
+
+	return err
+}
+
+// DeleteAllObjectVersionsAndDeleteMarkers deletes all versions of a given object
+func DeleteAllObjectVersionsAndDeleteMarkers(ctx context.Context, client *s3.Client, bucket, prefix string, bypassRetention, ignoreNotFound bool) error {
+	paginator := s3.NewListObjectVersionsPaginator(client, &s3.ListObjectVersionsInput{
+		Bucket: aws.String(bucket),
+		Prefix: aws.String(prefix),
+	})
+
+	var objectsToDelete []s3types.ObjectIdentifier
+	for paginator.HasMorePages() {
+		page, err := paginator.NextPage(ctx)
+		if page == nil {
+			continue
+		}
+		if err != nil {
+			if !IsObjNotFoundErr(err) || !ignoreNotFound {
+				return err
+			}
+		}
+
+		for _, version := range page.Versions {
+			objectsToDelete = append(
+				objectsToDelete,
+				s3types.ObjectIdentifier{
+					Key:       version.Key,
+					VersionId: version.VersionId,
+				},
+			)
+		}
+		for _, marker := range page.DeleteMarkers {
+			objectsToDelete = append(
+				objectsToDelete,
+				s3types.ObjectIdentifier{
+					Key:       marker.Key,
+					VersionId: marker.VersionId,
+				},
+			)
+		}
+	}
+
+	if len(objectsToDelete) == 0 {
+		return nil
+	}
+
+	_, err := client.DeleteObjects(
+		ctx,
+		&s3.DeleteObjectsInput{
+			Bucket:                    aws.String(bucket),
+			Delete:                    &s3types.Delete{Objects: objectsToDelete},
+			BypassGovernanceRetention: &bypassRetention,
+		},
+	)
+	if err != nil {
+		if !IsObjNotFoundErr(err) || !ignoreNotFound {
+			return err
+		}
+	}
+	return nil
+}
+
+// IsObjNotFoundErr checks if the error is a NotFound or Forbidden error from the S3 API.
+func IsObjNotFoundErr(err error) bool {
+	var apiErr smithy.APIError
+	// Error code is 'Forbidden' when the bucket has been removed
+	if errors.As(err, &apiErr) {
+		return apiErr.ErrorCode() == "NotFound" || apiErr.ErrorCode() == "Forbidden"
+	}
+	return false
+}