Upgrade aws-sdk-go-v2 (sourcegraph#19155)

This PR:
- Migrates to the AWS v2 API in the codeintel s3 adapter
- Upgrades the v2 package to the newest version

Author: eseliger

cmd/server/Dockerfile

@@ -54,7 +54,7 @@ COPY --from=comby/comby:0.18.4@sha256:b47ce282778bfea7f80d45f5ef0cc546ba0d6347ba
 COPY --from=sourcegraph/syntect_server:331beda@sha256:6b8950b41993af3d10300b5a160fab1c06c337cb4614978f4fdb76e2588afbfe /syntect_server /usr/local/bin/
 
 # install minio (keep this up to date with docker-images/minio/Dockerfile)
-ENV MINIO_VERSION=RELEASE.2020-10-18T21-54-12Z
+ENV MINIO_VERSION=RELEASE.2021-04-06T23-11-00Z
 RUN wget "https://dl.min.io/server/minio/release/linux-amd64/archive/minio.$MINIO_VERSION" && \
     chmod +x "minio.$MINIO_VERSION" && \
     mv "minio.$MINIO_VERSION" /usr/local/bin/minio

dev/go-mod-update.sh

@@ -14,9 +14,6 @@ go get -u all
 ## names. This is causing our use of our k8s client to panic.
 go get github.com/golang/[email protected]
 
-## Newer versions removed some types in the endpoint package we relied on. Unsure how to fix yet, so punting.
-go get github.com/aws/[email protected]
-
 # Cleanup and validate everything still works
 go mod tidy
 go test -short -failfast ./... >/dev/null

doc/dependency_decisions.yml

@@ -346,3 +346,10 @@
     :why: Internal module
     :versions: []
     :when: 2021-03-09 19:42:12.214862934 Z
+- - :license
+  - github.com/aws/smithy-go
+  - Apache 2.0
+  - :who:
+    :why: Inference broken, LICENSE file lives at https://github.com/aws/smithy-go/blob/main/LICENSE
+    :versions: []
+    :when: 2021-04-14 12:51:19.503802000 Z

docker-images/minio/build.sh

@@ -3,7 +3,7 @@
 set -ex
 cd "$(dirname "${BASH_SOURCE[0]}")"
 
-# Retag the 2020-10-18 MinIO release
-MINIO_RELEASE="RELEASE.2020-10-18T21-54-12Z"
+# Retag the 2021-04-06 MinIO release
+MINIO_RELEASE="RELEASE.2021-04-06T23-11-00Z"
 docker pull minio/minio:$MINIO_RELEASE
 docker tag minio/minio:$MINIO_RELEASE "$IMAGE"

enterprise/cmd/precise-code-intel-worker/main.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/aws/aws-sdk-go/aws/awserr"
+	smithyhttp "github.com/aws/smithy-go/transport/http"
 	"github.com/inconshreveable/log15"
 	"github.com/opentracing/opentracing-go"
 	"github.com/prometheus/client_golang/prometheus"
@@ -199,15 +199,6 @@ func initializeUploadStore(ctx context.Context, uploadStore uploadstore.Store) e
 }
 
 func isRequestError(err error) bool {
-	for err != nil {
-		if e, ok := err.(awserr.Error); ok {
-			if e.Code() == "RequestError" {
-				return true
-			}
-		}
-
-		err = errors.Unwrap(err)
-	}
-
-	return false
+	var rse *smithyhttp.RequestSendError
+	return errors.As(err, &rse)
 }

enterprise/internal/codeintel/stores/uploadstore/config_test.go

@@ -1,8 +1,13 @@
 package uploadstore
 
 import (
+	"context"
 	"testing"
 	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
+	"github.com/google/go-cmp/cmp"
 )
 
 func TestConfigDefaults(t *testing.T) {
@@ -70,25 +75,44 @@ func TestConfigS3(t *testing.T) {
 	}
 }
 
-func TestConfigMinIOSessionOptions(t *testing.T) {
+func TestS3ClientOptions(t *testing.T) {
 	config := Config{}
 	config.SetMockGetter(mapGetter(nil))
 	config.Load()
 
-	options := s3SessionOptions("minio", config.S3)
+	// minIO
+	{
+		options := &s3.Options{}
+		s3ClientOptions("minio", config.S3)(options)
 
-	if value := *options.Config.Region; value != "us-east-1" {
-		t.Errorf("unexpected region. want=%s have=%s", "us-east-1", value)
-	}
-	if value := *options.Config.Endpoint; value != "http://minio:9000" {
-		t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", value)
+		if options.EndpointResolver == nil {
+			t.Fatalf("unexpected endpoint option")
+		}
+		endpoint, err := options.EndpointResolver.ResolveEndpoint("us-east-2", s3.EndpointResolverOptions{})
+		if err != nil {
+			t.Fatal(err)
+		}
+		if endpoint.URL != "http://minio:9000" {
+			t.Errorf("unexpected endpoint. want=%s have=%s", "http://minio:9000", endpoint.URL)
+		}
+
+		if !options.UsePathStyle {
+			t.Errorf("invalid UsePathStyle setting for S3Options")
+		}
 	}
-	if options.Config.S3ForcePathStyle == nil || !*options.Config.S3ForcePathStyle {
-		t.Errorf("expected path style option")
+
+	// S3
+	{
+		options := &s3.Options{}
+		s3ClientOptions("s3", config.S3)(options)
+
+		if diff := cmp.Diff(&s3.Options{}, options); diff != "" {
+			t.Fatalf("invalid s3 options returned for S3: %s", diff)
+		}
 	}
 }
 
-func TestConfigS3SessionOptions(t *testing.T) {
+func TestS3ClientConfig(t *testing.T) {
 	env := map[string]string{
 		"PRECISE_CODE_INTEL_UPLOAD_BACKEND":               "S3",
 		"PRECISE_CODE_INTEL_UPLOAD_BUCKET":                "lsif-uploads",
@@ -104,16 +128,28 @@ func TestConfigS3SessionOptions(t *testing.T) {
 	config.SetMockGetter(mapGetter(env))
 	config.Load()
 
-	options := s3SessionOptions("s3", config.S3)
+	cfg, err := s3ClientConfig(context.Background(), config.S3)
+	if err != nil {
+		t.Fatal(err)
+	}
 
-	if value := *options.Config.Region; value != "us-east-2" {
+	if value := cfg.Region; value != "us-east-2" {
 		t.Errorf("unexpected region. want=%s have=%s", "us-east-2", value)
 	}
-	if options.Config.Endpoint != nil {
-		t.Errorf("unexpected endpoint option")
+	cred, err := cfg.Credentials.Retrieve(context.Background())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if diff := cmp.Diff(aws.Credentials{
+		AccessKeyID:     config.S3.AccessKeyID,
+		SecretAccessKey: config.S3.SecretAccessKey,
+		SessionToken:    config.S3.SessionToken,
+		Source:          "StaticCredentials",
+	}, cred); diff != "" {
+		t.Errorf("invalid credential returned: %s", diff)
 	}
-	if options.Config.S3ForcePathStyle != nil {
-		t.Errorf("unexpected path style option")
+	if cfg.EndpointResolver != nil {
+		t.Errorf("unexpected endpoint option")
 	}
 }
 

enterprise/internal/codeintel/stores/uploadstore/mock_s3_api_test.go

@@ -3,8 +3,8 @@ package uploadstore
 import (
 	"context"
 
-	"github.com/aws/aws-sdk-go/service/s3"
-	"github.com/aws/aws-sdk-go/service/s3/s3manager"
+	"github.com/aws/aws-sdk-go-v2/feature/s3/manager"
+	"github.com/aws/aws-sdk-go-v2/service/s3"
 )
 
 type s3API interface {
@@ -20,52 +20,52 @@ type s3API interface {
 }
 
 type s3Uploader interface {
-	Upload(ctx context.Context, input *s3manager.UploadInput) error
+	Upload(ctx context.Context, input *s3.PutObjectInput) error
 }
 
-type s3APIShim struct{ *s3.S3 }
-type s3UploaderShim struct{ *s3manager.Uploader }
+type s3APIShim struct{ *s3.Client }
+type s3UploaderShim struct{ *manager.Uploader }
 
 var _ s3API = &s3APIShim{}
 var _ s3Uploader = &s3UploaderShim{}
 
 func (s *s3APIShim) CreateBucket(ctx context.Context, input *s3.CreateBucketInput) (*s3.CreateBucketOutput, error) {
-	return s.S3.CreateBucketWithContext(ctx, input)
+	return s.Client.CreateBucket(ctx, input)
 }
 
 func (s *s3APIShim) PutBucketLifecycleConfiguration(ctx context.Context, input *s3.PutBucketLifecycleConfigurationInput) (*s3.PutBucketLifecycleConfigurationOutput, error) {
-	return s.S3.PutBucketLifecycleConfigurationWithContext(ctx, input)
+	return s.Client.PutBucketLifecycleConfiguration(ctx, input)
 }
 
 func (s *s3APIShim) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) {
-	return s.S3.HeadObjectWithContext(ctx, input)
+	return s.Client.HeadObject(ctx, input)
 }
 
 func (s *s3APIShim) GetObject(ctx context.Context, input *s3.GetObjectInput) (*s3.GetObjectOutput, error) {
-	return s.S3.GetObjectWithContext(ctx, input)
+	return s.Client.GetObject(ctx, input)
 }
 
 func (s *s3APIShim) DeleteObject(ctx context.Context, input *s3.DeleteObjectInput) (*s3.DeleteObjectOutput, error) {
-	return s.S3.DeleteObjectWithContext(ctx, input)
+	return s.Client.DeleteObject(ctx, input)
 }
 
 func (s *s3APIShim) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) {
-	return s.S3.CreateMultipartUploadWithContext(ctx, input)
+	return s.Client.CreateMultipartUpload(ctx, input)
 }
 
 func (s *s3APIShim) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) (*s3.AbortMultipartUploadOutput, error) {
-	return s.S3.AbortMultipartUploadWithContext(ctx, input)
+	return s.Client.AbortMultipartUpload(ctx, input)
 }
 
 func (s *s3APIShim) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyInput) (*s3.UploadPartCopyOutput, error) {
-	return s.S3.UploadPartCopyWithContext(ctx, input)
+	return s.Client.UploadPartCopy(ctx, input)
 }
 
 func (s *s3APIShim) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) {
-	return s.S3.CompleteMultipartUploadWithContext(ctx, input)
+	return s.Client.CompleteMultipartUpload(ctx, input)
 }
 
-func (s *s3UploaderShim) Upload(ctx context.Context, input *s3manager.UploadInput) error {
-	_, err := s.Uploader.UploadWithContext(ctx, input)
+func (s *s3UploaderShim) Upload(ctx context.Context, input *s3.PutObjectInput) error {
+	_, err := s.Uploader.Upload(ctx, input)
 	return err
 }