Skip to content

Commit bcc0ff1

Browse files
hjgracahjgraca
authored
Merge pull request #337 from lachriz-aws/feature/batch-processing
2 parents 13b5570 + 3736f2d commit bcc0ff1

File tree

92 files changed

+8035
-7
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

92 files changed

+8035
-7
lines changed

‎docs/snippets/batch/templates/dynamodb.yaml

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: AWS::Serverless-2016-10-31
3+
Description: partial batch response sample
4+
5+
Globals:
6+
Function:
7+
Timeout: 5
8+
MemorySize: 256
9+
Runtime: nodejs18.x
10+
Tracing: Active
11+
Environment:
12+
Variables:
13+
POWERTOOLS_SERVICE_NAME: powertools-dotnet-sample-batch-processing
14+
POWERTOOLS_LOG_LEVEL: Debug
15+
POWERTOOLS_LOGGER_CASE: PascalCase # Allowed values are: CamelCase, PascalCase and SnakeCase
16+
POWERTOOLS_BATCH_ERROR_HANDLING_POLICY: DeriveFromEvent
17+
POWERTOOLS_BATCH_MAX_DEGREE_OF_PARALLELISM: 1
18+
POWERTOOLS_BATCH_PARALLEL_ENABLED: false
19+
20+
Resources:
21+
HelloWorldFunction:
22+
Type: AWS::Serverless::Function
23+
Properties:
24+
CodeUri: ./src/HelloWorld/
25+
Handler: HelloWorld::HelloWorld.Function::DynamoDbStreamHandlerUsingAttribute
26+
Policies:
27+
# Lambda Destinations require additional permissions
28+
# to send failure records from Kinesis/DynamoDB
29+
- Version: '2012-10-17'
30+
Statement:
31+
Effect: 'Allow'
32+
Action:
33+
- sqs:GetQueueAttributes
34+
- sqs:GetQueueUrl
35+
- sqs:SendMessage
36+
Resource: !GetAtt SampleDLQ.Arn
37+
- KMSDecryptPolicy:
38+
KeyId: !Ref CustomerKey
39+
Events:
40+
DynamoDBStream:
41+
Type: DynamoDB
42+
Properties:
43+
Stream: !GetAtt SampleTable.StreamArn
44+
StartingPosition: LATEST
45+
MaximumRetryAttempts: 2
46+
DestinationConfig:
47+
OnFailure:
48+
Destination: !GetAtt SampleDLQ.Arn
49+
FunctionResponseTypes:
50+
- ReportBatchItemFailures
51+
52+
SampleDLQ:
53+
Type: AWS::SQS::Queue
54+
Properties:
55+
KmsMasterKeyId: !Ref CustomerKey
56+
57+
SampleTable:
58+
Type: AWS::DynamoDB::Table
59+
Properties:
60+
BillingMode: PAY_PER_REQUEST
61+
AttributeDefinitions:
62+
- AttributeName: pk
63+
AttributeType: S
64+
- AttributeName: sk
65+
AttributeType: S
66+
KeySchema:
67+
- AttributeName: pk
68+
KeyType: HASH
69+
- AttributeName: sk
70+
KeyType: RANGE
71+
SSESpecification:
72+
SSEEnabled: true
73+
StreamSpecification:
74+
StreamViewType: NEW_AND_OLD_IMAGES
75+
76+
# --------------
77+
# KMS key for encrypted queues
78+
CustomerKey:
79+
Type: AWS::KMS::Key
80+
Properties:
81+
Description: KMS key for encrypted queues
82+
Enabled: true
83+
KeyPolicy:
84+
Version: '2012-10-17'
85+
Statement:
86+
- Sid: Enable IAM User Permissions
87+
Effect: Allow
88+
Principal:
89+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
90+
Action: 'kms:*'
91+
Resource: '*'
92+
- Sid: Allow use of the key
93+
Effect: Allow
94+
Principal:
95+
Service: lambda.amazonaws.com
96+
Action:
97+
- kms:Decrypt
98+
- kms:GenerateDataKey
99+
Resource: '*'
100+
101+
CustomerKeyAlias:
102+
Type: AWS::KMS::Alias
103+
Properties:
104+
AliasName: alias/powertools-batch-sqs-demo
105+
TargetKeyId: !Ref CustomerKey

‎docs/snippets/batch/templates/kinesis.yaml

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: AWS::Serverless-2016-10-31
3+
Description: partial batch response sample
4+
5+
Globals:
6+
Function:
7+
Timeout: 5
8+
MemorySize: 256
9+
Runtime: nodejs18.x
10+
Tracing: Active
11+
Environment:
12+
Variables:
13+
POWERTOOLS_SERVICE_NAME: powertools-dotnet-sample-batch-processing
14+
POWERTOOLS_LOG_LEVEL: Debug
15+
POWERTOOLS_LOGGER_CASE: PascalCase # Allowed values are: CamelCase, PascalCase and SnakeCase
16+
POWERTOOLS_BATCH_ERROR_HANDLING_POLICY: DeriveFromEvent
17+
POWERTOOLS_BATCH_MAX_DEGREE_OF_PARALLELISM: 1
18+
POWERTOOLS_BATCH_PARALLEL_ENABLED: false
19+
20+
Resources:
21+
HelloWorldFunction:
22+
Type: AWS::Serverless::Function
23+
Properties:
24+
CodeUri: ./src/HelloWorld/
25+
Handler: HelloWorld::HelloWorld.Function::KinesisEventHandlerUsingAttribute
26+
Policies:
27+
# Lambda Destinations require additional permissions
28+
# to send failure records to DLQ from Kinesis/DynamoDB
29+
- Version: '2012-10-17'
30+
Statement:
31+
Effect: 'Allow'
32+
Action:
33+
- sqs:GetQueueAttributes
34+
- sqs:GetQueueUrl
35+
- sqs:SendMessage
36+
Resource: !GetAtt SampleDLQ.Arn
37+
- KMSDecryptPolicy:
38+
KeyId: !Ref CustomerKey
39+
Events:
40+
KinesisStream:
41+
Type: Kinesis
42+
Properties:
43+
Stream: !GetAtt SampleStream.Arn
44+
BatchSize: 100
45+
StartingPosition: LATEST
46+
MaximumRetryAttempts: 2
47+
DestinationConfig:
48+
OnFailure:
49+
Destination: !GetAtt SampleDLQ.Arn
50+
FunctionResponseTypes:
51+
- ReportBatchItemFailures
52+
53+
SampleDLQ:
54+
Type: AWS::SQS::Queue
55+
Properties:
56+
KmsMasterKeyId: !Ref CustomerKey
57+
58+
SampleStream:
59+
Type: AWS::Kinesis::Stream
60+
Properties:
61+
ShardCount: 1
62+
StreamEncryption:
63+
EncryptionType: KMS
64+
KeyId: alias/aws/kinesis
65+
66+
# --------------
67+
# KMS key for encrypted queues
68+
CustomerKey:
69+
Type: AWS::KMS::Key
70+
Properties:
71+
Description: KMS key for encrypted queues
72+
Enabled: true
73+
KeyPolicy:
74+
Version: '2012-10-17'
75+
Statement:
76+
- Sid: Enable IAM User Permissions
77+
Effect: Allow
78+
Principal:
79+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
80+
Action: 'kms:*'
81+
Resource: '*'
82+
- Sid: Allow use of the key
83+
Effect: Allow
84+
Principal:
85+
Service: lambda.amazonaws.com
86+
Action:
87+
- kms:Decrypt
88+
- kms:GenerateDataKey
89+
Resource: '*'
90+
91+
CustomerKeyAlias:
92+
Type: AWS::KMS::Alias
93+
Properties:
94+
AliasName: alias/powertools-batch-sqs-demo
95+
TargetKeyId: !Ref CustomerKey

‎docs/snippets/batch/templates/sqs.yaml

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
AWSTemplateFormatVersion: '2010-09-09'
2+
Transform: AWS::Serverless-2016-10-31
3+
Description: partial batch response sample
4+
5+
Globals:
6+
Function:
7+
Timeout: 5
8+
MemorySize: 256
9+
Runtime: nodejs18.x
10+
Tracing: Active
11+
Environment:
12+
Variables:
13+
POWERTOOLS_SERVICE_NAME: powertools-dotnet-sample-batch-processing
14+
POWERTOOLS_LOG_LEVEL: Debug
15+
POWERTOOLS_LOGGER_CASE: PascalCase # Allowed values are: CamelCase, PascalCase and SnakeCase
16+
POWERTOOLS_BATCH_ERROR_HANDLING_POLICY: DeriveFromEvent
17+
POWERTOOLS_BATCH_MAX_DEGREE_OF_PARALLELISM: 1
18+
POWERTOOLS_BATCH_PARALLEL_ENABLED: false
19+
20+
Resources:
21+
HelloWorldFunction:
22+
Type: AWS::Serverless::Function
23+
Properties:
24+
CodeUri: ./src/HelloWorld/
25+
Handler: HelloWorld::HelloWorld.Function::SqsHandlerUsingAttribute
26+
Policies:
27+
- SQSPollerPolicy:
28+
QueueName: !GetAtt SampleQueue.QueueName
29+
- KMSDecryptPolicy:
30+
KeyId: !Ref CustomerKey
31+
Events:
32+
Batch:
33+
Type: SQS
34+
Properties:
35+
Queue: !GetAtt SampleQueue.Arn
36+
FunctionResponseTypes:
37+
- ReportBatchItemFailures
38+
39+
SampleDLQ:
40+
Type: AWS::SQS::Queue
41+
Properties:
42+
KmsMasterKeyId: !Ref CustomerKey
43+
44+
SampleQueue:
45+
Type: AWS::SQS::Queue
46+
Properties:
47+
VisibilityTimeout: 30 # Fn timeout * 6
48+
SqsManagedSseEnabled: true
49+
RedrivePolicy:
50+
maxReceiveCount: 2
51+
deadLetterTargetArn: !GetAtt SampleDLQ.Arn
52+
KmsMasterKeyId: !Ref CustomerKey
53+
54+
# --------------
55+
# KMS key for encrypted queues
56+
CustomerKey:
57+
Type: AWS::KMS::Key
58+
Properties:
59+
Description: KMS key for encrypted queues
60+
Enabled: true
61+
KeyPolicy:
62+
Version: '2012-10-17'
63+
Statement:
64+
- Sid: Enable IAM User Permissions
65+
Effect: Allow
66+
Principal:
67+
AWS: !Sub 'arn:aws:iam::${AWS::AccountId}:root'
68+
Action: 'kms:*'
69+
Resource: '*'
70+
- Sid: Allow use of the key
71+
Effect: Allow
72+
Principal:
73+
Service: lambda.amazonaws.com
74+
Action:
75+
- kms:Decrypt
76+
- kms:GenerateDataKey
77+
Resource: '*'
78+
79+
CustomerKeyAlias:
80+
Type: AWS::KMS::Alias
81+
Properties:
82+
AliasName: alias/powertools-batch-sqs-demo
83+
TargetKeyId: !Ref CustomerKey

0 commit comments

Comments
 (0)