├── .gitignore ├── README.md ├── app.py ├── cdk.json ├── implement_the_priority_queue_pattern_with_sqs_and_lambda ├── __init__.py ├── fifo_priority_queue.py ├── fifo_priority_queue_concurrency_three.py ├── implement_the_priority_queue_pattern_with_sqs_and_lambda_stack.py ├── lambda_function.py ├── multiple_priority_fifo_queues.py ├── multiple_priority_queues.py └── simple_priority_queue.py ├── lambda └── functions │ ├── multi_queue_processor │ └── index.py │ ├── queue_processor │ └── index.py │ └── queue_processor_multi_concurrency │ └── index.py ├── requirements.txt └── source.bat /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | package-lock.json 3 | __pycache__ 4 | .pytest_cache 5 | .venv 6 | *.egg-info 7 | 8 | # CDK asset staging directory 9 | .cdk.staging 10 | cdk.out 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Bite-Sized Serverless: Implement the Priority Queue Pattern with SQS and Lambda 2 | 3 | This project contains the infrastructure definition used in the article Implement the Priority Queue Pattern with SQS and Lambda on Bite-Sized Serverless: https://bitesizedserverless.com/bite/implement-the-priority-queue-pattern-with-sqs-and-lambda/ 4 | 5 | The compiled CloudFormation files can be found in the `cdk.out` folder. The Python files for the Lambda functions are placed in `lambda/functions`. 6 | 7 | To compile the CloudFormation templates, follow these steps: 8 | 9 | 1. First create a `virtualenv` with `python3 -m venv .venv`. 10 | 2. Then activate the `virtualenv` with `source .venv/bin/activate`. 11 | 3. Next, install the required Python packages by running `pip install -r requirements.txt` 12 | 4. Then compile CloudFormation by running `cdk synth`. The output will be stored in `cdk.out`. 13 | 14 | To deploy the templates to your AWS account, run `cdk deploy`. 15 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | # Third party imports 4 | import aws_cdk as cdk 5 | 6 | # Local application/library specific imports 7 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.implement_the_priority_queue_pattern_with_sqs_and_lambda_stack import ( 8 | ImplementThePriorityQueuePatternWithSqsAndLambdaStack, 9 | ) 10 | 11 | 12 | app = cdk.App() 13 | ImplementThePriorityQueuePatternWithSqsAndLambdaStack( 14 | scope=app, 15 | construct_id="ImplementThePriorityQueuePatternWithSqsAndLambdaStack", 16 | ) 17 | 18 | app.synth() 19 | -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "python3 app.py", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "requirements*.txt", 11 | "source.bat", 12 | "**/__init__.py", 13 | "python/__pycache__", 14 | "tests" 15 | ] 16 | }, 17 | "context": { 18 | "@aws-cdk/aws-apigateway:usagePlanKeyOrderInsensitiveId": true, 19 | "@aws-cdk/core:stackRelativeExports": true, 20 | "@aws-cdk/aws-rds:lowercaseDbIdentifier": true, 21 | "@aws-cdk/aws-lambda:recognizeVersionProps": true, 22 | "@aws-cdk/aws-cloudfront:defaultSecurityPolicyTLSv1.2_2021": true, 23 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 24 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 25 | "@aws-cdk/aws-iam:minimizePolicies": true, 26 | "@aws-cdk/core:target-partitions": [ 27 | "aws", 28 | "aws-cn" 29 | ] 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/donkersgoed/implement-the-priority-queue-pattern-with-sqs-and-lambda/9b07d81d802df97f418baf6a58a83b0fec75eba3/implement_the_priority_queue_pattern_with_sqs_and_lambda/__init__.py -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/fifo_priority_queue.py: -------------------------------------------------------------------------------- 1 | # Third party imports 2 | from aws_cdk import ( 3 | CfnOutput, 4 | Duration, 5 | RemovalPolicy, 6 | aws_sqs as sqs, 7 | aws_lambda as lambda_, 8 | ) 9 | from constructs import Construct 10 | 11 | # Local application/library specific imports 12 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.lambda_function import ( 13 | LambdaFunction, 14 | ) 15 | 16 | 17 | class FifoPriorityQueue(Construct): 18 | def __init__( 19 | self, 20 | scope: Construct, 21 | construct_id: str, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__(scope, construct_id, **kwargs) 25 | 26 | main_queue = sqs.Queue( 27 | scope=self, 28 | id="MainQueue", 29 | removal_policy=RemovalPolicy.DESTROY, 30 | fifo=True, 31 | content_based_deduplication=True, 32 | ) 33 | CfnOutput(scope=self, id="FifoMainQueueUrl", value=main_queue.queue_url) 34 | 35 | priority_queue = sqs.Queue( 36 | scope=self, 37 | id="PriorityQueue", 38 | removal_policy=RemovalPolicy.DESTROY, 39 | fifo=True, 40 | content_based_deduplication=True, 41 | ) 42 | CfnOutput(scope=self, id="FifoPriorityQueueUrl", value=priority_queue.queue_url) 43 | 44 | processor_function = LambdaFunction( 45 | scope=self, 46 | construct_id="ProcessorFunction", 47 | code=lambda_.Code.from_asset("lambda/functions/queue_processor"), 48 | reserved_concurrent_executions=1, 49 | environment={ 50 | "PRIORITY_QUEUE_URL": priority_queue.queue_url, 51 | "PRIORITY_QUEUE_ARN": priority_queue.queue_arn, 52 | }, 53 | timeout=Duration.seconds(10), 54 | ) 55 | main_queue.grant_consume_messages(processor_function.function) 56 | priority_queue.grant_consume_messages(processor_function.function) 57 | 58 | lambda_.EventSourceMapping( 59 | scope=self, 60 | id="MainQueueEventSourceMapping", 61 | target=processor_function.function, 62 | event_source_arn=main_queue.queue_arn, 63 | batch_size=1, 64 | report_batch_item_failures=True, 65 | ) 66 | 67 | lambda_.EventSourceMapping( 68 | scope=self, 69 | id="PriorityQueueEventSourceMapping", 70 | target=processor_function.function, 71 | event_source_arn=priority_queue.queue_arn, 72 | batch_size=1, 73 | report_batch_item_failures=True, 74 | ) 75 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/fifo_priority_queue_concurrency_three.py: -------------------------------------------------------------------------------- 1 | # Third party imports 2 | from aws_cdk import ( 3 | CfnOutput, 4 | Duration, 5 | RemovalPolicy, 6 | aws_sqs as sqs, 7 | aws_lambda as lambda_, 8 | ) 9 | from constructs import Construct 10 | 11 | # Local application/library specific imports 12 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.lambda_function import ( 13 | LambdaFunction, 14 | ) 15 | 16 | 17 | class FifoPriorityQueueConcurrencyThree(Construct): 18 | def __init__( 19 | self, 20 | scope: Construct, 21 | construct_id: str, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__(scope, construct_id, **kwargs) 25 | 26 | main_queue = sqs.Queue( 27 | scope=self, 28 | id="MainQueue", 29 | removal_policy=RemovalPolicy.DESTROY, 30 | fifo=True, 31 | content_based_deduplication=True, 32 | ) 33 | CfnOutput(scope=self, id="FifoMainQueueUrl", value=main_queue.queue_url) 34 | 35 | priority_queue = sqs.Queue( 36 | scope=self, 37 | id="PriorityQueue", 38 | removal_policy=RemovalPolicy.DESTROY, 39 | fifo=True, 40 | content_based_deduplication=True, 41 | ) 42 | CfnOutput(scope=self, id="FifoPriorityQueueUrl", value=priority_queue.queue_url) 43 | 44 | concurrency = 3 45 | processor_function = LambdaFunction( 46 | scope=self, 47 | construct_id="ProcessorFunction", 48 | code=lambda_.Code.from_asset( 49 | "lambda/functions/queue_processor_multi_concurrency" 50 | ), 51 | reserved_concurrent_executions=concurrency, 52 | environment={ 53 | "PRIORITY_QUEUE_URL": priority_queue.queue_url, 54 | "PRIORITY_QUEUE_ARN": priority_queue.queue_arn, 55 | "CONCURRENCY": str(concurrency), 56 | }, 57 | timeout=Duration.seconds(10), 58 | ) 59 | main_queue.grant_consume_messages(processor_function.function) 60 | priority_queue.grant_consume_messages(processor_function.function) 61 | 62 | lambda_.EventSourceMapping( 63 | scope=self, 64 | id="MainQueueEventSourceMapping", 65 | target=processor_function.function, 66 | event_source_arn=main_queue.queue_arn, 67 | batch_size=1, 68 | report_batch_item_failures=True, 69 | ) 70 | 71 | lambda_.EventSourceMapping( 72 | scope=self, 73 | id="PriorityQueueEventSourceMapping", 74 | target=processor_function.function, 75 | event_source_arn=priority_queue.queue_arn, 76 | batch_size=1, 77 | report_batch_item_failures=True, 78 | ) 79 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/implement_the_priority_queue_pattern_with_sqs_and_lambda_stack.py: -------------------------------------------------------------------------------- 1 | """Module for the main ImplementThePriorityQueuePatternWithSqsAndLambda Stack.""" 2 | 3 | # Third party imports 4 | from aws_cdk import ( 5 | Stack, 6 | ) 7 | from constructs import Construct 8 | 9 | # Local application/library specific imports 10 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.simple_priority_queue import ( 11 | SimplePriorityQueue, 12 | ) 13 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.fifo_priority_queue import ( 14 | FifoPriorityQueue, 15 | ) 16 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.multiple_priority_queues import ( 17 | MultiplePriorityQueues, 18 | ) 19 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.multiple_priority_fifo_queues import ( 20 | MultiplePriorityFifoQueues, 21 | ) 22 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.fifo_priority_queue_concurrency_three import ( 23 | FifoPriorityQueueConcurrencyThree, 24 | ) 25 | 26 | 27 | class ImplementThePriorityQueuePatternWithSqsAndLambdaStack(Stack): 28 | """The ImplementThePriorityQueuePatternWithSqsAndLambda Stack.""" 29 | 30 | def __init__(self, scope: Construct, construct_id: str, **kwargs) -> None: 31 | """Construct a new ImplementThePriorityQueuePatternWithSqsAndLambdaStack.""" 32 | super().__init__(scope, construct_id, **kwargs) 33 | 34 | SimplePriorityQueue(scope=self, construct_id="SimplePriorityQueue") 35 | FifoPriorityQueue(scope=self, construct_id="FifoPriorityQueue") 36 | MultiplePriorityQueues(scope=self, construct_id="MultiplePriorityQueues") 37 | MultiplePriorityFifoQueues( 38 | scope=self, construct_id="MultiplePriorityFifoQueues" 39 | ) 40 | FifoPriorityQueueConcurrencyThree( 41 | scope=self, construct_id="FifoPriorityQueueConcurrencyThree" 42 | ) 43 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/lambda_function.py: -------------------------------------------------------------------------------- 1 | """Module for the Lambda Function L3 Pattern.""" 2 | 3 | 4 | # Third party imports 5 | from typing import Optional 6 | from aws_cdk import ( 7 | Duration, 8 | Fn, 9 | RemovalPolicy, 10 | aws_iam as iam, 11 | aws_logs as logs, 12 | aws_lambda as lambda_, 13 | ) 14 | from constructs import Construct 15 | 16 | 17 | class LambdaFunction(Construct): 18 | """CDK Construct for a Lambda Function and its supporting resources.""" 19 | 20 | def __init__( 21 | self, 22 | scope: Construct, 23 | construct_id: str, 24 | code: lambda_.Code, 25 | environment: Optional[dict] = None, 26 | reserved_concurrent_executions: Optional[Duration] = None, 27 | timeout: Optional[Duration] = None, 28 | **kwargs, 29 | ) -> None: 30 | """Construct a new LambdaFunction.""" 31 | super().__init__(scope, construct_id, **kwargs) 32 | 33 | if not environment: 34 | environment = {} 35 | 36 | # Create a role for the Lambda Function 37 | function_role = iam.Role( 38 | scope=self, 39 | id="FunctionRole", 40 | assumed_by=iam.ServicePrincipal("lambda.amazonaws.com"), 41 | ) 42 | 43 | optional_params = {} 44 | if reserved_concurrent_executions is not None: 45 | optional_params[ 46 | "reserved_concurrent_executions" 47 | ] = reserved_concurrent_executions 48 | if timeout is not None: 49 | optional_params["timeout"] = timeout 50 | 51 | # Create the Lambda Function 52 | self.function = lambda_.Function( 53 | scope=self, 54 | id="Function", 55 | role=function_role, 56 | runtime=lambda_.Runtime.PYTHON_3_9, 57 | code=code, 58 | handler="index.event_handler", 59 | environment=environment, 60 | **optional_params, 61 | ) 62 | 63 | # Create the Lambda Function Log Group 64 | function_log_group = logs.LogGroup( 65 | scope=self, 66 | id="FunctionLogGroup", 67 | retention=logs.RetentionDays.ONE_MONTH, 68 | log_group_name=Fn.sub( 69 | "/aws/lambda/${Function}", 70 | {"Function": self.function.function_name}, 71 | ), 72 | removal_policy=RemovalPolicy.DESTROY, 73 | ) 74 | 75 | # Give Lambda permission to write log streams, but not to create log groups 76 | log_policy = iam.Policy( 77 | scope=self, 78 | id="FunctionLogPolicy", 79 | document=iam.PolicyDocument( 80 | statements=[ 81 | iam.PolicyStatement( 82 | actions=["logs:PutLogEvents", "logs:CreateLogStream"], 83 | effect=iam.Effect.ALLOW, 84 | resources=[function_log_group.log_group_arn], 85 | ), 86 | ] 87 | ), 88 | ) 89 | log_policy.attach_to_role(function_role) 90 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/multiple_priority_fifo_queues.py: -------------------------------------------------------------------------------- 1 | # Third party imports 2 | from aws_cdk import ( 3 | CfnOutput, 4 | Duration, 5 | RemovalPolicy, 6 | aws_sqs as sqs, 7 | aws_lambda as lambda_, 8 | ) 9 | from constructs import Construct 10 | 11 | # Local application/library specific imports 12 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.lambda_function import ( 13 | LambdaFunction, 14 | ) 15 | 16 | 17 | class MultiplePriorityFifoQueues(Construct): 18 | def __init__( 19 | self, 20 | scope: Construct, 21 | construct_id: str, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__(scope, construct_id, **kwargs) 25 | 26 | main_queue = sqs.Queue( 27 | scope=self, 28 | id="MainFifoQueue", 29 | removal_policy=RemovalPolicy.DESTROY, 30 | fifo=True, 31 | content_based_deduplication=True, 32 | ) 33 | CfnOutput(scope=self, id="MultiMainFifoQueueUrl", value=main_queue.queue_url) 34 | 35 | medium_priority_queue = sqs.Queue( 36 | scope=self, 37 | id="MediumPriorityFifoQueue", 38 | removal_policy=RemovalPolicy.DESTROY, 39 | fifo=True, 40 | content_based_deduplication=True, 41 | ) 42 | CfnOutput( 43 | scope=self, 44 | id="FifoMediumPriorityFifoQueueUrl", 45 | value=medium_priority_queue.queue_url, 46 | ) 47 | 48 | high_priority_queue = sqs.Queue( 49 | scope=self, 50 | id="HighPriorityFifoQueue", 51 | removal_policy=RemovalPolicy.DESTROY, 52 | fifo=True, 53 | content_based_deduplication=True, 54 | ) 55 | CfnOutput( 56 | scope=self, 57 | id="FifoHighPriorityFifoQueueUrl", 58 | value=high_priority_queue.queue_url, 59 | ) 60 | 61 | processor_function = LambdaFunction( 62 | scope=self, 63 | construct_id="ProcessorFunction", 64 | code=lambda_.Code.from_asset("lambda/functions/multi_queue_processor"), 65 | reserved_concurrent_executions=1, 66 | environment={ 67 | "MEDIUM_PRIORITY_QUEUE_URL": medium_priority_queue.queue_url, 68 | "MEDIUM_PRIORITY_QUEUE_ARN": medium_priority_queue.queue_arn, 69 | "HIGH_PRIORITY_QUEUE_URL": high_priority_queue.queue_url, 70 | "HIGH_PRIORITY_QUEUE_ARN": high_priority_queue.queue_arn, 71 | }, 72 | timeout=Duration.seconds(10), 73 | ) 74 | main_queue.grant_consume_messages(processor_function.function) 75 | medium_priority_queue.grant_consume_messages(processor_function.function) 76 | high_priority_queue.grant_consume_messages(processor_function.function) 77 | 78 | lambda_.EventSourceMapping( 79 | scope=self, 80 | id="MainQueueEventSourceMapping", 81 | target=processor_function.function, 82 | event_source_arn=main_queue.queue_arn, 83 | batch_size=1, 84 | report_batch_item_failures=True, 85 | ) 86 | 87 | lambda_.EventSourceMapping( 88 | scope=self, 89 | id="MediumPriorityQueueEventSourceMapping", 90 | target=processor_function.function, 91 | event_source_arn=medium_priority_queue.queue_arn, 92 | batch_size=1, 93 | report_batch_item_failures=True, 94 | ) 95 | 96 | lambda_.EventSourceMapping( 97 | scope=self, 98 | id="HighPriorityQueueEventSourceMapping", 99 | target=processor_function.function, 100 | event_source_arn=high_priority_queue.queue_arn, 101 | batch_size=1, 102 | report_batch_item_failures=True, 103 | ) 104 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/multiple_priority_queues.py: -------------------------------------------------------------------------------- 1 | # Third party imports 2 | from aws_cdk import ( 3 | CfnOutput, 4 | Duration, 5 | RemovalPolicy, 6 | aws_sqs as sqs, 7 | aws_lambda as lambda_, 8 | ) 9 | from constructs import Construct 10 | 11 | # Local application/library specific imports 12 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.lambda_function import ( 13 | LambdaFunction, 14 | ) 15 | 16 | 17 | class MultiplePriorityQueues(Construct): 18 | def __init__( 19 | self, 20 | scope: Construct, 21 | construct_id: str, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__(scope, construct_id, **kwargs) 25 | 26 | main_queue = sqs.Queue( 27 | scope=self, 28 | id="MainQueue", 29 | removal_policy=RemovalPolicy.DESTROY, 30 | ) 31 | CfnOutput(scope=self, id="MultiMainQueueUrl", value=main_queue.queue_url) 32 | 33 | medium_priority_queue = sqs.Queue( 34 | scope=self, 35 | id="MediumPriorityQueue", 36 | removal_policy=RemovalPolicy.DESTROY, 37 | ) 38 | CfnOutput( 39 | scope=self, 40 | id="MediumPriorityQueueUrl", 41 | value=medium_priority_queue.queue_url, 42 | ) 43 | 44 | high_priority_queue = sqs.Queue( 45 | scope=self, 46 | id="HighPriorityQueue", 47 | removal_policy=RemovalPolicy.DESTROY, 48 | ) 49 | CfnOutput( 50 | scope=self, 51 | id="HighPriorityQueueUrl", 52 | value=high_priority_queue.queue_url, 53 | ) 54 | 55 | processor_function = LambdaFunction( 56 | scope=self, 57 | construct_id="ProcessorFunction", 58 | code=lambda_.Code.from_asset("lambda/functions/multi_queue_processor"), 59 | reserved_concurrent_executions=1, 60 | environment={ 61 | "MEDIUM_PRIORITY_QUEUE_URL": medium_priority_queue.queue_url, 62 | "MEDIUM_PRIORITY_QUEUE_ARN": medium_priority_queue.queue_arn, 63 | "HIGH_PRIORITY_QUEUE_URL": high_priority_queue.queue_url, 64 | "HIGH_PRIORITY_QUEUE_ARN": high_priority_queue.queue_arn, 65 | }, 66 | timeout=Duration.seconds(10), 67 | ) 68 | main_queue.grant_consume_messages(processor_function.function) 69 | medium_priority_queue.grant_consume_messages(processor_function.function) 70 | high_priority_queue.grant_consume_messages(processor_function.function) 71 | 72 | lambda_.EventSourceMapping( 73 | scope=self, 74 | id="MainQueueEventSourceMapping", 75 | target=processor_function.function, 76 | event_source_arn=main_queue.queue_arn, 77 | max_batching_window=Duration.seconds(1), 78 | batch_size=1, 79 | report_batch_item_failures=True, 80 | ) 81 | 82 | lambda_.EventSourceMapping( 83 | scope=self, 84 | id="MediumPriorityQueueEventSourceMapping", 85 | target=processor_function.function, 86 | event_source_arn=medium_priority_queue.queue_arn, 87 | max_batching_window=Duration.seconds(1), 88 | batch_size=1, 89 | report_batch_item_failures=True, 90 | ) 91 | 92 | lambda_.EventSourceMapping( 93 | scope=self, 94 | id="HighPriorityQueueEventSourceMapping", 95 | target=processor_function.function, 96 | event_source_arn=high_priority_queue.queue_arn, 97 | max_batching_window=Duration.seconds(1), 98 | batch_size=1, 99 | report_batch_item_failures=True, 100 | ) 101 | -------------------------------------------------------------------------------- /implement_the_priority_queue_pattern_with_sqs_and_lambda/simple_priority_queue.py: -------------------------------------------------------------------------------- 1 | # Third party imports 2 | from aws_cdk import ( 3 | CfnOutput, 4 | Duration, 5 | RemovalPolicy, 6 | aws_sqs as sqs, 7 | aws_lambda as lambda_, 8 | ) 9 | from constructs import Construct 10 | 11 | # Local application/library specific imports 12 | from implement_the_priority_queue_pattern_with_sqs_and_lambda.lambda_function import ( 13 | LambdaFunction, 14 | ) 15 | 16 | 17 | class SimplePriorityQueue(Construct): 18 | def __init__( 19 | self, 20 | scope: Construct, 21 | construct_id: str, 22 | **kwargs, 23 | ) -> None: 24 | super().__init__(scope, construct_id, **kwargs) 25 | 26 | main_queue = sqs.Queue( 27 | scope=self, id="MainQueue", removal_policy=RemovalPolicy.DESTROY 28 | ) 29 | CfnOutput(scope=self, id="SimpleMainQueueUrl", value=main_queue.queue_url) 30 | 31 | priority_queue = sqs.Queue( 32 | scope=self, id="PriorityQueue", removal_policy=RemovalPolicy.DESTROY 33 | ) 34 | CfnOutput( 35 | scope=self, id="SimplePriorityQueueUrl", value=priority_queue.queue_url 36 | ) 37 | 38 | processor_function = LambdaFunction( 39 | scope=self, 40 | construct_id="ProcessorFunction", 41 | code=lambda_.Code.from_asset("lambda/functions/queue_processor"), 42 | reserved_concurrent_executions=1, 43 | environment={ 44 | "PRIORITY_QUEUE_URL": priority_queue.queue_url, 45 | "PRIORITY_QUEUE_ARN": priority_queue.queue_arn, 46 | }, 47 | timeout=Duration.seconds(10), 48 | ) 49 | main_queue.grant_consume_messages(processor_function.function) 50 | priority_queue.grant_consume_messages(processor_function.function) 51 | 52 | lambda_.EventSourceMapping( 53 | scope=self, 54 | id="MainQueueEventSourceMapping", 55 | target=processor_function.function, 56 | event_source_arn=main_queue.queue_arn, 57 | max_batching_window=Duration.seconds(1), 58 | batch_size=1, 59 | report_batch_item_failures=True, 60 | ) 61 | 62 | lambda_.EventSourceMapping( 63 | scope=self, 64 | id="PriorityQueueEventSourceMapping", 65 | target=processor_function.function, 66 | event_source_arn=priority_queue.queue_arn, 67 | max_batching_window=Duration.seconds(1), 68 | batch_size=1, 69 | report_batch_item_failures=True, 70 | ) 71 | -------------------------------------------------------------------------------- /lambda/functions/multi_queue_processor/index.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import time 4 | import boto3 5 | 6 | MEDIUM_PRIORITY_QUEUE_URL = os.environ["MEDIUM_PRIORITY_QUEUE_URL"] 7 | MEDIUM_PRIORITY_QUEUE_ARN = os.environ["MEDIUM_PRIORITY_QUEUE_ARN"] 8 | HIGH_PRIORITY_QUEUE_URL = os.environ["HIGH_PRIORITY_QUEUE_URL"] 9 | HIGH_PRIORITY_QUEUE_ARN = os.environ["HIGH_PRIORITY_QUEUE_ARN"] 10 | 11 | sqs_client = boto3.client("sqs") 12 | 13 | 14 | class PriorityMessageAvailable(Exception): 15 | pass 16 | 17 | 18 | def event_handler(event, _context): 19 | returned_messages = [] 20 | for record in event["Records"]: 21 | try: 22 | _process_record(record) 23 | except PriorityMessageAvailable: 24 | print("Yielding to priority message") 25 | returned_messages.append(record["messageId"]) 26 | except Exception as exc: 27 | print(f"Got unexpected error: {type(exc)} - {exc}") 28 | returned_messages.append(record["messageId"]) 29 | 30 | return { 31 | "batchItemFailures": [ 32 | {"itemIdentifier": msg_id} for msg_id in returned_messages 33 | ] 34 | } 35 | 36 | 37 | def _process_record(record): 38 | """Handle the record or raise an error when higher priority messages are available.""" 39 | is_high_priority_msg = record["eventSourceARN"] == HIGH_PRIORITY_QUEUE_ARN 40 | is_medium_priority_msg = record["eventSourceARN"] == MEDIUM_PRIORITY_QUEUE_ARN 41 | is_main_msg = not is_medium_priority_msg and not is_high_priority_msg 42 | if is_medium_priority_msg: 43 | # Medium Prio messages should yield to High Prio messages 44 | _check_high_priority_messages_available() 45 | if is_main_msg: 46 | # Messages on the main queue should yield to High and Medium Prio messages 47 | _check_high_priority_messages_available() 48 | _check_medium_priority_messages_available() 49 | 50 | _handle_record(record) 51 | 52 | 53 | def _check_high_priority_messages_available(): 54 | return _check_priority_messages_available(queue_url=HIGH_PRIORITY_QUEUE_URL) 55 | 56 | 57 | def _check_medium_priority_messages_available(): 58 | return _check_priority_messages_available(queue_url=MEDIUM_PRIORITY_QUEUE_URL) 59 | 60 | 61 | def _check_priority_messages_available(queue_url: str): 62 | response = sqs_client.get_queue_attributes( 63 | QueueUrl=queue_url, 64 | AttributeNames=[ 65 | "ApproximateNumberOfMessages", 66 | "ApproximateNumberOfMessagesNotVisible", 67 | ], 68 | ) 69 | for key, value in response["Attributes"].items(): 70 | # Check if any attribute is > 0, raise error if 71 | # messages are in any way available on the priority queue. 72 | if int(value) > 0: 73 | raise PriorityMessageAvailable(key) 74 | 75 | 76 | def _handle_record(record): 77 | body = record["body"] 78 | time.sleep(5) 79 | print(f"successfully processed {body}") 80 | -------------------------------------------------------------------------------- /lambda/functions/queue_processor/index.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import boto3 4 | 5 | PRIORITY_QUEUE_URL = os.environ["PRIORITY_QUEUE_URL"] 6 | PRIORITY_QUEUE_ARN = os.environ["PRIORITY_QUEUE_ARN"] 7 | 8 | sqs_client = boto3.client("sqs") 9 | 10 | 11 | class PriorityMessageAvailable(Exception): 12 | pass 13 | 14 | 15 | def event_handler(event, _context): 16 | returned_messages = [] 17 | for record in event["Records"]: 18 | try: 19 | _process_record(record) 20 | except PriorityMessageAvailable: 21 | print("Yielding to priority message") 22 | returned_messages.append(record["messageId"]) 23 | except Exception as exc: 24 | print(f"Got unexpected error: {type(exc)} - {exc}") 25 | returned_messages.append(record["messageId"]) 26 | 27 | return { 28 | "batchItemFailures": [ 29 | {"itemIdentifier": msg_id} for msg_id in returned_messages 30 | ] 31 | } 32 | 33 | 34 | def _process_record(record): 35 | """Handle the record or raise an error when higher priority messages are available.""" 36 | is_priority_msg = record["eventSourceARN"] == PRIORITY_QUEUE_ARN 37 | if not is_priority_msg: 38 | _check_priority_messages_available() 39 | 40 | _handle_record(record) 41 | 42 | 43 | def _check_priority_messages_available(): 44 | response = sqs_client.get_queue_attributes( 45 | QueueUrl=PRIORITY_QUEUE_URL, 46 | AttributeNames=[ 47 | "ApproximateNumberOfMessages", 48 | "ApproximateNumberOfMessagesNotVisible", 49 | ], 50 | ) 51 | for key, value in response["Attributes"].items(): 52 | # Check if any attribute is > 0, raise error if 53 | # messages are in any way available on the priority queue. 54 | if int(value) > 0: 55 | raise PriorityMessageAvailable(key) 56 | 57 | 58 | def _handle_record(record): 59 | body = record["body"] 60 | time.sleep(5) 61 | print(f"successfully processed {body}") 62 | -------------------------------------------------------------------------------- /lambda/functions/queue_processor_multi_concurrency/index.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import boto3 4 | 5 | PRIORITY_QUEUE_URL = os.environ["PRIORITY_QUEUE_URL"] 6 | PRIORITY_QUEUE_ARN = os.environ["PRIORITY_QUEUE_ARN"] 7 | CONCURRENCY = int(os.environ["CONCURRENCY"]) 8 | 9 | sqs_client = boto3.client("sqs") 10 | 11 | 12 | class PriorityMessageAvailable(Exception): 13 | pass 14 | 15 | 16 | def event_handler(event, _context): 17 | returned_messages = [] 18 | for record in event["Records"]: 19 | try: 20 | _process_record(record) 21 | except PriorityMessageAvailable: 22 | print("Yielding to priority message") 23 | returned_messages.append(record["messageId"]) 24 | except Exception as exc: 25 | print(f"Got unexpected error: {type(exc)} - {exc}") 26 | returned_messages.append(record["messageId"]) 27 | 28 | return { 29 | "batchItemFailures": [ 30 | {"itemIdentifier": msg_id} for msg_id in returned_messages 31 | ] 32 | } 33 | 34 | 35 | def _process_record(record): 36 | """Handle the record or raise an error when higher priority messages are available.""" 37 | is_priority_msg = record["eventSourceARN"] == PRIORITY_QUEUE_ARN 38 | if not is_priority_msg: 39 | _check_priority_messages_available() 40 | 41 | _handle_record(record) 42 | 43 | 44 | def _check_priority_messages_available(): 45 | response = sqs_client.get_queue_attributes( 46 | QueueUrl=PRIORITY_QUEUE_URL, 47 | AttributeNames=[ 48 | "ApproximateNumberOfMessages", 49 | "ApproximateNumberOfMessagesNotVisible", 50 | ], 51 | ) 52 | for key, value in response["Attributes"].items(): 53 | # Check if any attribute is > 0, raise error if 54 | # messages are in any way available on the priority queue. 55 | if int(value) > CONCURRENCY: 56 | raise PriorityMessageAvailable(key) 57 | 58 | 59 | def _handle_record(record): 60 | body = record["body"] 61 | message_group_id = record["attributes"]["MessageGroupId"] 62 | time.sleep(5) 63 | print(f"successfully processed {body} (Group ID {message_group_id})") 64 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | aws-cdk-lib==2.20.0 2 | constructs>=10.0.0,<11.0.0 3 | black==22.3.0 4 | boto3==1.21.42 5 | -------------------------------------------------------------------------------- /source.bat: -------------------------------------------------------------------------------- 1 | @echo off 2 | 3 | rem The sole purpose of this script is to make the command 4 | rem 5 | rem source .venv/bin/activate 6 | rem 7 | rem (which activates a Python virtualenv on Linux or Mac OS X) work on Windows. 8 | rem On Windows, this command just runs this batch file (the argument is ignored). 9 | rem 10 | rem Now we don't need to document a Windows command for activating a virtualenv. 11 | 12 | echo Executing .venv\Scripts\activate.bat for you 13 | .venv\Scripts\activate.bat 14 | --------------------------------------------------------------------------------