├── .circleci └── config.yml ├── .envrc.sample ├── .gitignore ├── README.md ├── accounts.png ├── layer ├── layer.yml └── requirements.txt ├── lib └── utils.py ├── package.json ├── requirements-dev.txt ├── resources ├── dynamodb.yml ├── s3.yml └── ssm.yml ├── serverless-common.yml ├── service.png ├── services ├── api │ ├── README.md │ ├── architecture.png │ ├── hello.py │ └── serverless.yml ├── message-service │ ├── README.md │ ├── architecture.png │ ├── eventbus.py │ ├── serverless.yml │ └── worker.py ├── stream-service │ ├── README.md │ ├── architecture.png │ ├── dds.py │ ├── kinesis.py │ └── serverless.yml └── workflow-service │ ├── README.md │ ├── architecture.png │ ├── hello.py │ └── serverless.yml ├── setup.cfg ├── tests ├── integration_tests │ ├── api │ │ ├── conftest.py │ │ └── test_hello_get.py │ ├── sfn.py │ ├── utils.py │ └── workflow_service │ │ ├── conftest.py │ │ └── test_main.py └── unit_tests │ └── lib │ └── test_utils.py └── yarn.lock /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | executors: 4 | default: 5 | working_directory: ~/workspace 6 | docker: 7 | - image: lambci/lambda:build-python3.8 8 | environment: 9 | PYTHONPATH: ./ 10 | 11 | references: 12 | commands: 13 | install_sls: &install_sls 14 | name: Install Serverless 15 | command: | 16 | curl -sL https://rpm.nodesource.com/setup_10.x | bash - 17 | yum install -y nodejs 18 | npm install -g yarn 19 | install_dependencies: &install_dependencies 20 | name: Install dependencies 21 | command: | 22 | python3 -m venv venv 23 | . venv/bin/activate 24 | pip install -r requirements-dev.txt 25 | yarn install 26 | 27 | jobs: 28 | build: 29 | executor: 30 | name: default 31 | steps: 32 | - checkout 33 | - run: *install_sls 34 | - run: *install_dependencies 35 | - persist_to_workspace: 36 | root: ~/workspace 37 | paths: 38 | - ./* 39 | lint: 40 | executor: 41 | name: default 42 | steps: 43 | - attach_workspace: 44 | at: ~/workspace 45 | - run: *install_sls 46 | - run: 47 | name: Lint 48 | command: | 49 | . venv/bin/activate 50 | yarn lint 51 | unit_test: 52 | executor: 53 | name: default 54 | steps: 55 | - attach_workspace: 56 | at: ~/workspace 57 | - run: *install_sls 58 | - run: 59 | name: Run Unit Test 60 | command: | 61 | . venv/bin/activate 62 | yarn test:unit 63 | api_test: 64 | executor: 65 | name: default 66 | steps: 67 | - attach_workspace: 68 | at: ~/workspace 69 | - run: *install_sls 70 | - run: 71 | name: Deploy API for integration test 72 | command: | 73 | . venv/bin/activate 74 | yarn deploy:db 75 | yarn deploy:layer 76 | yarn deploy:api 77 | environment: 78 | STAGE: 1 79 | - run: 80 | name: Run API integration test 81 | command: | 82 | . venv/bin/activate 83 | yarn test:api 84 | environment: 85 | STAGE: 1 86 | workflow_test: 87 | executor: 88 | name: default 89 | steps: 90 | - attach_workspace: 91 | at: ~/workspace 92 | - run: *install_sls 93 | - run: 94 | name: Deploy workflow service for integration test 95 | command: | 96 | . venv/bin/activate 97 | yarn deploy:db 98 | yarn deploy:s3 99 | yarn deploy:layer 100 | yarn deploy:workflow 101 | environment: 102 | STAGE: 2 103 | - run: 104 | name: Run workflow service integration test 105 | command: | 106 | . venv/bin/activate 107 | yarn test:workflow 108 | environment: 109 | STAGE: 2 110 | deploy_staging: 111 | executor: 112 | name: default 113 | steps: 114 | - attach_workspace: 115 | at: ~/workspace 116 | - run: *install_sls 117 | - run: 118 | name: Deploy services to staging 119 | command: | 120 | . venv/bin/activate 121 | yarn deploy:db 122 | yarn deploy:s3 123 | yarn deploy:layer 124 | yarn deploy:workflow 125 | yarn deploy:api 126 | yarn deploy:stream 127 | yarn deploy:message 128 | environment: 129 | STAGE: staging 130 | deploy_prod: 131 | executor: 132 | name: default 133 | steps: 134 | - attach_workspace: 135 | at: ~/workspace 136 | - run: *install_sls 137 | - run: 138 | name: Deploy services to prod 139 | command: | 140 | . venv/bin/activate 141 | yarn deploy:db 142 | yarn deploy:s3 143 | yarn deploy:layer 144 | yarn deploy:workflow 145 | yarn deploy:api 146 | yarn deploy:stream 147 | yarn deploy:message 148 | environment: 149 | STAGE: prod 150 | 151 | workflows: 152 | version: 2 153 | build-flow: 154 | jobs: 155 | - build: 156 | filters: 157 | tags: 158 | only: /.*/ 159 | - lint: 160 | requires: 161 | - build 162 | - unit_test: 163 | requires: 164 | - lint 165 | - api_test: 166 | requires: 167 | - unit_test 168 | context: dev 169 | - workflow_test: 170 | requires: 171 | - unit_test 172 | context: dev 173 | - deploy_staging: 174 | filters: 175 | branches: 176 | only: master 177 | requires: 178 | - workflow_test 179 | - api_test 180 | context: dev 181 | - deploy_prod: 182 | requires: 183 | - build 184 | filters: 185 | tags: 186 | only: /.*/ 187 | branches: 188 | ignore: /.*/ 189 | context: prod 190 | -------------------------------------------------------------------------------- /.envrc.sample: -------------------------------------------------------------------------------- 1 | export PYTHONPATH=$PWD 2 | export AWS_DEFAULT_REGION=us-east-1 3 | export AWS_ACCESS_KEY_ID=xxxxxx 4 | export AWS_SECRET_ACCESS_KEY=yyyyyy 5 | export STAGE=dev -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Distribution / packaging 2 | .Python 3 | env/ 4 | build/ 5 | develop-eggs/ 6 | dist/ 7 | downloads/ 8 | eggs/ 9 | .eggs/ 10 | lib64/ 11 | parts/ 12 | sdist/ 13 | var/ 14 | *.egg-info/ 15 | .installed.cfg 16 | *.egg 17 | venv 18 | node_modules 19 | .DS_Store 20 | __pycache__ 21 | .envrc 22 | .pytest_cache 23 | 24 | # Serverless directories 25 | .serverless -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![serverless](http://public.serverless.com/badges/v3.svg)](http://www.serverless.com) [![MIT License](http://img.shields.io/badge/license-MIT-blue.svg?style=flat)](LICENSE) 2 | 3 | # Serverless Enterprise Application Boilerplate For Python 4 | This is a boilerplate to build an AWS serverless enterprise application. In general, a serverless application is composed of some CloudFormation stacks. This repository shows you all the things which build that like how to separate each stack and build a directory structure using Serverless Framework, Python, and CircleCI. 5 | 6 | ## Deploy image 7 | Deploy this app to AWS using Serverless Framework via CircleCI. 8 | 9 | Architecture 10 | 11 | ## Directory Structure 12 | 13 | | Directory | Description | 14 | |:---|:---| 15 | |layer |Lambda layers. Put Python external libraries and common libraries that can use each service. | 16 | |lib |Common libraries that can use each service. | 17 | |services/api |API Service which a part of this application. Here is [the architecture](https://github.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/tree/master/services/api) | 18 | |services/workflow |API Service which a part of this application. Here is [the architecture](https://github.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/tree/master/services/workflow-service) | 19 | |services/message-service |Message Service which a part of this application. Here is [the architecture](https://github.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/tree/master/services/message-service) | 20 | |services/stream-service |Stream Service which a part of this application. Here is [the architecture](https://github.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/tree/master/services/stream-service) | 21 | |tests/unit_tests |Put unit tests. | 22 | |tests/integration_tests |Put E2E tests. | 23 | 24 | ## Commands 25 | 26 | All commands you need to build this application is defined as yarn script. 27 | Here is a part of that. 28 | | Command | Description | 29 | |:---|:---| 30 | | yarn lint | Run lint with flake8. | 31 | | yarn test:unit | Run unit testing. | 32 | | yarn test:workflow | Run E2E testing for workflow service. | 33 | | yarn deploy:workflow | Deploy workflow service. | 34 | | yarn deploy:db | Deploy tables. | 35 | 36 | All commands are defined in [package.json](https://github.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/blob/master/package.json). See that. 37 | 38 | ## Setup 39 | 40 | You can use this boilerplate as a skeleton to build your serverless application. First, check out the code and remove `.git` directory so that you can put it in your repository. 41 | 42 | ``` 43 | $ git clone git@github.com:serverless-operations/serverless-enterprise-application-boilerplate-for-python.git 44 | $ cd serverless-enterprise-application-boilerplate-for-python 45 | $ rm -rf .git 46 | $ yarn install 47 | ``` 48 | 49 | Setup needed environment valiables via `direnv`. 50 | ``` 51 | $ cp -pr .envrc.sample .envrc 52 | $ vi .envrc # edit 53 | 54 | # allow 55 | $ direnv allow 56 | ``` 57 | 58 | Install Python external libraries to develop into `venv`. 59 | ``` 60 | $ python3 -m venv venv 61 | $ . venv/bin/activate 62 | $ pip3 install -r requirements-dev.txt 63 | ``` 64 | 65 | Create a deployment S3 bucket to your AWS account with following schema. There valiables are defined in `serverless-common.yml` 66 | ``` 67 | ...deploys 68 | ``` 69 | 70 | 71 | Run deploy API to see this setup successfully. 72 | ``` 73 | $ yarn deploy:api 74 | ``` 75 | 76 | ## AWS Account 77 | 78 | This boilerplate supposes to use two AWS accounts, which are for production and other than that. 79 | You can switch AWS accounts to deploy using the CircleCI context feature. 80 | 81 | Accounts 82 | -------------------------------------------------------------------------------- /accounts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/accounts.png -------------------------------------------------------------------------------- /layer/layer.yml: -------------------------------------------------------------------------------- 1 | service: python-layer 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | runtime: ${file(serverless-common.yml):runtime} 8 | memorySize: ${file(serverless-common.yml):memorySize.${self:provider.stage}, file(serverless-common.yml):memorySize.default} 9 | deploymentBucket: 10 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 11 | 12 | custom: 13 | dockerizePipDefault: false 14 | pythonRequirements: 15 | layer: 16 | name: ${self:provider.stage}-python-requirements 17 | description: Python requirements lambda layer 18 | compatibleRuntimes: 19 | - ${self:provider.runtime} 20 | fileName: layer/requirements.txt 21 | dockerizePip: ${opt:dockerizePip, self:custom.dockerizePipDefault} 22 | 23 | layers: 24 | lib: 25 | path: lib 26 | name: ${self:provider.stage}-lib 27 | description: libraries for this project 28 | compatibleRuntimes: 29 | - ${self:provider.runtime} 30 | 31 | plugins: 32 | - serverless-python-requirements -------------------------------------------------------------------------------- /layer/requirements.txt: -------------------------------------------------------------------------------- 1 | jeffy==0.1.4 -------------------------------------------------------------------------------- /lib/utils.py: -------------------------------------------------------------------------------- 1 | """utility functions.""" 2 | import json 3 | import decimal 4 | 5 | 6 | def load_body(body): 7 | """Load request body.""" 8 | return json.loads(body) 9 | 10 | 11 | def response_builder(status_code, body={}): 12 | """Generate api response.""" 13 | return { 14 | 'statusCode': status_code, 15 | 'headers': { 16 | 'Content-Type': 'application/json; charset=utf-8', 17 | 'Access-Control-Allow-Origin': '*' 18 | }, 19 | 'body': json.dumps(body, cls=DecimalEncoder) 20 | } 21 | 22 | 23 | class DecimalEncoder(json.JSONEncoder): 24 | """JSON encoder.""" 25 | 26 | def default(self, obj): 27 | """encoding.""" 28 | if isinstance(obj, decimal.Decimal): 29 | return int(obj) 30 | return super(DecimalEncoder, self).default(obj) 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "devDependencies": { 3 | "serverless": "^1.68.0", 4 | "serverless-iam-roles-per-function": "^2.0.2", 5 | "serverless-pseudo-parameters": "^2.5.0", 6 | "serverless-python-requirements": "^5.1.0", 7 | "serverless-step-functions": "^2.18.0" 8 | }, 9 | "scripts": { 10 | "lint": "flake8 ./", 11 | "print:layer": "serverless print --config layer/layer.yml", 12 | "package:layer": "serverless package --config layer/layer.yml", 13 | "deploy:layer": "serverless deploy --config layer/layer.yml", 14 | "remove:layer": "serverless deploy --config layer/layer.yml", 15 | "print:db": "serverless print --config resources/dynamodb.yml", 16 | "package:db": "serverless package --config resources/dynamodb.yml", 17 | "deploy:db": "serverless deploy --config resources/dynamodb.yml", 18 | "remove:db": "serverless deploy --config resources/dynamodb.yml", 19 | "print:s3": "serverless print --config resources/s3.yml", 20 | "package:s3": "serverless package --config resources/s3.yml", 21 | "deploy:s3": "serverless deploy --config resources/s3.yml", 22 | "remove:s3": "serverless remove --config resources/s3.yml", 23 | "print:ssm": "serverless print --config resources/ssm.yml", 24 | "package:ssm": "serverless package --config resources/ssm.yml", 25 | "deploy:ssm": "serverless deploy --config resources/ssm.yml", 26 | "remove:ssm": "serverless remove --config resources/ssm.yml", 27 | "print:api": "serverless print --config services/api/serverless.yml", 28 | "package:api": "serverless package --config services/api/serverless.yml", 29 | "deploy:api": "serverless deploy --config services/api/serverless.yml", 30 | "remove:api": "serverless remove --config services/api/serverless.yml", 31 | "print:workflow": "serverless print --config services/workflow-service/serverless.yml", 32 | "package:workflow": "serverless package --config services/workflow-service/serverless.yml", 33 | "deploy:workflow": "serverless deploy --config services/workflow-service/serverless.yml", 34 | "remove:workflow": "serverless remove --config services/workflow-service/serverless.yml", 35 | "print:stream": "serverless print --config services/stream-service/serverless.yml", 36 | "package:stream": "serverless package --config services/stream-service/serverless.yml", 37 | "deploy:stream": "serverless deploy --config services/stream-service/serverless.yml", 38 | "remove:stream": "serverless remove --config services/stream-service/serverless.yml", 39 | "print:message": "serverless print --config services/message-service/serverless.yml", 40 | "package:message": "serverless package --config services/message-service/serverless.yml", 41 | "deploy:message": "serverless deploy --config services/message-service/serverless.yml", 42 | "remove:message": "serverless remove --config services/message-service/serverless.yml", 43 | "test:unit": "pytest tests/unit_tests -s", 44 | "test:workflow": "pytest tests/integration_tests/workflow_service -s", 45 | "test:api": "pytest tests/integration_tests/api -s" 46 | }, 47 | "name": "python" 48 | } 49 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | flake8==3.7.9 2 | pytest==5.4.1 3 | requests==2.22.0 4 | boto3==1.13.4 -------------------------------------------------------------------------------- /resources/dynamodb.yml: -------------------------------------------------------------------------------- 1 | service: db 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | deploymentBucket: 8 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 9 | 10 | custom: 11 | deploy: 12 | prod: prod 13 | default: dev 14 | 15 | resources: 16 | Parameters: 17 | ExampleTableName: 18 | Type: String 19 | Default: Example-${self:provider.stage} 20 | Resources: 21 | ExampleTable: 22 | Type: AWS::DynamoDB::Table 23 | Properties: 24 | TableName: !Ref ExampleTableName 25 | AttributeDefinitions: 26 | - AttributeName: ID 27 | AttributeType: S 28 | - AttributeName: Name 29 | AttributeType: S 30 | KeySchema: 31 | - AttributeName: ID 32 | KeyType: HASH 33 | - AttributeName: Name 34 | KeyType: RANGE 35 | BillingMode: PAY_PER_REQUEST 36 | PointInTimeRecoverySpecification: 37 | PointInTimeRecoveryEnabled: true 38 | StreamSpecification: 39 | StreamViewType: NEW_AND_OLD_IMAGES 40 | Outputs: 41 | ExampleTableName: 42 | Value: !Ref ExampleTable 43 | Export: 44 | Name: ExampleTableName-${self:provider.stage} 45 | ExampleTableArn: 46 | Value: !GetAtt ExampleTable.Arn 47 | Export: 48 | Name: ExampleTableArn-${self:provider.stage} 49 | ExampleTableStreamArn: 50 | Value: !GetAtt ExampleTable.StreamArn 51 | Export: 52 | Name: ExampleTableStreamArn-${self:provider.stage} -------------------------------------------------------------------------------- /resources/s3.yml: -------------------------------------------------------------------------------- 1 | service: s3 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | deploymentBucket: 8 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 9 | 10 | resources: 11 | Resources: 12 | ExampleBucket: 13 | Type: AWS::S3::Bucket 14 | Properties: 15 | BucketName: slsopsexample-bucket-${self:provider.stage} 16 | Outputs: 17 | ExampleBucketName: 18 | Value: !Ref ExampleBucket 19 | Export: 20 | Name: ExampleBucketName-${self:provider.stage} -------------------------------------------------------------------------------- /resources/ssm.yml: -------------------------------------------------------------------------------- 1 | service: ssm 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | deploymentBucket: 8 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 9 | 10 | resources: 11 | Resources: 12 | ExampleSecretValue: 13 | Type: AWS::SSM::Parameter 14 | Properties: 15 | Name: example-${self:provider.stage} 16 | Description: An example secret value 17 | Type: String 18 | Value: ${env:EXAMPLE_SECRET_VALUE, ''} -------------------------------------------------------------------------------- /serverless-common.yml: -------------------------------------------------------------------------------- 1 | appname: myapp 2 | runtime: python3.8 3 | logRetentionInDays: 4 | prod: 90 # 90 days in [prod] stage 5 | default: 3 # 3 days in other stages 6 | memorySize: 7 | prod: 512 8 | default: 128 9 | stage: ${env:STAGE} 10 | region: ${env:AWS_DEFAULT_REGION} 11 | deploymentBucketPath: 12 | prod: prod 13 | default: dev 14 | -------------------------------------------------------------------------------- /service.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/service.png -------------------------------------------------------------------------------- /services/api/README.md: -------------------------------------------------------------------------------- 1 | # API 2 | 3 | This is an API service example with API Gateway based. All APIs which connect with a service like a frontend are collected here. 4 | 5 | ## Architecture 6 | ![Architecture](https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/master/services/api/architecture.png) -------------------------------------------------------------------------------- /services/api/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/services/api/architecture.png -------------------------------------------------------------------------------- /services/api/hello.py: -------------------------------------------------------------------------------- 1 | """Hello Lambda function.""" 2 | import json 3 | 4 | 5 | def handler(event, context): 6 | """Hello Lambda function.""" 7 | body = { 8 | "message": "Go Serverless v1.0! Your function executed successfully!", 9 | "input": event 10 | } 11 | 12 | response = { 13 | "statusCode": 200, 14 | "body": json.dumps(body) 15 | } 16 | 17 | return response 18 | 19 | # Use this code if you don't use the http event with the LAMBDA-PROXY 20 | # integration 21 | """ 22 | return { 23 | "message": "Go Serverless v1.0! Your function executed successfully!", 24 | "event": event 25 | } 26 | """ 27 | -------------------------------------------------------------------------------- /services/api/serverless.yml: -------------------------------------------------------------------------------- 1 | service: api 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | runtime: ${file(serverless-common.yml):runtime} 8 | memorySize: ${file(serverless-common.yml):memorySize.${self:provider.stage}, file(serverless-common.yml):memorySize.default} 9 | deploymentBucket: 10 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 11 | timeout: 30 12 | logRetentionInDays: ${file(serverless-common.yml):logRetentionInDays.${self:provider.stage}, file(serverless-common.yml):logRetentionInDays.default} 13 | versionFunctions: false 14 | logs: 15 | restApi: 16 | accessLogging: true 17 | format: "requestId: $context.requestId" 18 | executionLogging: true 19 | level: INFO 20 | fullExecutionData: true 21 | iamRoleStatements: 22 | - Effect: Allow 23 | Action: 24 | - dynamodb:DescribeTable 25 | - dynamodb:Query 26 | - dynamodb:Scan 27 | - dynamodb:GetItem 28 | - dynamodb:BatchGetItem 29 | - dynamodb:PutItem 30 | - dynamodb:UpdateItem 31 | - dynamodb:DeleteItem 32 | - dynamodb:BatchWriteItem 33 | Resource: 34 | - ${cf:db-${self:provider.stage}.ExampleTableArn} 35 | 36 | package: 37 | exclude: 38 | - "**" 39 | include: 40 | - "services/api/**" 41 | 42 | functions: 43 | hello: 44 | handler: services/api/hello.handler 45 | events: 46 | - http: 47 | path: /hello 48 | method: get 49 | cors: true 50 | layers: 51 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 52 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} -------------------------------------------------------------------------------- /services/message-service/README.md: -------------------------------------------------------------------------------- 1 | # Message Service 2 | 3 | This is a message service example with SQS, SNS and EventBridge. 4 | 5 | ## Architecture 6 | ![Architecture](https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/master/services/message-service/architecture.png) -------------------------------------------------------------------------------- /services/message-service/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/services/message-service/architecture.png -------------------------------------------------------------------------------- /services/message-service/eventbus.py: -------------------------------------------------------------------------------- 1 | def handler(event, context): 2 | pass 3 | -------------------------------------------------------------------------------- /services/message-service/serverless.yml: -------------------------------------------------------------------------------- 1 | service: message 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | runtime: ${file(serverless-common.yml):runtime} 8 | memorySize: ${file(serverless-common.yml):memorySize.${self:provider.stage}, file(serverless-common.yml):memorySize.default} 9 | deploymentBucket: 10 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 11 | timeout: 900 12 | logRetentionInDays: ${file(serverless-common.yml):logRetentionInDays.${self:provider.stage}, file(serverless-common.yml):logRetentionInDays.default} 13 | versionFunctions: false 14 | 15 | functions: 16 | worker: 17 | handler: services/message-service/worker.handler 18 | events: 19 | - sqs: 20 | arn: 21 | Fn::GetAtt: 22 | - WorkerQueue 23 | - Arn 24 | layers: 25 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 26 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} 27 | environment: 28 | EVENT_SOURCE: test.action 29 | EVENT_BUS_NAME: ${self:custom.eventBusName} 30 | iamRoleStatements: 31 | - Effect: Allow 32 | Action: 33 | - events:PutEvents 34 | Resource: 35 | - arn:aws:events:#{AWS::Region}:#{AWS::AccountId}:event-bus/${self:custom.eventBusName} 36 | eventbus: 37 | handler: services/message-service/eventbus.handler 38 | events: 39 | - eventBridge: 40 | eventBus: arn:aws:events:#{AWS::Region}:#{AWS::AccountId}:event-bus/${self:custom.eventBusName} 41 | pattern: 42 | source: 43 | - test.action 44 | layers: 45 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 46 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} 47 | destinations: 48 | onSuccess: arn:aws:sns:#{AWS::Region}:#{AWS::AccountId}:${self:custom.notificationTopicName} 49 | onFailure: arn:aws:sqs:#{AWS::Region}:#{AWS::AccountId}:${self:custom.errorEventBusQueueName} 50 | iamRoleStatements: 51 | - Effect: Allow 52 | Action: 53 | - sqs:SendMessage 54 | Resource: 55 | Fn::GetAtt: 56 | - ErrorEventBusQueue 57 | - Arn 58 | - Effect: Allow 59 | Action: 60 | - sns:Publish 61 | Resource: !Ref Notification 62 | 63 | 64 | resources: 65 | Resources: 66 | Notification: 67 | Type: AWS::SNS::Topic 68 | Properties: 69 | TopicName: ${self:custom.notificationTopicName} 70 | EventBus: 71 | Type: AWS::Events::EventBus 72 | Properties: 73 | Name: ${self:custom.eventBusName} 74 | WorkerQueue: 75 | Type: AWS::SQS::Queue 76 | Properties: 77 | QueueName: ${self:provider.stage}-worker.fifo 78 | VisibilityTimeout: 901 79 | FifoQueue: true 80 | RedrivePolicy: 81 | deadLetterTargetArn: 82 | Fn::GetAtt: 83 | - ErrorWorkerQueue 84 | - Arn 85 | maxReceiveCount: 2 86 | ErrorWorkerQueue: 87 | Type: AWS::SQS::Queue 88 | Properties: 89 | QueueName: ${self:provider.stage}-worker-error.fifo 90 | FifoQueue: true 91 | ErrorEventBusQueue: 92 | Type: AWS::SQS::Queue 93 | Properties: 94 | QueueName: ${self:custom.errorEventBusQueueName} 95 | 96 | package: 97 | exclude: 98 | - "**" 99 | include: 100 | - "services/message-service/**" 101 | 102 | custom: 103 | serverless-iam-roles-per-function: 104 | defaultInherit: true 105 | eventBusName: ${self:provider.stage}-eventBus 106 | errorEventBusQueueName: ${self:provider.stage}-event-bus-error 107 | notificationTopicName: ${self:provider.stage}-notification 108 | 109 | plugins: 110 | - serverless-iam-roles-per-function 111 | - serverless-pseudo-parameters -------------------------------------------------------------------------------- /services/message-service/worker.py: -------------------------------------------------------------------------------- 1 | import os 2 | import boto3 3 | 4 | events = boto3.client('events') 5 | 6 | 7 | def handler(event, context): 8 | events.put_events( 9 | Entries=[ 10 | { 11 | 'Source': os.environ['EVENT_SOURCE'], 12 | 'DetailType': os.environ['EVENT_SOURCE'], 13 | 'Detail': '{}', 14 | 'EventBusName': os.environ['EVENT_BUS_NAME'], 15 | } 16 | ] 17 | ) 18 | -------------------------------------------------------------------------------- /services/stream-service/README.md: -------------------------------------------------------------------------------- 1 | # Stream Service 2 | 3 | This is a stream service example with Kinesis, Dynamodb Streams. 4 | 5 | ## Architecture 6 | 7 | ![Architecture](https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/master/services/stream-service/architecture.png) -------------------------------------------------------------------------------- /services/stream-service/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/services/stream-service/architecture.png -------------------------------------------------------------------------------- /services/stream-service/dds.py: -------------------------------------------------------------------------------- 1 | """DynamoDB Sreams backed Lambda.""" 2 | import os 3 | import json 4 | import boto3 5 | 6 | 7 | kinesis = boto3.client('kinesis') 8 | 9 | 10 | def handler(event, context): 11 | """DynamoDB Sreams backed Lambda.""" 12 | for record in event['Records']: 13 | kinesis.put_record( 14 | StreamName=os.environ['STREAM_NAME'], 15 | Data=json.dumps(record), 16 | PartitionKey='uuid', 17 | ) 18 | -------------------------------------------------------------------------------- /services/stream-service/kinesis.py: -------------------------------------------------------------------------------- 1 | """Hello Lambda function.""" 2 | 3 | 4 | def handler(event, context): 5 | """Hello Lambda function.""" 6 | pass 7 | -------------------------------------------------------------------------------- /services/stream-service/serverless.yml: -------------------------------------------------------------------------------- 1 | service: stream 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | runtime: ${file(serverless-common.yml):runtime} 8 | memorySize: ${file(serverless-common.yml):memorySize.${self:provider.stage}, file(serverless-common.yml):memorySize.default} 9 | deploymentBucket: 10 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 11 | timeout: 900 12 | logRetentionInDays: ${file(serverless-common.yml):logRetentionInDays.${self:provider.stage}, file(serverless-common.yml):logRetentionInDays.default} 13 | versionFunctions: false 14 | 15 | functions: 16 | dbStream: 17 | handler: services/stream-service/dds.handler 18 | events: 19 | - stream: 20 | arn: ${cf:db-${self:provider.stage}.ExampleTableStreamArn} 21 | batchSize: 100 22 | startingPosition: LATEST 23 | layers: 24 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 25 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} 26 | environment: 27 | STREAM_NAME: 28 | Ref: MyStream 29 | iamRoleStatements: 30 | - Effect: Allow 31 | Action: 32 | - kinesis:PutRecord 33 | Resource: 34 | Fn::GetAtt: 35 | - MyStream 36 | - Arn 37 | kinesisStream: 38 | handler: services/stream-service/kinesis.handler 39 | events: 40 | - stream: 41 | type: kinesis 42 | batchSize: 100 43 | arn: 44 | Fn::GetAtt: 45 | - MyStream 46 | - Arn 47 | layers: 48 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 49 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} 50 | 51 | resources: 52 | Resources: 53 | MyStream: 54 | Type: AWS::Kinesis::Stream 55 | Properties: 56 | ShardCount: 1 57 | Name: ${self:service}-${self:provider.stage}-mystream 58 | 59 | package: 60 | exclude: 61 | - "**" 62 | include: 63 | - "services/stream-service/**" 64 | 65 | custom: 66 | serverless-iam-roles-per-function: 67 | defaultInherit: true 68 | 69 | plugins: 70 | modules: 71 | - serverless-iam-roles-per-function -------------------------------------------------------------------------------- /services/workflow-service/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Workflow Service 3 | 4 | This is a workflow service example with StepFunctions. 5 | 6 | ## Architecture 7 | 8 | ![Architecture](https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/master/services/workflow-service/architecture.png) -------------------------------------------------------------------------------- /services/workflow-service/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/serverless-operations/serverless-enterprise-application-boilerplate-for-python/cf742cbc05639431a6b7e9ada1183cb7fc367bbd/services/workflow-service/architecture.png -------------------------------------------------------------------------------- /services/workflow-service/hello.py: -------------------------------------------------------------------------------- 1 | """Hello Lambda function.""" 2 | 3 | 4 | def handler(event, context): 5 | """Hello Lambda function.""" 6 | pass 7 | -------------------------------------------------------------------------------- /services/workflow-service/serverless.yml: -------------------------------------------------------------------------------- 1 | service: workflow 2 | 3 | provider: 4 | name: aws 5 | region: ${file(serverless-common.yml):region} 6 | stage: ${file(serverless-common.yml):stage} 7 | runtime: ${file(serverless-common.yml):runtime} 8 | memorySize: ${file(serverless-common.yml):memorySize.${self:provider.stage}, file(serverless-common.yml):memorySize.default} 9 | deploymentBucket: 10 | name: ${file(serverless-common.yml):appname}.${file(serverless-common.yml):deploymentBucketPath.${self:provider.stage}, file(serverless-common.yml):deploymentBucketPath.default}.${self:provider.region}.deploys 11 | timeout: 900 12 | logRetentionInDays: ${file(serverless-common.yml):logRetentionInDays.${self:provider.stage}, file(serverless-common.yml):logRetentionInDays.default} 13 | versionFunctions: false 14 | 15 | functions: 16 | hello: 17 | handler: services/workflow-service/hello.handler 18 | layers: 19 | - ${cf:python-layer-${self:provider.stage}.PythonRequirementsLambdaLayerQualifiedArn} 20 | - ${cf:python-layer-${self:provider.stage}.LibLambdaLayerQualifiedArn} 21 | iamRoleStatements: 22 | - Effect: Allow 23 | Action: 24 | - dynamodb:DescribeTable 25 | - dynamodb:GetItem 26 | Resource: 27 | - ${cf:db-${self:provider.stage}.ExampleTableArn} 28 | environment: 29 | TABLE_NAME: ${cf:db-${self:provider.stage}.ExampleTableName} 30 | 31 | stepFunctions: 32 | validate: true 33 | stateMachines: 34 | myStateMachine: 35 | name: myStateMachine-${self:provider.stage} 36 | loggingConfig: 37 | level: ERROR 38 | includeExecutionData: true 39 | destinations: 40 | - Fn::GetAtt: [MyStateMachineLogGroup, Arn] 41 | definition: 42 | StartAt: FirstState 43 | States: 44 | FirstState: 45 | Type: Task 46 | Resource: 47 | Fn::GetAtt: [hello, Arn] 48 | End: true 49 | 50 | resources: 51 | Outputs: 52 | MyStateMachine: 53 | Value: 54 | Ref: MyStateMachineDash${self:provider.stage} 55 | Resources: 56 | MyStateMachineLogGroup: 57 | Type: AWS::Logs::LogGroup 58 | Properties: 59 | LogGroupName: /aws/states/MyStateMachine-${self:provider.stage} 60 | RetentionInDays: ${self:provider.logRetentionInDays} 61 | 62 | package: 63 | exclude: 64 | - "**" 65 | include: 66 | - "services/workflow-service/**" 67 | 68 | plugins: 69 | - serverless-step-functions 70 | - serverless-iam-roles-per-function 71 | 72 | custom: 73 | serverless-iam-roles-per-function: 74 | defaultInherit: true -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | exclude = .git,.serverless,venv,node_modules 3 | max-line-length = 200 -------------------------------------------------------------------------------- /tests/integration_tests/api/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from tests.integration_tests.utils import set_table_names 4 | from tests.integration_tests.utils import get_endpoint_url 5 | service = 'api' 6 | 7 | 8 | @pytest.fixture(scope='session', autouse=True) 9 | def setup_teardown(): 10 | """Set table name to environment valiables.""" 11 | set_table_names(os.environ['STAGE']) 12 | yield 13 | 14 | 15 | @pytest.fixture(scope='module', autouse=True) 16 | def endpoint(): 17 | """Pass API Gateway endpoint URL.""" 18 | yield(get_endpoint_url(service, os.environ['STAGE'])) 19 | -------------------------------------------------------------------------------- /tests/integration_tests/api/test_hello_get.py: -------------------------------------------------------------------------------- 1 | import requests 2 | 3 | 4 | class TestHello(object): 5 | 6 | def test_return_200_with_success(self, endpoint): 7 | """Should return 200 response.""" 8 | response = requests.get( 9 | url=endpoint + '/hello', 10 | headers={ 11 | 'Content-Type': 'application/json' 12 | } 13 | ) 14 | 15 | assert response.status_code == 200 16 | -------------------------------------------------------------------------------- /tests/integration_tests/sfn.py: -------------------------------------------------------------------------------- 1 | """Utilities of Step Functions for integration tests.""" 2 | import boto3 3 | import json 4 | 5 | 6 | class SfnTestUtils(): 7 | """Utilities of Step Functions for integration tests.""" 8 | 9 | _resource = None 10 | 11 | @classmethod 12 | def get_resource(cls): 13 | """Get sfn object.""" 14 | if SfnTestUtils._resource is None: 15 | SfnTestUtils._resource = boto3.client('stepfunctions') 16 | return SfnTestUtils._resource 17 | 18 | @classmethod 19 | def start(cls, arn, input={}): 20 | """Execute your statemachine.""" 21 | return cls.get_resource().start_execution( 22 | stateMachineArn=arn, 23 | input=json.dumps(input) 24 | )['executionArn'] 25 | 26 | @classmethod 27 | def describe(cls, arn): 28 | """Describe execution status.""" 29 | return cls.get_resource().describe_execution( 30 | executionArn=arn 31 | )['status'] 32 | -------------------------------------------------------------------------------- /tests/integration_tests/utils.py: -------------------------------------------------------------------------------- 1 | import boto3 2 | import os 3 | import time 4 | 5 | 6 | def sleep(seconds): 7 | """sleep for waiting end of async process.""" 8 | time.sleep(seconds) 9 | 10 | 11 | def get_endpoint_url(service_name, stage): 12 | """Get APIGateway endpoint URL.""" 13 | cloudformation = boto3.client('cloudformation') 14 | stackname = '{}-{}'.format(service_name, stage) 15 | response = cloudformation.describe_stacks( 16 | StackName=stackname 17 | ) 18 | 19 | for output in response['Stacks'][0]['Outputs']: 20 | if output['OutputKey'] == 'ServiceEndpoint': 21 | return output['OutputValue'] 22 | 23 | 24 | def set_table_names(stage): 25 | """Set tablename to environment variables.""" 26 | cloudformation = boto3.client('cloudformation') 27 | stackname = 'db-{}'.format(stage) 28 | response = cloudformation.describe_stacks( 29 | StackName=stackname 30 | ) 31 | 32 | for output in response['Stacks'][0]['Outputs']: 33 | os.environ[output.get('OutputKey')] = output.get('OutputValue') 34 | 35 | 36 | def set_s3_bucket_names(stage): 37 | """Set S3 info to environment variables.""" 38 | cloudformation = boto3.client('cloudformation') 39 | stackname = 's3-{}'.format(stage) 40 | response = cloudformation.describe_stacks( 41 | StackName=stackname 42 | ) 43 | 44 | for output in response['Stacks'][0]['Outputs']: 45 | os.environ[output.get('OutputKey')] = output.get('OutputValue') 46 | 47 | 48 | def set_sfn_arn(service_name, stage): 49 | """Set Step Functions info to environment variables.""" 50 | cloudformation = boto3.client('cloudformation') 51 | stackname = '{}-{}'.format(service_name, stage) 52 | response = cloudformation.describe_stacks( 53 | StackName=stackname 54 | ) 55 | 56 | for output in response['Stacks'][0]['Outputs']: 57 | os.environ[output.get('OutputKey')] = output.get('OutputValue') 58 | -------------------------------------------------------------------------------- /tests/integration_tests/workflow_service/conftest.py: -------------------------------------------------------------------------------- 1 | import os 2 | import pytest 3 | from tests.integration_tests.utils import set_table_names, set_sfn_arn, set_s3_bucket_names 4 | 5 | 6 | @pytest.fixture(scope='session', autouse=True) 7 | def setup_teardown(): 8 | """Set some resources info to environment valiables.""" 9 | set_table_names(stage=os.environ['STAGE']) 10 | set_sfn_arn('workflow', stage=os.environ['STAGE']) 11 | set_s3_bucket_names(stage=os.environ['STAGE']) 12 | 13 | 14 | @pytest.fixture 15 | def fixture_success(): 16 | # Write process before runing test 17 | yield 18 | # After process before runing test 19 | -------------------------------------------------------------------------------- /tests/integration_tests/workflow_service/test_main.py: -------------------------------------------------------------------------------- 1 | import os 2 | from tests.integration_tests.sfn import SfnTestUtils 3 | from tests.integration_tests.utils import sleep 4 | range_count = 10 5 | wait_time = 3 6 | 7 | 8 | class TestWorkflow(object): 9 | """Integration tests for workflow service.""" 10 | 11 | def test_success_case(self, fixture_success): 12 | """test for a normal situation.""" 13 | execution_arn = SfnTestUtils.start(os.environ['MyStateMachine']) 14 | for _ in range(range_count): 15 | sleep(wait_time) 16 | if SfnTestUtils.describe(execution_arn) == 'SUCCEEDED': 17 | break 18 | 19 | # Assert for a normal case. 20 | assert True 21 | -------------------------------------------------------------------------------- /tests/unit_tests/lib/test_utils.py: -------------------------------------------------------------------------------- 1 | """tests for utils.py.""" 2 | from unittest import TestCase 3 | from lib.utils import load_body 4 | 5 | 6 | class TestUtiles(TestCase): 7 | """tests for utils.py.""" 8 | 9 | def test_load_body(self): 10 | """Should return dict type object.""" 11 | assert load_body('{"a":"b"}') == {'a': 'b'} 12 | --------------------------------------------------------------------------------