├── cdk ├── .npmignore ├── jest.config.js ├── .gitignore ├── bin │ └── cdk-typescript.ts ├── cdk.json ├── test │ └── cdk-typescript.test.ts ├── package-projects.sh ├── README.md ├── tsconfig.json ├── package.json └── lib │ └── AdsStack.ts ├── images ├── architecture.png └── updateCredentials.png ├── backend └── src │ ├── ReplayAPI │ ├── .gitignore │ ├── model │ │ └── Ad.ts │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── helper │ │ └── dynamodbHelper.ts │ ├── webpack.config.js │ └── handler.ts │ ├── AdClientAPI │ ├── .gitignore │ ├── model │ │ └── ESObject.ts │ ├── .vscode │ │ └── launch.json │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── handler.ts │ └── webpack.config.js │ ├── AdPublisherAPI │ ├── .gitignore │ ├── model │ │ └── ESObject.ts │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── webpack.config.js │ └── handler.ts │ ├── AdReaderAPI │ ├── .gitignore │ ├── model │ │ └── ESObject.ts │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── webpack.config.js │ └── handler.ts │ ├── DynamoProcessor │ ├── .gitignore │ ├── model │ │ └── DynamoAd.ts │ ├── tsconfig.json │ ├── serverless.ts │ ├── package.json │ ├── webpack.config.js │ ├── helper │ │ └── dynamodbHelper.ts │ └── handler.ts │ ├── ESProcessor │ ├── .gitignore │ ├── model │ │ ├── TransformResult.ts │ │ └── ESAd.ts │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── helper │ │ ├── ElasticSearchHelper.ts │ │ └── HttpHelper.ts │ ├── webpack.config.js │ └── handler.ts │ └── PeriodicSnapshot │ ├── .gitignore │ ├── tsconfig.json │ ├── package.json │ ├── serverless.ts │ ├── handler.ts │ └── webpack.config.js ├── .github ├── dependabot.yml └── workflows │ └── codeql-analysis.yml ├── .gitignore ├── README.md └── LICENSE /cdk/.npmignore: -------------------------------------------------------------------------------- 1 | *.ts 2 | !*.d.ts 3 | 4 | # CDK asset staging directory 5 | .cdk.staging 6 | cdk.out 7 | -------------------------------------------------------------------------------- /images/architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/t1agob/eventsourcing-qldb/HEAD/images/architecture.png -------------------------------------------------------------------------------- /images/updateCredentials.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/t1agob/eventsourcing-qldb/HEAD/images/updateCredentials.png -------------------------------------------------------------------------------- /cdk/jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | roots: ['/test'], 3 | testMatch: ['**/*.test.ts'], 4 | transform: { 5 | '^.+\\.tsx?$': 'ts-jest' 6 | } 7 | }; 8 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/AdClientAPI/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/ESProcessor/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # Webpack directories 9 | .webpack -------------------------------------------------------------------------------- /cdk/.gitignore: -------------------------------------------------------------------------------- 1 | *.js 2 | !jest.config.js 3 | *.d.ts 4 | node_modules 5 | 6 | # CDK asset staging directory 7 | .cdk.staging 8 | cdk.out 9 | 10 | # Parcel default cache directory 11 | .parcel-cache 12 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/model/TransformResult.ts: -------------------------------------------------------------------------------- 1 | import ESAd from "./ESAd"; 2 | 3 | export default class TransformResult { 4 | public recordId: string; 5 | public result: string; 6 | public data: string 7 | } -------------------------------------------------------------------------------- /cdk/bin/cdk-typescript.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import 'source-map-support/register'; 3 | import * as cdk from '@aws-cdk/core'; 4 | import { AdsStack } from '../lib/AdsStack'; 5 | 6 | const app = new cdk.App(); 7 | new AdsStack(app, 'AdsStack'); 8 | 9 | -------------------------------------------------------------------------------- /cdk/cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node bin/cdk-typescript.ts", 3 | "context": { 4 | "@aws-cdk/core:enableStackNameDuplicates": "true", 5 | "aws-cdk:enableDiffNoFail": "true", 6 | "@aws-cdk/core:stackRelativeExports": "true" 7 | } 8 | } 9 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/model/ESObject.ts: -------------------------------------------------------------------------------- 1 | export default class Ad { 2 | public adId: string; 3 | public publisherId: string; 4 | public adTitle: string; 5 | public adDescription: string; 6 | public price: number; 7 | public currency: string; 8 | public category: string; 9 | } 10 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/model/ESAd.ts: -------------------------------------------------------------------------------- 1 | export default class ESAd { 2 | public adId: string; 3 | public publisherId: string; 4 | public adTitle: string; 5 | public adDescription: string; 6 | public price: number; 7 | public currency: string; 8 | public category: string; 9 | public tags: Array; 10 | } 11 | -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/model/ESObject.ts: -------------------------------------------------------------------------------- 1 | export default class Ad { 2 | public adId: string; 3 | public publisherId: string; 4 | public adTitle: string; 5 | public adDescription: string; 6 | public price: number; 7 | public currency: string; 8 | public category: string; 9 | public tags: Array 10 | public version: number; 11 | public timestamp: string; 12 | } 13 | -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/model/ESObject.ts: -------------------------------------------------------------------------------- 1 | export default class Ad { 2 | public adId: string; 3 | public publisherId: string; 4 | public adTitle: string; 5 | public adDescription: string; 6 | public price: number; 7 | public currency: string; 8 | public category: string; 9 | public tags: Array 10 | public version: number; 11 | public timestamp: string; 12 | } 13 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/model/DynamoAd.ts: -------------------------------------------------------------------------------- 1 | export default class DynamoAd { 2 | public id: string; 3 | public adId: string; 4 | public publisherId: string; 5 | public adTitle: string; 6 | public adDescription: string; 7 | public price: number; 8 | public currency: string; 9 | public category: string; 10 | public tags: Array; 11 | public version: number; 12 | } 13 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/model/Ad.ts: -------------------------------------------------------------------------------- 1 | export default class Ad { 2 | public id: string; 3 | public adId: string; 4 | public publisherId: string; 5 | public adTitle: string; 6 | public adDescription: string; 7 | public price: number; 8 | public currency: string; 9 | public category: string; 10 | public tags: Array 11 | public version: number; 12 | public timestamp: string; 13 | } 14 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Lambda", 5 | "type": "node", 6 | "request": "launch", 7 | "runtimeArgs": ["--inspect", "--debug-port=9229"], 8 | "program": "${workspaceFolder}/node_modules/serverless/bin/serverless", 9 | "args": ["offline"], 10 | "port": 9229, 11 | "console": "integratedTerminal" 12 | } 13 | ] 14 | } 15 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "npm" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /cdk/test/cdk-typescript.test.ts: -------------------------------------------------------------------------------- 1 | import { expect as expectCDK, matchTemplate, MatchStyle } from '@aws-cdk/assert'; 2 | import * as cdk from '@aws-cdk/core'; 3 | import * as CdkTypescript from '../lib/AdsStack'; 4 | 5 | test('Empty Stack', () => { 6 | const app = new cdk.App(); 7 | // WHEN 8 | const stack = new CdkTypescript.AdsStack(app, 'MyTestStack'); 9 | // THEN 10 | expectCDK(stack).to(matchTemplate({ 11 | "Resources": {} 12 | }, MatchStyle.EXACT)) 13 | }); 14 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": ["es2017"], 4 | "removeComments": true, 5 | "moduleResolution": "node", 6 | "noUnusedLocals": true, 7 | "noUnusedParameters": true, 8 | "sourceMap": true, 9 | "target": "es2017", 10 | "outDir": "lib" 11 | }, 12 | "include": ["./**/*.ts"], 13 | "exclude": [ 14 | "node_modules/**/*", 15 | ".serverless/**/*", 16 | ".webpack/**/*", 17 | "_warmup/**/*", 18 | ".vscode/**/*" 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Binaries for programs and plugins 2 | *.exe 3 | *.exe~ 4 | *.dll 5 | *.so 6 | *.dylib 7 | 8 | # Test binary, built with `go test -c` 9 | *.test 10 | 11 | # Output of the go coverage tool, specifically when used with LiteIDE 12 | *.out 13 | 14 | # Dependency directories (remove the comment below to include it) 15 | # vendor/ 16 | 17 | # Visual Studio specific directories and files 18 | .vs 19 | bin 20 | obj 21 | *.sln 22 | .vscode 23 | backend/src/.DS_Store 24 | backend/src/StreamProcessor/.DS_Store 25 | cdk/.DS_Store 26 | .DS_Store 27 | !cdk/bin/ 28 | -------------------------------------------------------------------------------- /cdk/package-projects.sh: -------------------------------------------------------------------------------- 1 | ## Package AdClientAPI 2 | cd ../backend/src/AdClientAPI && npm install && sls package 3 | 4 | ## Package AdPublisher 5 | cd ../AdPublisherAPI && npm install && sls package 6 | 7 | ## Package AdReaderAPI 8 | cd ../AdReaderAPI && npm install && sls package 9 | 10 | ## Package DynamoProcessor 11 | cd ../DynamoProcessor && npm install && sls package 12 | 13 | ## Package ESProcessor 14 | cd ../ESProcessor && npm install && sls package 15 | 16 | ## Package ESProcessor 17 | cd ../PeriodicSnapshot && npm install && sls package 18 | 19 | ## Return to CDK folder 20 | cd ../../../cdk -------------------------------------------------------------------------------- /cdk/README.md: -------------------------------------------------------------------------------- 1 | # Welcome to your CDK TypeScript project! 2 | 3 | This is a blank project for TypeScript development with CDK. 4 | 5 | The `cdk.json` file tells the CDK Toolkit how to execute your app. 6 | 7 | ## Useful commands 8 | 9 | * `npm run build` compile typescript to js 10 | * `npm run watch` watch for changes and compile 11 | * `npm run test` perform the jest unit tests 12 | * `cdk deploy` deploy this stack to your default AWS account/region 13 | * `cdk diff` compare deployed stack with current state 14 | * `cdk synth` emits the synthesized CloudFormation template 15 | -------------------------------------------------------------------------------- /cdk/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2018", 4 | "module": "commonjs", 5 | "lib": ["es2018"], 6 | "declaration": true, 7 | "strict": true, 8 | "noImplicitAny": true, 9 | "strictNullChecks": true, 10 | "noImplicitThis": true, 11 | "alwaysStrict": true, 12 | "noUnusedLocals": false, 13 | "noUnusedParameters": false, 14 | "noImplicitReturns": true, 15 | "noFallthroughCasesInSwitch": false, 16 | "inlineSourceMap": true, 17 | "inlineSources": true, 18 | "experimentalDecorators": true, 19 | "strictPropertyInitialization": false, 20 | "typeRoots": ["./node_modules/@types"] 21 | }, 22 | "exclude": ["cdk.out"] 23 | } 24 | -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "opsapi", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "aws-sdk": "^2.779.0", 11 | "source-map-support": "^0.5.10" 12 | }, 13 | "devDependencies": { 14 | "@types/aws-lambda": "^8.10.17", 15 | "@types/node": "^10.12.18", 16 | "@types/serverless": "^1.72.5", 17 | "fork-ts-checker-webpack-plugin": "^3.0.1", 18 | "serverless-webpack": "^5.2.0", 19 | "ts-loader": "^5.3.3", 20 | "ts-node": "^8.10.2", 21 | "typescript": "^3.2.4", 22 | "webpack": "^4.29.0", 23 | "webpack-node-externals": "^1.7.2" 24 | }, 25 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 26 | "license": "MIT" 27 | } 28 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adclientapi", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "axios": "^0.21.1", 11 | "class-transformer": "^0.3.1", 12 | "source-map-support": "^0.5.10" 13 | }, 14 | "devDependencies": { 15 | "@types/aws-lambda": "^8.10.17", 16 | "@types/node": "^10.12.18", 17 | "@types/serverless": "^1.72.5", 18 | "fork-ts-checker-webpack-plugin": "^3.0.1", 19 | "serverless-webpack": "^5.2.0", 20 | "ts-loader": "^5.3.3", 21 | "ts-node": "^8.10.2", 22 | "typescript": "^3.2.4", 23 | "webpack": "^4.29.0", 24 | "webpack-node-externals": "^1.7.2" 25 | }, 26 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 27 | "license": "MIT" 28 | } 29 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'dynamoprocessor', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | // Add the serverless-webpack plugin 18 | plugins: ['serverless-webpack'], 19 | provider: { 20 | name: 'aws', 21 | runtime: 'nodejs12.x', 22 | apiGateway: { 23 | minimumCompressionSize: 1024, 24 | }, 25 | environment: { 26 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 27 | }, 28 | }, 29 | functions: { 30 | hello: { 31 | handler: 'handler.handler', 32 | } 33 | } 34 | } 35 | 36 | module.exports = serverlessConfiguration; 37 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "replayapi", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "amazon-qldb-driver-nodejs": "^2.0.0", 11 | "aws-sdk": "^2.782.0", 12 | "source-map-support": "^0.5.10" 13 | }, 14 | "devDependencies": { 15 | "@types/aws-lambda": "^8.10.17", 16 | "@types/node": "^10.12.18", 17 | "@types/serverless": "^1.72.5", 18 | "fork-ts-checker-webpack-plugin": "^3.0.1", 19 | "serverless-webpack": "^5.2.0", 20 | "ts-loader": "^5.3.3", 21 | "ts-node": "^8.10.2", 22 | "typescript": "^3.2.4", 23 | "webpack": "^4.29.0", 24 | "webpack-node-externals": "^1.7.2" 25 | }, 26 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 27 | "license": "MIT" 28 | } 29 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "esprocessor", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "aws-kinesis-agg": "^4.1.2", 11 | "aws-sdk": "^2.771.0", 12 | "ion-js": "^4.0.1", 13 | "source-map-support": "^0.5.10" 14 | }, 15 | "devDependencies": { 16 | "@types/aws-lambda": "^8.10.17", 17 | "@types/node": "^10.12.18", 18 | "@types/serverless": "^1.72.5", 19 | "fork-ts-checker-webpack-plugin": "^3.0.1", 20 | "serverless-webpack": "^5.2.0", 21 | "ts-loader": "^5.3.3", 22 | "ts-node": "^8.10.2", 23 | "typescript": "^3.2.4", 24 | "webpack": "^4.29.0", 25 | "webpack-node-externals": "^1.7.2" 26 | }, 27 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 28 | "license": "MIT" 29 | } 30 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dynamoprocessor", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "aws-kinesis-agg": "^4.1.2", 11 | "aws-sdk": "^2.769.0", 12 | "ion-js": "^4.0.1", 13 | "source-map-support": "^0.5.10" 14 | }, 15 | "devDependencies": { 16 | "@types/aws-lambda": "^8.10.17", 17 | "@types/node": "^10.12.18", 18 | "@types/serverless": "^1.72.5", 19 | "fork-ts-checker-webpack-plugin": "^3.0.1", 20 | "serverless-webpack": "^5.2.0", 21 | "ts-loader": "^5.3.3", 22 | "ts-node": "^8.10.2", 23 | "typescript": "^3.2.4", 24 | "webpack": "^4.29.0", 25 | "webpack-node-externals": "^1.7.2" 26 | }, 27 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 28 | "license": "MIT" 29 | } 30 | -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adreaderapi", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "amazon-qldb-driver-nodejs": "^2.0.0", 11 | "aws-sdk": "^2.769.0", 12 | "ion-js": "^4.0.1", 13 | "jsbi": "^3.1.4", 14 | "source-map-support": "^0.5.10" 15 | }, 16 | "devDependencies": { 17 | "@types/aws-lambda": "^8.10.17", 18 | "@types/node": "^10.12.18", 19 | "@types/serverless": "^1.72.5", 20 | "fork-ts-checker-webpack-plugin": "^3.0.1", 21 | "serverless-webpack": "^5.2.0", 22 | "ts-loader": "^5.3.3", 23 | "ts-node": "^8.10.2", 24 | "typescript": "^3.2.4", 25 | "webpack": "^4.29.0", 26 | "webpack-node-externals": "^1.7.2" 27 | }, 28 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 29 | "license": "MIT" 30 | } 31 | -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adpublisherapi", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "dependencies": { 10 | "amazon-qldb-driver-nodejs": "^2.0.0", 11 | "aws-sdk": "^2.762.0", 12 | "ion-js": "^4.0.1", 13 | "jsbi": "^3.1.4", 14 | "source-map-support": "^0.5.10" 15 | }, 16 | "devDependencies": { 17 | "@types/aws-lambda": "^8.10.17", 18 | "@types/node": "^10.12.18", 19 | "@types/serverless": "^1.72.5", 20 | "fork-ts-checker-webpack-plugin": "^3.0.1", 21 | "serverless-webpack": "^5.2.0", 22 | "ts-loader": "^5.3.3", 23 | "ts-node": "^8.10.2", 24 | "typescript": "^3.2.4", 25 | "webpack": "^4.29.0", 26 | "webpack-node-externals": "^1.7.2" 27 | }, 28 | "author": "The serverless webpack authors (https://github.com/elastic-coders/serverless-webpack)", 29 | "license": "MIT" 30 | } 31 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'esprocessor', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | // Add the serverless-webpack plugin 18 | plugins: ['serverless-webpack'], 19 | provider: { 20 | name: 'aws', 21 | runtime: 'nodejs12.x', 22 | apiGateway: { 23 | minimumCompressionSize: 1024, 24 | }, 25 | environment: { 26 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 27 | }, 28 | }, 29 | functions: { 30 | hello: { 31 | handler: 'handler.handler', 32 | events: [ 33 | { 34 | http: { 35 | method: 'get', 36 | path: 'hello', 37 | } 38 | } 39 | ] 40 | } 41 | } 42 | } 43 | 44 | module.exports = serverlessConfiguration; 45 | -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'opsapi', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | // Add the serverless-webpack plugin 18 | plugins: ['serverless-webpack'], 19 | provider: { 20 | name: 'aws', 21 | runtime: 'nodejs12.x', 22 | region: 'eu-west-1', 23 | apiGateway: { 24 | minimumCompressionSize: 1024, 25 | }, 26 | environment: { 27 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 28 | ledgerName: 'adLedger', 29 | roleArn: 'arn:aws:iam::414275540131:role/PRODROLE', 30 | bucketName: 'ads-snapshot-8912762', 31 | }, 32 | }, 33 | functions: { 34 | handler: { 35 | handler: 'handler.handler', 36 | } 37 | } 38 | } 39 | 40 | module.exports = serverlessConfiguration; 41 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'adclientapi', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | configValidationMode: 'warn', 18 | // Add the serverless-webpack plugin 19 | plugins: ['serverless-webpack'], 20 | provider: { 21 | name: 'aws', 22 | runtime: 'nodejs12.x', 23 | apiGateway: { 24 | minimumCompressionSize: 1024, 25 | }, 26 | environment: { 27 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 28 | }, 29 | }, 30 | functions: { 31 | hello: { 32 | handler: 'handler.handler', 33 | events: [ 34 | { 35 | http: { 36 | method: 'get', 37 | path: 'handler', 38 | } 39 | } 40 | ] 41 | } 42 | } 43 | } 44 | 45 | module.exports = serverlessConfiguration; 46 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/helper/ElasticSearchHelper.ts: -------------------------------------------------------------------------------- 1 | import ESAd from '../model/ESAd'; 2 | import sendRequest from '../helper/HttpHelper'; 3 | 4 | 5 | const createOrUpdateAd = async (id: string, version: number, ad: ESAd) => { 6 | 7 | const doc = { 8 | adId: ad.adId, 9 | publisherId: ad.publisherId, 10 | adTitle: ad.adTitle, 11 | adDescription: ad.adDescription, 12 | price: ad.price, 13 | currency: ad.currency, 14 | category: ad.category, 15 | tags: ad.tags 16 | }; 17 | 18 | const response = await sendRequest( 19 | "PUT", 20 | `/ad/_doc/${id}?version=${version}&version_type=external`, 21 | doc, 22 | ); 23 | console.log(`RESPONSE: ${JSON.stringify(response)}`); 24 | }; 25 | 26 | const deleteAd = async (id: string, version: number) => { 27 | console.log("No data section so handle as a delete"); 28 | 29 | const response = await sendRequest( 30 | "DELETE", 31 | `/ad/_doc/${id}?version=${version}&version_type=external`, 32 | ); 33 | console.log(`RESPONSE: ${JSON.stringify(response)}`); 34 | }; 35 | 36 | 37 | export { createOrUpdateAd, deleteAd }; -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'adpublisherapi', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | configValidationMode: 'warn', 18 | // Add the serverless-webpack plugin 19 | plugins: ['serverless-webpack'], 20 | provider: { 21 | name: 'aws', 22 | runtime: 'nodejs12.x', 23 | region: "eu-west-1", 24 | apiGateway: { 25 | minimumCompressionSize: 1024, 26 | }, 27 | environment: { 28 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 29 | }, 30 | }, 31 | functions: { 32 | hello: { 33 | handler: 'handler.handler', 34 | environment: { 35 | ledgerName: "quick-start" 36 | }, 37 | events: [ 38 | { 39 | http: { 40 | method: 'delete', 41 | path: '{publisher}/ad/{adId}' 42 | } 43 | } 44 | ] 45 | } 46 | } 47 | } 48 | 49 | module.exports = serverlessConfiguration; 50 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'replayapi', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | frameworkVersion: '2', 11 | custom: { 12 | webpack: { 13 | webpackConfig: './webpack.config.js', 14 | includeModules: true 15 | } 16 | }, 17 | // Add the serverless-webpack plugin 18 | plugins: ['serverless-webpack'], 19 | provider: { 20 | name: 'aws', 21 | runtime: 'nodejs12.x', 22 | region: 'eu-west-1', 23 | apiGateway: { 24 | minimumCompressionSize: 1024, 25 | }, 26 | environment: { 27 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 28 | }, 29 | }, 30 | functions: { 31 | handler: { 32 | handler: 'handler.handler', 33 | events: [ 34 | { 35 | http: { 36 | method: 'get', 37 | path: '/replay/{id}', 38 | } 39 | }, 40 | { 41 | http: { 42 | method: 'get', 43 | path: '/replay', 44 | } 45 | } 46 | ] 47 | } 48 | } 49 | } 50 | 51 | module.exports = serverlessConfiguration; 52 | -------------------------------------------------------------------------------- /cdk/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cdk-typescript", 3 | "version": "0.1.0", 4 | "bin": { 5 | "cdk-typescript": "bin/cdk-typescript.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk" 12 | }, 13 | "devDependencies": { 14 | "@aws-cdk/assert": "1.64.0", 15 | "@types/jest": "^26.0.10", 16 | "@types/node": "10.17.27", 17 | "aws-cdk": "1.70.0", 18 | "jest": "^26.5.2", 19 | "ts-jest": "^26.4.1", 20 | "ts-node": "^8.1.0", 21 | "typescript": "~3.9.7" 22 | }, 23 | "dependencies": { 24 | "@aws-cdk/aws-apigateway": "^1.70.0", 25 | "@aws-cdk/aws-cloudformation": "^1.70.0", 26 | "@aws-cdk/aws-dynamodb": "^1.70.0", 27 | "@aws-cdk/aws-elasticsearch": "^1.70.0", 28 | "@aws-cdk/aws-events": "^1.70.0", 29 | "@aws-cdk/aws-events-targets": "^1.70.0", 30 | "@aws-cdk/aws-iam": "^1.70.0", 31 | "@aws-cdk/aws-kinesis": "^1.70.0", 32 | "@aws-cdk/aws-kinesisfirehose": "^1.70.0", 33 | "@aws-cdk/aws-kms": "^1.70.0", 34 | "@aws-cdk/aws-lambda": "^1.70.0", 35 | "@aws-cdk/aws-lambda-event-sources": "^1.70.0", 36 | "@aws-cdk/aws-qldb": "^1.70.0", 37 | "@aws-cdk/aws-s3": "^1.70.0", 38 | "@aws-cdk/core": "^1.70.0", 39 | "source-map-support": "^0.5.16" 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/handler.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayProxyHandler } from 'aws-lambda'; 2 | import axios from 'axios'; 3 | import { plainToClass } from 'class-transformer'; 4 | import Ad from './model/ESObject'; 5 | 6 | import 'source-map-support/register'; 7 | 8 | 9 | const esUsername = process.env.masterUserName; 10 | const esPassword = process.env.masterUserPassword; 11 | const esUrl = process.env.esUrl; 12 | 13 | export const handler: APIGatewayProxyHandler = async (event, _context) => { 14 | 15 | const query = event.queryStringParameters["q"]; 16 | 17 | console.log("received query: " + query); 18 | 19 | try { 20 | let list: Array = new Array(); 21 | 22 | const response = await axios.get(`https://${esUrl}/ad/_search`, { 23 | auth: { 24 | username: esUsername, 25 | password: esPassword 26 | }, 27 | data: `{"size": 25, "query": { "multi_match": { "query": "${query}","fields": ["adTitle","adDescription","category","tags"]}}}`, 28 | responseType: 'json' 29 | }); 30 | response.data.hits.hits.forEach(element => { 31 | let ad: Ad = plainToClass(Ad, element._source); 32 | 33 | list.push(ad); 34 | }); 35 | 36 | return { 37 | statusCode: 200, 38 | body: JSON.stringify(list) 39 | } 40 | } 41 | catch(err){ 42 | console.error(err.message); 43 | 44 | return { 45 | statusCode: 500, 46 | body: err.message 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/handler.ts: -------------------------------------------------------------------------------- 1 | import { QLDB } from 'aws-sdk'; 2 | import 'source-map-support/register'; 3 | 4 | const qldb = new QLDB(); 5 | const ledgerName = process.env.ledgerName; 6 | const roleArn = process.env.roleArn; 7 | const bucketName = process.env.bucketName; 8 | 9 | export const handler = () => { 10 | 11 | var timeRef = new Date(); 12 | timeRef.setDate(0); 13 | 14 | var startTime = new Date(timeRef.getFullYear(), timeRef.getMonth(), 1); 15 | startTime.setUTCHours(0,0,0); 16 | 17 | var endTime = new Date(timeRef.getFullYear(), timeRef.getMonth() + 1, 1); 18 | endTime.setUTCHours(0,0,0); 19 | 20 | console.log(`creating snapshot for '${ledgerName} with start time: ${startTime.toISOString()} and end time: ${endTime.toISOString()}`); 21 | 22 | var params = { 23 | Name: ledgerName, 24 | RoleArn: roleArn, 25 | S3ExportConfiguration: { 26 | Bucket: bucketName, 27 | EncryptionConfiguration: { 28 | ObjectEncryptionType: "SSE_S3", 29 | }, 30 | Prefix: "JournalExports" 31 | }, 32 | ExclusiveEndTime: endTime, 33 | InclusiveStartTime: startTime, 34 | }; 35 | 36 | qldb.exportJournalToS3(params, function(err, data) { 37 | if(err){ 38 | console.log(err, err.stack); 39 | } 40 | else{ 41 | console.log(data); 42 | } 43 | }); 44 | 45 | } 46 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/helper/dynamodbHelper.ts: -------------------------------------------------------------------------------- 1 | import DynamoAd from '../model/Ad'; 2 | import * as DynamoDB from 'aws-sdk/clients/dynamodb'; 3 | 4 | const tableName = process.env.tableName; 5 | const dynamodb = new DynamoDB.DocumentClient(); 6 | 7 | const createOrUpdateAd = async (id: string, version: number, ad: DynamoAd) => { 8 | console.log(`In createOrUpdateAd function with id ${id} and version ${version}`); 9 | const params = { 10 | TableName: tableName, 11 | Key: { id: id }, 12 | UpdateExpression: 'set adId=:adId, publisherId=:publisherId, version=:version, adTitle=:adTitle, adDescription=:adDescription, price=:price, currency=:currency, category=:category, tags=:tags', 13 | ExpressionAttributeValues: { 14 | ':adId': ad.adId, 15 | ':publisherId': ad.publisherId, 16 | ':adTitle': ad.adTitle, 17 | ':adDescription': ad.adDescription, 18 | ':price': ad.price, 19 | ':currency': ad.currency, 20 | ':category': ad.category, 21 | ':tags': ad.tags.toString(), 22 | ':version': version 23 | }, 24 | ConditionExpression: 'attribute_not_exists(pk) OR version <= :version', 25 | // ConditionExpression: 'attribute_not_exists(pk)', 26 | }; 27 | 28 | try { 29 | await dynamodb.update(params).promise(); 30 | 31 | } catch (err) { 32 | console.log(`Unable to update ad: ${id}. Error: ${err}`); 33 | } 34 | } 35 | 36 | 37 | export { createOrUpdateAd }; -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/serverless.ts: -------------------------------------------------------------------------------- 1 | import type { Serverless } from 'serverless/aws'; 2 | 3 | const serverlessConfiguration: Serverless = { 4 | service: { 5 | name: 'adreaderapi', 6 | // app and org for use with dashboard.serverless.com 7 | // app: your-app-name, 8 | // org: your-org-name, 9 | }, 10 | configValidationMode: 'warn', 11 | frameworkVersion: '2', 12 | custom: { 13 | webpack: { 14 | webpackConfig: './webpack.config.js', 15 | includeModules: true 16 | } 17 | }, 18 | // Add the serverless-webpack plugin 19 | plugins: ['serverless-webpack'], 20 | provider: { 21 | name: 'aws', 22 | runtime: 'nodejs12.x', 23 | region: "eu-west-1", 24 | apiGateway: { 25 | minimumCompressionSize: 1024, 26 | }, 27 | environment: { 28 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1', 29 | }, 30 | }, 31 | functions: { 32 | hello: { 33 | handler: 'handler.handler', 34 | environment: { 35 | ledgerName: "adLedger" 36 | }, 37 | role: "arn:aws:iam::414275540131:role/adstack-lambda-qldbaccess-role" , 38 | events: [ 39 | { 40 | http: { 41 | method: 'get', 42 | path: '{publisher}/ad/{adId}', 43 | request: { 44 | parameters: { 45 | querystrings: { 46 | versions: false 47 | } 48 | } 49 | } 50 | } 51 | } 52 | ] 53 | } 54 | } 55 | } 56 | 57 | module.exports = serverlessConfiguration; 58 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/AdClientAPI/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/PeriodicSnapshot/webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const slsw = require('serverless-webpack'); 3 | const nodeExternals = require('webpack-node-externals'); 4 | const ForkTsCheckerWebpackPlugin = require('fork-ts-checker-webpack-plugin'); 5 | 6 | module.exports = { 7 | context: __dirname, 8 | mode: slsw.lib.webpack.isLocal ? 'development' : 'production', 9 | entry: slsw.lib.entries, 10 | devtool: slsw.lib.webpack.isLocal ? 'cheap-module-eval-source-map' : 'source-map', 11 | resolve: { 12 | extensions: ['.mjs', '.json', '.ts'], 13 | symlinks: false, 14 | cacheWithContext: false, 15 | }, 16 | output: { 17 | libraryTarget: 'commonjs', 18 | path: path.join(__dirname, '.webpack'), 19 | filename: '[name].js', 20 | }, 21 | target: 'node', 22 | externals: [nodeExternals()], 23 | module: { 24 | rules: [ 25 | // all files with a `.ts` or `.tsx` extension will be handled by `ts-loader` 26 | { 27 | test: /\.(tsx?)$/, 28 | loader: 'ts-loader', 29 | exclude: [ 30 | [ 31 | path.resolve(__dirname, 'node_modules'), 32 | path.resolve(__dirname, '.serverless'), 33 | path.resolve(__dirname, '.webpack'), 34 | ], 35 | ], 36 | options: { 37 | transpileOnly: true, 38 | experimentalWatchApi: true, 39 | }, 40 | }, 41 | ], 42 | }, 43 | plugins: [ 44 | // new ForkTsCheckerWebpackPlugin({ 45 | // eslint: true, 46 | // eslintOptions: { 47 | // cache: true 48 | // } 49 | // }) 50 | ], 51 | }; 52 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/helper/dynamodbHelper.ts: -------------------------------------------------------------------------------- 1 | import DynamoAd from '../model/DynamoAd'; 2 | import * as DynamoDB from 'aws-sdk/clients/dynamodb'; 3 | 4 | const tableName = process.env.tableName; 5 | const dynamodb = new DynamoDB.DocumentClient(); 6 | 7 | const createOrUpdateAd = async (id: string, version: number, ad: DynamoAd) => { 8 | console.log(`In createOrUpdateAd function with id ${id} and version ${version}`); 9 | const params = { 10 | TableName: tableName, 11 | Key: { id: id }, 12 | UpdateExpression: 'set adId=:adId, publisherId=:publisherId, version=:version, adTitle=:adTitle, adDescription=:adDescription, price=:price, currency=:currency, category=:category, tags=:tags', 13 | ExpressionAttributeValues: { 14 | ':adId': ad.adId, 15 | ':publisherId': ad.publisherId, 16 | ':adTitle': ad.adTitle, 17 | ':adDescription': ad.adDescription, 18 | ':price': ad.price, 19 | ':currency': ad.currency, 20 | ':category': ad.category, 21 | ':tags': ad.tags.toString(), 22 | ':version': version 23 | }, 24 | ConditionExpression: 'attribute_not_exists(pk) OR version <= :version', 25 | }; 26 | 27 | try { 28 | await dynamodb.update(params).promise(); 29 | console.log(`Successful updated ad with id ${id}.`); 30 | } catch (err) { 31 | console.log(`Unable to update ad: ${id}. Error: ${err}`); 32 | } 33 | }; 34 | 35 | const deleteAd = async (id: string, version: number) => { 36 | console.log(`In deleteAd function with id ${id} and version ${version}`); 37 | 38 | const params = { 39 | TableName: tableName, 40 | Key: { id: id } 41 | }; 42 | 43 | try { 44 | await dynamodb.delete(params).promise(); 45 | console.log(`Successful deleted id ${id} with version ${version}`); 46 | } catch (err) { 47 | console.log(`Unable to update ad: ${id}. Error: ${err}`); 48 | } 49 | }; 50 | 51 | 52 | export { createOrUpdateAd, deleteAd }; -------------------------------------------------------------------------------- /backend/src/ESProcessor/helper/HttpHelper.ts: -------------------------------------------------------------------------------- 1 | import * as AWS from 'aws-sdk'; 2 | import * as path from 'path'; 3 | 4 | const REGION = process.env.AWS_REGION; 5 | const ELASTICSEARCH_DOMAIN = 6 | process.env.esUrl; 7 | const esUserName = process.env.masterUserName; 8 | const esPassword = process.env.masterUserPassword; 9 | const endpoint = new AWS.Endpoint(ELASTICSEARCH_DOMAIN); 10 | 11 | // @ts-ignore 12 | const httpClient = new AWS.HttpClient(); 13 | 14 | const sendRequest = async (httpMethod: string, requestPath: string, payload?) => { 15 | console.log( 16 | `In sendRequest with method ${httpMethod} path ${requestPath} and payload ${payload}` 17 | ); 18 | 19 | const request = new AWS.HttpRequest(endpoint, REGION); 20 | 21 | request.method = httpMethod; 22 | request.path = path.join(request.path, requestPath); 23 | request.body = JSON.stringify(payload); 24 | request.headers["Content-Type"] = "application/json"; 25 | request.headers["Authorization"] = 26 | "Basic " + Buffer.from(`${esUserName}:${esPassword}`).toString("base64"); 27 | request.headers.Host = ELASTICSEARCH_DOMAIN; 28 | 29 | return new Promise((resolve, reject) => { 30 | httpClient.handleRequest( 31 | request, 32 | null, 33 | (response) => { 34 | const { statusCode, statusMessage, headers } = response; 35 | console.log( 36 | `statusCode ${statusCode} statusMessage ${statusMessage} headers ${headers}` 37 | ); 38 | 39 | let body = ""; 40 | response.on("data", (chunk) => { 41 | body += chunk; 42 | }); 43 | response.on("end", () => { 44 | const data = { 45 | statusCode, 46 | statusMessage, 47 | headers, 48 | body, 49 | }; 50 | if (body) { 51 | data.body = JSON.parse(body); 52 | } 53 | resolve(data); 54 | }); 55 | }, 56 | (err) => { 57 | console.log(`Error inserting into ES: ${err}`); 58 | reject(err); 59 | } 60 | ); 61 | }); 62 | } 63 | 64 | export default sendRequest; -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | # For most projects, this workflow file will not need changing; you simply need 2 | # to commit it to your repository. 3 | # 4 | # You may wish to alter this file to override the set of languages analyzed, 5 | # or to provide custom queries or build logic. 6 | name: "CodeQL" 7 | 8 | on: 9 | push: 10 | branches: [master] 11 | pull_request: 12 | # The branches below must be a subset of the branches above 13 | branches: [master] 14 | schedule: 15 | - cron: '0 20 * * 6' 16 | 17 | jobs: 18 | analyze: 19 | name: Analyze 20 | runs-on: ubuntu-latest 21 | 22 | strategy: 23 | fail-fast: false 24 | matrix: 25 | # Override automatic language detection by changing the below list 26 | # Supported options are ['csharp', 'cpp', 'go', 'java', 'javascript', 'python'] 27 | language: ['javascript'] 28 | # Learn more... 29 | # https://docs.github.com/en/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#overriding-automatic-language-detection 30 | 31 | steps: 32 | - name: Checkout repository 33 | uses: actions/checkout@v2 34 | with: 35 | # We must fetch at least the immediate parents so that if this is 36 | # a pull request then we can checkout the head. 37 | fetch-depth: 2 38 | 39 | # If this run was triggered by a pull request event, then checkout 40 | # the head of the pull request instead of the merge commit. 41 | - run: git checkout HEAD^2 42 | if: ${{ github.event_name == 'pull_request' }} 43 | 44 | # Initializes the CodeQL tools for scanning. 45 | - name: Initialize CodeQL 46 | uses: github/codeql-action/init@v1 47 | with: 48 | languages: ${{ matrix.language }} 49 | # If you wish to specify custom queries, you can do so here or in a config file. 50 | # By default, queries listed here will override any specified in a config file. 51 | # Prefix the list here with "+" to use these queries and those in the config file. 52 | # queries: ./path/to/local/query, your-org/your-repo/queries@main 53 | 54 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 55 | # If this step fails, then you should remove it and run the build manually (see below) 56 | - name: Autobuild 57 | uses: github/codeql-action/autobuild@v1 58 | 59 | # ℹ️ Command-line programs to run using the OS shell. 60 | # 📚 https://git.io/JvXDl 61 | 62 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 63 | # and modify them (or add more) to build your code if your project 64 | # uses a compiled language 65 | 66 | #- run: | 67 | # make bootstrap 68 | # make release 69 | 70 | - name: Perform CodeQL Analysis 71 | uses: github/codeql-action/analyze@v1 72 | -------------------------------------------------------------------------------- /backend/src/ESProcessor/handler.ts: -------------------------------------------------------------------------------- 1 | 2 | // @ts-nocheck 3 | import * as deagg from 'aws-kinesis-agg'; 4 | import * as ion from 'ion-js'; 5 | import { deleteAd, createOrUpdateAd } from './helper/ElasticSearchHelper'; 6 | 7 | import 'source-map-support/register'; 8 | import ESAd from './model/ESAd'; 9 | 10 | const promiseDeaggregate = async (record) => new Promise((resolve, reject) => { 11 | deagg.deaggregateSync(record, true, (err, responseObject) => { 12 | if (err) { 13 | return reject(err); 14 | } 15 | 16 | return resolve(responseObject); 17 | }) 18 | }); 19 | 20 | const processIon = async (ionRecord) => { 21 | const version = ionRecord.payload.revision.metadata.version.numberValue(); 22 | const id = ionRecord.payload.revision.metadata.id.stringValue(); 23 | 24 | console.log(`Version ${version} and id ${id}`); 25 | 26 | if (ionRecord.payload.revision.data == null) { 27 | console.log("No data section, it will be handled as a delete."); 28 | 29 | await deleteAd(id, version); 30 | } 31 | else { 32 | const adId = ionRecord.payload.revision.data.adId.stringValue(); 33 | const publisherId = ionRecord.payload.revision.data.publisherId.stringValue(); 34 | const adTitle = ionRecord.payload.revision.data.adTitle.stringValue(); 35 | const adDescription = ionRecord.payload.revision.data.adDescription.stringValue(); 36 | const price = ionRecord.payload.revision.data.price.numberValue(); 37 | const currency = ionRecord.payload.revision.data.currency.stringValue(); 38 | const category = ionRecord.payload.revision.data.category.stringValue(); 39 | const tags = ionRecord.payload.revision.data.tags.toString(); 40 | 41 | console.log( 42 | `adId: ${adId}, publisherId: ${publisherId}, adTitle: ${adTitle}, adDescription: ${adDescription}, price: ${price}, currency: ${currency}, category: ${category}, tags: ${tags}` 43 | ); 44 | 45 | const ad: ESAd = new ESAd(); 46 | ad.adId = adId; 47 | ad.publisherId = publisherId; 48 | ad.adTitle = adTitle; 49 | ad.adDescription = adDescription; 50 | ad.price = price; 51 | ad.currency = currency; 52 | ad.category = category; 53 | ad.tags = tags; 54 | 55 | await createOrUpdateAd(id, version, ad); 56 | } 57 | }; 58 | 59 | const processRecords = async (records) => { 60 | await Promise.all( 61 | records.map(async (record) => { 62 | const payload = Buffer.from(record.data, 'base64'); 63 | 64 | const ionRecord = ion.load(payload); 65 | 66 | console.log(`ionRecord: ${ionRecord}`); 67 | 68 | // Only process records where the record type is REVISION_DETAILS 69 | if (ionRecord.recordType.stringValue() !== "REVISION_DETAILS") { 70 | console.log(`Skipping record of type ${ion.dumpPrettyText(ionRecord.recordType)}`); 71 | } else { 72 | console.log(`Ion Record: ${ion.dumpPrettyText(ionRecord.payload)}`); 73 | await processIon(ionRecord); 74 | } 75 | }), 76 | ); 77 | }; 78 | 79 | 80 | export const handler = async (event, context) => { 81 | 82 | console.log(`In ${context.functionName} processing ${event.Records.length} Kinesis Input Records`); 83 | 84 | await Promise.all( 85 | event.Records.map(async (element) => { 86 | console.log(element); 87 | const records = await promiseDeaggregate(element.kinesis); 88 | await processRecords(records); 89 | })); 90 | } 91 | -------------------------------------------------------------------------------- /backend/src/DynamoProcessor/handler.ts: -------------------------------------------------------------------------------- 1 | // @ts-nocheck 2 | import * as deagg from 'aws-kinesis-agg'; 3 | import * as ion from 'ion-js'; 4 | import { deleteAd, createOrUpdateAd } from './helper/dynamodbHelper'; 5 | 6 | import 'source-map-support/register'; 7 | import DynamoAd from './model/DynamoAd'; 8 | 9 | const promiseDeaggregate = async (record) => new Promise((resolve, reject) => { 10 | deagg.deaggregateSync(record, true, (err, responseObject) => { 11 | if(err){ 12 | return reject(err); 13 | } 14 | 15 | return resolve(responseObject); 16 | }) 17 | }); 18 | 19 | const processIon = async (ionRecord) => { 20 | const version = ionRecord.payload.revision.metadata.version.numberValue(); 21 | const id = ionRecord.payload.revision.metadata.id.stringValue(); 22 | 23 | console.log(`Version ${version} and id ${id}`); 24 | 25 | if (ionRecord.payload.revision.data == null){ 26 | console.log("No data section, it will be handled as a delete."); 27 | 28 | await deleteAd(id, version); 29 | } 30 | else { 31 | const adId = ionRecord.payload.revision.data.adId.stringValue(); 32 | const publisherId = ionRecord.payload.revision.data.publisherId.stringValue(); 33 | const adTitle = ionRecord.payload.revision.data.adTitle.stringValue(); 34 | const adDescription = ionRecord.payload.revision.data.adDescription.stringValue(); 35 | const price = ionRecord.payload.revision.data.price.numberValue(); 36 | const currency = ionRecord.payload.revision.data.currency.stringValue(); 37 | const category = ionRecord.payload.revision.data.category.stringValue(); 38 | const tags = ionRecord.payload.revision.data.tags.toString(); 39 | 40 | console.log( 41 | `adId: ${adId}, publisherId: ${publisherId}, adTitle: ${adTitle}, adDescription: ${adDescription}, price: ${price}, currency: ${currency}, category: ${category}, tags: ${tags}` 42 | ); 43 | 44 | const ad: DynamoAd = new DynamoAd(); 45 | ad.id = id; 46 | ad.version = version; 47 | ad.adId = adId; 48 | ad.publisherId = publisherId; 49 | ad.adTitle = adTitle; 50 | ad.adDescription = adDescription; 51 | ad.price = price; 52 | ad.currency = currency; 53 | ad.category = category; 54 | ad.tags = tags; 55 | 56 | await createOrUpdateAd(id, version, ad); 57 | } 58 | }; 59 | 60 | const processRecords = async (records) => { 61 | await Promise.all( 62 | records.map(async (record) => { 63 | const payload = Buffer.from(record.data, 'base64'); 64 | 65 | const ionRecord = ion.load(payload); 66 | 67 | console.log(`ionRecord: ${ionRecord}`); 68 | 69 | // Only process records where the record type is REVISION_DETAILS 70 | if (ionRecord.recordType.stringValue() !== "REVISION_DETAILS") { 71 | console.log(`Skipping record of type ${ion.dumpPrettyText(ionRecord.recordType)}`); 72 | } else { 73 | console.log(`Ion Record: ${ion.dumpPrettyText(ionRecord.payload)}`); 74 | await processIon(ionRecord); 75 | } 76 | }), 77 | ); 78 | }; 79 | 80 | 81 | export const handler = async (event, context) => { 82 | 83 | console.log(`In ${context.functionName} processing ${event.Records.length} Kinesis Input Records`); 84 | 85 | await Promise.all( 86 | event.Records.map(async (element) => { 87 | console.log(element); 88 | const records = await promiseDeaggregate(element.kinesis); 89 | await processRecords(records); 90 | })); 91 | } 92 | -------------------------------------------------------------------------------- /backend/src/ReplayAPI/handler.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayProxyHandler } from 'aws-lambda'; 2 | import 'source-map-support/register'; 3 | import { QldbDriver } from 'amazon-qldb-driver-nodejs'; 4 | import Ad from "./model/Ad"; 5 | import { createOrUpdateAd } from './helper/dynamodbHelper'; 6 | 7 | 8 | const serviceConfigurationOptions = { 9 | region: process.env.AWS_REGION 10 | }; 11 | 12 | const adLedger = "adLedger";// process.env.ledgerName; 13 | const driver: QldbDriver = new QldbDriver(adLedger, serviceConfigurationOptions); 14 | 15 | export const handler: APIGatewayProxyHandler = async (event, _context) => { 16 | 17 | try { 18 | if (event.httpMethod == "GET") { 19 | 20 | let startDateTime, endDateTime; 21 | 22 | const id = event.pathParameters["id"]; 23 | if (id != null && id != undefined) { // replay for specific ID - use query history 24 | if (event.queryStringParameters != null) { 25 | startDateTime = event.queryStringParameters["startDateTime"]; 26 | endDateTime = event.queryStringParameters["endDateTime"]; 27 | } 28 | 29 | var baseQuery = `SELECT h.data, h.metadata.id, h.metadata.version FROM history(Ads) AS h WHERE h.metadata.id = '${id}'`; 30 | 31 | if (startDateTime != undefined) { 32 | baseQuery = `${baseQuery} and h.metadata.txTime >= \`${startDateTime}\``; 33 | } 34 | 35 | if (endDateTime != undefined) { 36 | baseQuery = `${baseQuery} and h.metadata.txTime <= \`${endDateTime}\``; 37 | } 38 | 39 | // await deleteAd(id); 40 | 41 | const adList: Array = new Array(); 42 | await driver.executeLambda(async (txn) => { 43 | console.log(`running: ${baseQuery}`); 44 | 45 | return txn.execute(baseQuery); 46 | }).then((result) => { 47 | 48 | let ad: Ad; 49 | const resultList = result.getResultList(); 50 | 51 | resultList.forEach(element => { 52 | ad = new Ad(); 53 | ad.id = id; 54 | ad.adId = element.data.get("adId").stringValue(); 55 | ad.publisherId = element.data.get("publisherId").stringValue(); 56 | ad.adTitle = element.data.get("adTitle").stringValue(); 57 | ad.category = element.data.get("category").stringValue(); 58 | ad.adDescription = element.data.get("adDescription").stringValue(); 59 | ad.currency = element.data.get("currency").stringValue(); 60 | ad.price = element.data.get("price").numberValue(); 61 | ad.tags = element.data.get("tags").toString(); 62 | ad.version = element.version.numberValue(); 63 | 64 | console.log(ad); 65 | 66 | adList.push(ad); 67 | }); 68 | }); 69 | 70 | for (const [idx, ad] of adList.entries()){ 71 | await createOrUpdateAd(ad.id, ad.version, ad).then(() => { 72 | console.log(`replayed version ${ad.version} of ad '${ad.id}'`); 73 | }); 74 | } 75 | 76 | 77 | return { 78 | statusCode: 200, 79 | body: JSON.stringify(adList), 80 | }; 81 | } 82 | else { // replay for all ids - use streams 83 | if (event.queryStringParameters != null) { 84 | startDateTime = event.queryStringParameters["startDateTime"]; 85 | endDateTime = event.queryStringParameters["endDateTime"]; 86 | } 87 | else { 88 | 89 | } 90 | 91 | 92 | } 93 | } 94 | else { 95 | console.log(`${event.httpMethod} operation is not supported.`); 96 | return { 97 | statusCode: 400, 98 | body: `${event.httpMethod} operation is not supported.` 99 | }; 100 | } 101 | } 102 | catch (err) { 103 | console.error(err.message); 104 | 105 | return { 106 | statusCode: 500, 107 | body: err.message 108 | } 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /backend/src/AdPublisherAPI/handler.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayProxyHandler } from 'aws-lambda'; 2 | import 'source-map-support/register'; 3 | import { QldbDriver, TransactionExecutor } from 'amazon-qldb-driver-nodejs'; 4 | import Ad from './model/ESObject'; 5 | 6 | const tableName = "Ads"; 7 | const indexes = ["adId"]; 8 | const adLedger = process.env.ledgerName; 9 | 10 | const serviceConfigurationOptions = { 11 | region: process.env.AWS_REGION 12 | }; 13 | 14 | const driver: QldbDriver = new QldbDriver(adLedger, serviceConfigurationOptions); 15 | let tableExists = false; 16 | 17 | export const handler: APIGatewayProxyHandler = async (event, _context) => { 18 | 19 | if (event.httpMethod == "POST") { 20 | try { 21 | if (!tableExists) { 22 | await ensureTable(); 23 | } 24 | 25 | let adBody: Ad = JSON.parse(event.body); 26 | 27 | if (event.pathParameters["publisher"] == undefined) { 28 | return { 29 | statusCode: 400, 30 | body: "You need to specify publisherId" 31 | }; 32 | } 33 | 34 | adBody.publisherId = event.pathParameters["publisher"]; 35 | adBody.adId = generateUniqueAdId(); 36 | 37 | await driver.executeLambda(async txn => { 38 | txn.execute(`INSERT INTO ${tableName} ?`, adBody); 39 | }); 40 | 41 | return { 42 | statusCode: 200, 43 | body: `Ad with id ${adBody.adId} created successfuly.` 44 | }; 45 | 46 | } catch (e) { 47 | console.error(e); 48 | 49 | return { 50 | statusCode: 500, 51 | body: `Unable to create Ad: ${e}` 52 | }; 53 | } 54 | } 55 | else if (event.httpMethod == "PATCH") { 56 | try { 57 | if (!tableExists) { 58 | await ensureTable(); 59 | } 60 | 61 | let adBody: Ad = JSON.parse(event.body); 62 | 63 | if (event.pathParameters["publisher"] == undefined) { 64 | return { 65 | statusCode: 400, 66 | body: "You need to specify publisherId" 67 | }; 68 | } 69 | 70 | if (event.pathParameters["adId"] == undefined) { 71 | return { 72 | statusCode: 400, 73 | body: "You need to specify adId" 74 | }; 75 | } 76 | 77 | adBody.publisherId = event.pathParameters["publisher"]; 78 | adBody.adId = event.pathParameters["adId"]; 79 | 80 | const [exists, owner] = await ensureOwner(adBody.publisherId, adBody.adId); 81 | 82 | if (owner && exists) { 83 | await driver.executeLambda(async txn => { 84 | await txn.execute("UPDATE Ads SET adTitle = ?, adDescription = ?, price = ?, category = ?, tags = ? WHERE adId = ?", adBody.adTitle, adBody.adDescription, adBody.price, adBody.category, adBody.tags, adBody.adId); 85 | console.log(`Ad ${adBody.adId} updated.`); 86 | }); 87 | } 88 | 89 | if (!exists) { 90 | return { 91 | statusCode: 400, 92 | body: `Ad ${adBody.adId} does not exist` 93 | }; 94 | } 95 | 96 | if (!owner) { 97 | return { 98 | statusCode: 400, 99 | body: `Publisher ${adBody.publisherId} is not the owner of Ad ${adBody.adId}` 100 | }; 101 | } 102 | 103 | return { 104 | statusCode: 200, 105 | body: `Ad ${adBody.adId} updated successfuly.` 106 | }; 107 | 108 | } catch (e) { 109 | console.error(`patching error: ${e}`); 110 | return { 111 | statusCode: 500, 112 | body: e 113 | }; 114 | } 115 | } 116 | else if (event.httpMethod == "DELETE") { 117 | try { 118 | if (!tableExists) { 119 | await ensureTable(); 120 | } 121 | 122 | if (event.pathParameters["publisher"] == undefined) { 123 | return { 124 | statusCode: 400, 125 | body: "You need to specify publisherId" 126 | }; 127 | } 128 | 129 | if (event.pathParameters["adId"] == undefined) { 130 | return { 131 | statusCode: 400, 132 | body: "You need to specify adId" 133 | }; 134 | } 135 | 136 | const publisherId = event.pathParameters["publisher"]; 137 | const adId = event.pathParameters["adId"]; 138 | 139 | const [exists, owner] = await ensureOwner(publisherId, adId); 140 | 141 | if (owner && exists) { 142 | await driver.executeLambda(async txn => { 143 | await txn.execute("DELETE FROM Ads WHERE adId = ?", adId); 144 | console.log(`Ad ${adId} updated.`); 145 | }); 146 | } 147 | 148 | if (!exists) { 149 | return { 150 | statusCode: 400, 151 | body: `Ad ${adId} does not exist` 152 | }; 153 | } 154 | 155 | if (!owner) { 156 | return { 157 | statusCode: 400, 158 | body: `Publisher ${publisherId} is not the owner of Ad ${adId}` 159 | }; 160 | } 161 | 162 | return { 163 | statusCode: 200, 164 | body: `Ad ${adId} removed successfuly.` 165 | }; 166 | 167 | } catch (e) { 168 | console.error(`delete error: ${e}`); 169 | return { 170 | statusCode: 500, 171 | body: e 172 | }; 173 | } 174 | } 175 | else { 176 | console.log(`${event.httpMethod} operation is not supported.`); 177 | return { 178 | statusCode: 400, 179 | body: `${event.httpMethod} operation is not supported.` 180 | }; 181 | } 182 | } 183 | 184 | async function ensureTable() { 185 | try { 186 | const tables = await driver.getTableNames(); 187 | 188 | let exists = false; 189 | tables.forEach(table => { 190 | if (table == tableName) { 191 | exists = true; 192 | tableExists = true; 193 | } 194 | }); 195 | 196 | if (!exists) { 197 | await createTableandIndexes(); 198 | } 199 | 200 | } catch (e) { 201 | console.error(e); 202 | 203 | await createTableandIndexes(); 204 | } 205 | } 206 | 207 | async function createTableandIndexes() { 208 | console.log(`Table ${tableName} does not exist. Creating...`); 209 | 210 | await driver.executeLambda(async (txn: TransactionExecutor) => { 211 | await txn.execute(`CREATE TABLE ${tableName}`); 212 | 213 | console.log("Creating indexes..."); 214 | indexes.forEach(async index => { 215 | await txn.execute(`CREATE INDEX on ${tableName}(${index})`); 216 | }); 217 | }); 218 | 219 | tableExists = true; 220 | } 221 | 222 | async function ensureOwner(publisher: string, id: string) { 223 | 224 | let exists = false; 225 | let owner = false; 226 | 227 | await driver.executeLambda(async txn => { 228 | const result = await txn.execute("SELECT * FROM Ads WHERE adId = ?", id); 229 | 230 | result.getResultList().forEach(element => { 231 | exists = true; 232 | 233 | console.log(`Ad ${id} exists.`); 234 | 235 | if (element.get("publisherId").stringValue() == publisher) { 236 | console.log(`Valid owner. Ad ${id} belongs to publisher ${publisher}`); 237 | owner = true; 238 | } 239 | }); 240 | }); 241 | return [exists, owner]; 242 | } 243 | 244 | 245 | 246 | function generateUniqueAdId(): string { 247 | return Math.random().toString(36).substr(2, 9); 248 | } 249 | 250 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Event Sourcing and System of Record using Amazon QLDB 2 | This is an example implementation of a System of Record for a **Classified Ads** platform. This example follows an **Event Sourcing and CQRS** (Command Query Responsibility Segregation) pattern using [Amazon QLDB](https://aws.amazon.com/qldb/) as an immutable append-only event store and source of truth. 3 | 4 | > For this specific use case we have focused on the backend services only. 5 | 6 | ## Why Event Sourcing and CQRS? 7 | **Event Sourcing** is a pattern that tipically introduces the concept of **Event Store** - where all events are tracked - and **State Store** - where the latest and final state of each object is stored. While this may add some complexity to the architecture it has some advantages such as: 8 | - **Auditing**: Events are immutable and each state change corresponds to one or more events stored on the event store. 9 | - **Replay events**: in case of application failure, we are able to reconstruct the state of an entity by replaying all the events since the event store maintains the complete history of every single entity. 10 | - **Temporal queries**: we can determine the application state at any point in time very easily. This can be achieved by starting with a blank state store and replaying the events up to a specific point in time. 11 | - **Extensibility**: The event store raises events, and tasks perform operations in response to those events. This decoupling of the tasks from events allows more flexibility and extensibility. 12 | 13 | In traditional data management systems, both commands and queries are executed against the same data store. This introduces some challenges for applications with a large customer base, such as increased risk for data contention, additional complexity managing security and running routine operations. **CQRS** (Command Query Responsibility Segregation) pattern aims to solve this issues by segregating read and write operations which as some obvious benefits: 14 | - **Indepedent scaling**: CQRS allows the read and write workloads to scale independently which may result in fewer lock contentions and optimized costs. 15 | - **Optimized data schemas**: The read side can use a schema optimized for queries, while the write side uses a schema optimized for updates. 16 | - **Security**: It's easier to ensure that only the right entity as access to perform reads or writes on the correct data store. 17 | - **Separation of concerns**: Models will be more maintainable and flexible. Most of the complex business logic goes into the write model. 18 | - **Simpler queries**: By storing a materialized view in the read database the application can avoid complex joins that are compute intensive and potentially more expensive. 19 | 20 | 21 | ## Architecture 22 | The below architecture represents all the components used in setting up this example. **Amazon QLDB** was the obvious choice for this scenario since it is an immutable data store that triggers events which allows the platform to be highly extensible and support virtually any type of integration (*ex: ElasticSearch for Client APIs or EventBridge for integration across AWS Accounts*). 23 | 24 | ![architecture](images/architecture.png) 25 | 26 | ## Requirements 27 | - Visual Studio Code ([install](https://code.visualstudio.com/download)) 28 | - AWS Toolkit for Visual Studio Code ([install](https://docs.aws.amazon.com/toolkit-for-vscode/latest/userguide/setup-toolkit.html)) 29 | - Node.js ([install](https://nodejs.org/en/download/)) 30 | - Serverless Framework ([install](https://www.serverless.com/framework/docs/providers/aws/guide/installation/)) 31 | 32 | 33 | ## How to deploy 34 | Every single component of this infrastructure is automatically deploy by using [AWS CDK](https://aws.amazon.com/cdk/). To deploy follow this steps: 35 | 36 | 1. **Clone the project** 37 | 38 | ```bash 39 | $ mkdir projects && cd projects 40 | $ git clone https://github.com/t1agob/eventsourcing-qldb.git eventsourcing-qldb 41 | ``` 42 | 43 | 2. **Open the project in Visual Studio Code** 44 | 45 | ```bash 46 | $ cd eventsourcing-qldb 47 | $ code . 48 | ``` 49 | 50 | 3. **Navigate to cdk folder and install dependencies** 51 | 52 | ```bash 53 | $ cd cdk 54 | $ npm install 55 | ``` 56 | 57 | 4. **Update master username and password for ElasticSearch** 58 | 59 | On lines 19 and 20 you may find the master username and password that are going to be used as the credentials to access ElasticSearch service. You may want to use your own credentials so you should update the values here. 60 | 61 | > ElasticSearch enforces that the password contains one uppercase letter, one lowercase letter, one number and a special character. 62 | 63 | 64 | ![updateCredentials](images/updateCredentials.png) 65 | 66 | 5. **Install project dependencies and package projects** 67 | 68 | In order for CDK to be able to deploy, not only the infrastructure required but also the Lambda Functions that do the actual work, we need to make sure they are packaged correctly. 69 | 70 | Run the following scirpt from the current location to make sure all dependencies are installed and the projects are packaged. 71 | 72 | ```bash 73 | $ ./package-projects.sh 74 | ``` 75 | 76 | 6. **Deploy with CDK** 77 | 78 | ```bash 79 | $ cdk deploy 80 | ``` 81 | 82 | The full deployment takes around 15min to complete since this is a complex infrastructure but once deployed everything should be working right away. 83 | 84 | 85 | ## How to use 86 | Since we just created the APIs for the services we don't have a UI that allows us to test the scenario so it needs to be done by calling the APIs directly. For that you can use cURL, Postman or any other tool you prefer - I will use cURL for simplicity. 87 | 88 | > **Make sure to replace placeholders in the Url.** 89 | 90 | #### Create Ad 91 | 92 | ```json 93 | curl --location --request POST '[API ENDPOINT]/publisher/[PUBLISHER ID]/ad' \ 94 | --header 'Content-Type: application/json' \ 95 | --data-raw '{ 96 | "adTitle": "awesome title", 97 | "adDescription": "awesome description", 98 | "price": 10, 99 | "currency": "€", 100 | "category": "awesome category", 101 | "tags": [ 102 | "awesome", 103 | "category" 104 | ] 105 | }' 106 | ``` 107 | 108 | #### Update Ad 109 | 110 | ```json 111 | curl --location --request PATCH '[API ENDPOINT]/publisher/[PUBLISHER ID]/ad/[AD ID]' \ 112 | --header 'Content-Type: application/json' \ 113 | --data-raw '{ 114 | "adTitle": "awesome title", 115 | "adDescription": "awesome description", 116 | "price": 150, 117 | "currency": "€", 118 | "category": "awesome category", 119 | "tags": [ 120 | "awesome", 121 | "category" 122 | ] 123 | }' 124 | ``` 125 | 126 | #### Delete Ad 127 | 128 | ```json 129 | curl --location --request DELETE '[API_ENDPOINT]/publisher/[PUBLISHER_ID]/ad/[AD_ID]' 130 | ``` 131 | 132 | #### Get All Ads for specific Publisher 133 | 134 | ```json 135 | curl --location --request GET '[API_ENDPOINT]/publisher/[PUBLISHER_ID]/ad' 136 | ``` 137 | 138 | #### Get specific Ad 139 | 140 | ```json 141 | curl --location --request GET '[API_ENDPOINT]/publisher/[PUBLISHER_ID]/ad/[AD_ID]' 142 | ``` 143 | 144 | #### Get specific Ad with versions 145 | 146 | ```json 147 | curl --location --request GET '[API_ENDPOINT]/publisher/[PUBLISHER_ID]/ad/[AD_ID]?versions=true' 148 | ``` 149 | 150 | #### Search Ads on ElasticSearch 151 | 152 | ```json 153 | curl --location --request GET '[API_ENDPOINT]/?q=[QUERY]' 154 | ``` 155 | 156 | # Work in progress 157 | 158 | - [x] Add DynamoDB as the state store 159 | - [x] Update internal GET operations to query DynamoDB instead of QLDB (for best practices and scalability purposes) 160 | - [ ] Implement Event Sourcing features 161 | - [x] Snapshot 162 | - [x] Create a **snapshot** of state store after specific timeframe (eg. every 1st day of each month) 163 | - [ ] Replay 164 | - [x] Allow **replay of a single entity** within a specific time range. Start and end dates are optional. 165 | - [ ] Allow **replay of all items** - full state store loss 166 | - [ ] Integration with EventBridge 167 | 168 | 169 | # Important references 170 | In order to process the items being streamed from QLDB we need to deaggregate these into separate Ion Objects and convert them into the right format so we can push them to Elastic Search and DynamoDB. For that we used the [Kinesis Record Aggregation & Deaggregation Modules for AWS Lambda](https://github.com/awslabs/kinesis-aggregation) open source project from [AWSLabs](https://github.com/awslabs) and took as a reference implementation the one created by [Matt Lewis](https://github.com/mlewis7127) in [QLDB Simple Demo](https://github.com/AWS-South-Wales-User-Group/qldb-simple-demo). 171 | 172 | 173 | 174 | ## Contributions are welcome! 175 | 176 | If you feel that there is space for improvement in this project or if you find a bug please raise an issue or submit a PR. 177 | -------------------------------------------------------------------------------- /backend/src/AdReaderAPI/handler.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayProxyHandler } from 'aws-lambda'; 2 | import 'source-map-support/register'; 3 | import { QldbDriver, TransactionExecutor } from 'amazon-qldb-driver-nodejs'; 4 | import Ad from './model/ESObject'; 5 | import * as DynamoDB from 'aws-sdk/clients/dynamodb'; 6 | 7 | const dynamoTableName = process.env.tableName; 8 | const dynamodb = new DynamoDB.DocumentClient(); 9 | 10 | console.log(`DynamoDB Table Name: ${dynamoTableName}`); 11 | 12 | const tableName = "Ads"; 13 | const indexes = ["adId"]; 14 | 15 | const adLedger = process.env.ledgerName; 16 | 17 | const serviceConfigurationOptions = { 18 | region: process.env.AWS_REGION 19 | }; 20 | 21 | const driver: QldbDriver = new QldbDriver(adLedger, serviceConfigurationOptions); 22 | 23 | let tableExists = false; 24 | 25 | export const handler: APIGatewayProxyHandler = async (event, _context) => { 26 | 27 | if (event.httpMethod == "GET") { 28 | const publisher = event.pathParameters["publisher"]; 29 | const adId = event.pathParameters["adId"]; 30 | 31 | 32 | if (adId != undefined) { 33 | console.log(`Got the followin AdId: ${adId}`); 34 | 35 | if (event.queryStringParameters != null && event.queryStringParameters["versions"] != undefined) { 36 | if (event.queryStringParameters["versions"] == "false") { 37 | 38 | try { 39 | const ad = await getDynamoAd(adId, publisher); 40 | 41 | return { 42 | statusCode: 200, 43 | body: JSON.stringify(ad) 44 | }; 45 | } 46 | catch (e) { 47 | console.error(e); 48 | 49 | return { 50 | statusCode: 500, 51 | body: e 52 | }; 53 | } 54 | 55 | } 56 | else { // RETURN ALL VERSIONS OF AD 57 | try { 58 | if (!tableExists) { 59 | await ensureTable(); 60 | } 61 | 62 | const documentId = await getDynamoAdId(adId, publisher); 63 | console.log(`querying history for documentID: ${documentId}`); 64 | 65 | const query = "SELECT h.data.adId, h.data.publisherId, h.data.adTitle, h.data.category, h.data.adDescription, h.data.price, h.data.currency, h.data.tags, h.metadata.version, h.metadata.txTime FROM history(Ads) AS h WHERE h.metadata.id = ?"; 66 | 67 | const adList = await getAdList(query, true, documentId); 68 | 69 | if (adList.length == 0) { 70 | return { 71 | statusCode: 200, 72 | body: "[]" 73 | }; 74 | } 75 | else { 76 | return { 77 | statusCode: 200, 78 | body: JSON.stringify(adList) 79 | }; 80 | } 81 | } 82 | catch (e) { 83 | console.error(e); 84 | } 85 | } 86 | } 87 | else { // RETURN LATEST VERSION OF AD 88 | try { 89 | const ad = await getDynamoAd(adId, publisher); 90 | 91 | return { 92 | statusCode: 200, 93 | body: JSON.stringify(ad) 94 | }; 95 | } 96 | catch (e) { 97 | console.error(e); 98 | 99 | return { 100 | statusCode: 500, 101 | body: e 102 | }; 103 | } 104 | } 105 | 106 | } 107 | else { // RETRIEVE ALL ADS FROM PUBLISHER 108 | console.log(`retrieving all ads from publisher ${publisher}`); 109 | 110 | try { 111 | const adList = await getDynamoAdList(publisher); 112 | 113 | if (adList.length == 0) { 114 | return { 115 | statusCode: 200, 116 | body: "[]" 117 | }; 118 | } 119 | else { 120 | return { 121 | statusCode: 200, 122 | body: JSON.stringify(adList) 123 | }; 124 | } 125 | } 126 | catch (e) { 127 | console.error(e); 128 | } 129 | } 130 | } 131 | else { 132 | return { 133 | statusCode: 400, 134 | body: `${event.httpMethod} is not a supported operation.` 135 | } 136 | } 137 | } 138 | 139 | async function getAdList(query: string, versions: boolean, ...args: any[]): Promise> { 140 | let adList: Array = new Array(); 141 | 142 | await driver.executeLambda(async (txn) => { 143 | return txn.execute(query, ...args); 144 | }).then((result) => { 145 | let ad: Ad; 146 | const resultList = result.getResultList(); 147 | 148 | resultList.forEach(element => { 149 | ad = new Ad(); 150 | ad.adId = element.get("adId"); 151 | ad.publisherId = element.get("publisherId"); 152 | ad.adTitle = element.get("adTitle"); 153 | ad.category = element.get("category"); 154 | ad.adDescription = element.get("adDescription"); 155 | ad.currency = element.get("currency"); 156 | ad.price = element.get("price"); 157 | ad.tags = element.get("tags"); 158 | 159 | if (versions) { 160 | ad.version = element.get("version"); 161 | ad.timestamp = element.get("txTime"); 162 | } 163 | 164 | adList.push(ad); 165 | }); 166 | }); 167 | 168 | return adList; 169 | } 170 | 171 | async function getDynamoAd(adId: string, publisherId: string): Promise{ 172 | var params = { 173 | TableName: dynamoTableName, 174 | IndexName: 'publisherId-adId-index', 175 | KeyConditionExpression: 'publisherId = :publisherId and adId = :adId', 176 | ExpressionAttributeValues: { 177 | ':publisherId': publisherId, 178 | ':adId': adId 179 | } 180 | }; 181 | 182 | let queryResult; 183 | 184 | await dynamodb.query(params, (err, data) => { 185 | const rs: Ad = new Ad(); 186 | 187 | if (err) { 188 | console.error(err); 189 | return { 190 | statusCode: 500, 191 | body: err.message 192 | }; 193 | } 194 | else { 195 | if (data.Items.length > 0) { 196 | const dataItem = data.Items[0]; 197 | 198 | rs.adId = dataItem.adId; 199 | rs.publisherId = dataItem.publisherId; 200 | rs.adTitle = dataItem.adTitle; 201 | rs.adDescription = dataItem.adDescription; 202 | rs.category = dataItem.category; 203 | rs.currency = dataItem.currency; 204 | rs.price = dataItem.price; 205 | rs.tags = dataItem.tags; 206 | 207 | queryResult = rs; 208 | } 209 | } 210 | }).promise(); 211 | 212 | console.log(`result: ${queryResult}`); 213 | return queryResult; 214 | } 215 | 216 | async function getDynamoAdId(adId: string, publisherId: string): Promise { 217 | var params = { 218 | TableName: dynamoTableName, 219 | IndexName: 'publisherId-adId-index', 220 | KeyConditionExpression: 'publisherId = :publisherId and adId = :adId', 221 | ExpressionAttributeValues: { 222 | ':publisherId': publisherId, 223 | ':adId': adId 224 | } 225 | }; 226 | 227 | let queryResult; 228 | 229 | await dynamodb.query(params, (err, data) => { 230 | if (err) { 231 | console.error(err); 232 | return { 233 | statusCode: 500, 234 | body: err.message 235 | }; 236 | } 237 | else { 238 | if (data.Items.length > 0) { 239 | const dataItem = data.Items[0]; 240 | 241 | queryResult = dataItem.id; 242 | } 243 | } 244 | }).promise(); 245 | 246 | return queryResult; 247 | } 248 | 249 | async function getDynamoAdList(publisherId: string): Promise> { 250 | let adList: Array = new Array(); 251 | 252 | var params = { 253 | TableName: dynamoTableName, 254 | IndexName: 'publisherId-adId-index', 255 | KeyConditionExpression: 'publisherId = :publisherId', 256 | ExpressionAttributeValues: { 257 | ':publisherId': publisherId 258 | } 259 | }; 260 | 261 | await dynamodb.query(params, (err, data) => { 262 | if (err) { 263 | console.error(err); 264 | return { 265 | statusCode: 500, 266 | body: err.message 267 | }; 268 | } 269 | else { 270 | let ad: Ad; 271 | 272 | data.Items.forEach(dataItem => { 273 | ad = new Ad(); 274 | 275 | ad.adId = dataItem.adId; 276 | ad.publisherId = dataItem.publisherId; 277 | ad.adTitle = dataItem.adTitle; 278 | ad.adDescription = dataItem.adDescription; 279 | ad.category = dataItem.category; 280 | ad.currency = dataItem.currency; 281 | ad.price = dataItem.price; 282 | ad.tags = dataItem.tags; 283 | 284 | adList.push(ad); 285 | }); 286 | } 287 | }).promise(); 288 | 289 | return adList; 290 | } 291 | 292 | async function ensureTable() { 293 | try { 294 | const tables = await driver.getTableNames(); 295 | 296 | let exists = false; 297 | tables.forEach(table => { 298 | if (table == tableName) { 299 | exists = true; 300 | tableExists = true; 301 | } 302 | }); 303 | 304 | if (!exists) { 305 | await createTableandIndexes(); 306 | } 307 | 308 | } catch (e) { 309 | console.error(e); 310 | 311 | await createTableandIndexes(); 312 | } 313 | } 314 | 315 | async function createTableandIndexes() { 316 | console.log(`Table ${tableName} does not exist. Creating...`); 317 | 318 | await driver.executeLambda(async (txn: TransactionExecutor) => { 319 | await txn.execute(`CREATE TABLE ${tableName}`); 320 | 321 | console.log("Creating indexes..."); 322 | indexes.forEach(async index => { 323 | await txn.execute(`CREATE INDEX on ${tableName}(${index})`); 324 | }); 325 | }); 326 | 327 | tableExists = true; 328 | } 329 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /cdk/lib/AdsStack.ts: -------------------------------------------------------------------------------- 1 | import * as cdk from '@aws-cdk/core'; 2 | import { CfnLedger, CfnStream } from '@aws-cdk/aws-qldb'; 3 | import { Role, ServicePrincipal, ManagedPolicy, PolicyDocument, PolicyStatement, Effect } from '@aws-cdk/aws-iam'; 4 | import * as Kinesis from '@aws-cdk/aws-kinesis'; 5 | import { Runtime, Code, Function, StartingPosition } from '@aws-cdk/aws-lambda'; 6 | import { Duration, CfnOutput, RemovalPolicy } from '@aws-cdk/core'; 7 | import { RestApi, LambdaIntegration } from '@aws-cdk/aws-apigateway'; 8 | import { CfnDomain } from "@aws-cdk/aws-elasticsearch"; 9 | import { Alias } from "@aws-cdk/aws-kms"; 10 | import { StreamEncryption } from '@aws-cdk/aws-kinesis'; 11 | import * as dynamodb from '@aws-cdk/aws-dynamodb'; 12 | import { KinesisEventSource } from '@aws-cdk/aws-lambda-event-sources'; 13 | import { AttributeType } from '@aws-cdk/aws-dynamodb'; 14 | import { BlockPublicAccess, Bucket, LifecycleRule, StorageClass } from '@aws-cdk/aws-s3'; 15 | import { Rule, Schedule } from '@aws-cdk/aws-events'; 16 | import { LambdaFunction } from '@aws-cdk/aws-events-targets'; 17 | 18 | 19 | export class AdsStack extends cdk.Stack { 20 | constructor(scope: cdk.Construct, id: string, props?: cdk.StackProps) { 21 | super(scope, id, props); 22 | 23 | // UPDATE THIS SECTION WITH YOUR CREDENTIALS 24 | const esUserName = "elasticads"; 25 | const esPassword = "Elastic4d$"; 26 | 27 | // 28 | // CLIENT API - API GATEWAY -> LAMBDA -> ELASTIC SEARCH 29 | // 30 | 31 | const lambdaRole = new Role(this, "adstack-lambda-qldbaccess", { 32 | assumedBy: new ServicePrincipal("lambda.amazonaws.com"), 33 | roleName: "adstack-lambda-qldbaccess-role", 34 | }); 35 | 36 | lambdaRole.addManagedPolicy( 37 | ManagedPolicy.fromAwsManagedPolicyName("AWSLambdaFullAccess") 38 | ); 39 | lambdaRole.addManagedPolicy( 40 | ManagedPolicy.fromAwsManagedPolicyName("AmazonQLDBFullAccess") 41 | ); 42 | 43 | const key = Alias.fromAliasName(this, "esKey", "alias/aws/es"); 44 | 45 | const esPolicyStatement = new PolicyStatement({ 46 | effect: Effect.ALLOW, 47 | actions: ["es:*"], 48 | }); 49 | esPolicyStatement.addAnyPrincipal(); 50 | 51 | const esAccessPolicy = new PolicyDocument({ 52 | assignSids: true, 53 | statements: [esPolicyStatement], 54 | }); 55 | 56 | const es = new CfnDomain(this, "esDomain", { 57 | domainName: "elasticads", 58 | domainEndpointOptions: { 59 | enforceHttps: true, 60 | }, 61 | elasticsearchClusterConfig: { 62 | instanceCount: 1, 63 | dedicatedMasterEnabled: false, 64 | }, 65 | elasticsearchVersion: "7.7", 66 | encryptionAtRestOptions: { 67 | enabled: true, 68 | kmsKeyId: key.keyId, 69 | }, 70 | nodeToNodeEncryptionOptions: { 71 | enabled: true, 72 | }, 73 | ebsOptions: { 74 | ebsEnabled: true, 75 | volumeSize: 10, 76 | }, 77 | advancedSecurityOptions: { 78 | enabled: true, 79 | internalUserDatabaseEnabled: true, 80 | masterUserOptions: { 81 | masterUserName: esUserName, 82 | masterUserPassword: esPassword, 83 | }, 84 | }, 85 | accessPolicies: esAccessPolicy, 86 | }); 87 | 88 | const clientHandler = new Function(this, "ClientHandler", { 89 | runtime: Runtime.NODEJS_12_X, 90 | code: Code.fromAsset( 91 | "../backend/src/AdClientAPI/.serverless/adclientapi.zip" 92 | ), 93 | handler: "handler.handler", 94 | role: lambdaRole, 95 | timeout: Duration.seconds(15), 96 | environment: { 97 | masterUserName: esUserName, 98 | masterUserPassword: esPassword, 99 | esUrl: es.attrDomainEndpoint, 100 | }, 101 | }); 102 | 103 | new CfnOutput(this, "esEndpointOutput", { 104 | exportName: "esEndpoint", 105 | value: `https://${es.attrDomainEndpoint}/_plugin/kibana`, 106 | }); 107 | 108 | const clientApi = new RestApi(this, "ClientAPI", { 109 | restApiName: "Ads Client API", 110 | description: "This API allows customers to search for Ads", 111 | }); 112 | 113 | const searchAdsIntegration = new LambdaIntegration(clientHandler); 114 | clientApi.root.addMethod("GET", searchAdsIntegration, { 115 | requestParameters: { 116 | "method.request.querystring.q": false, 117 | }, 118 | }); 119 | 120 | // 121 | // PUBLISHER API - API GATEWAY -> LAMBDA -> QLDB 122 | // 123 | 124 | const ledger = new CfnLedger(this, "adLedger", { 125 | name: "adLedger", 126 | permissionsMode: "ALLOW_ALL", 127 | }); 128 | 129 | const QldbToKinesisRole = new Role(this, "qldbToKinesisRole", { 130 | roleName: "QldbToKinesisRole", 131 | assumedBy: new ServicePrincipal("qldb.amazonaws.com"), 132 | }); 133 | QldbToKinesisRole.addManagedPolicy( 134 | ManagedPolicy.fromAwsManagedPolicyName("AmazonKinesisFullAccess") 135 | ); 136 | 137 | const streamKey = Alias.fromAliasName(this, "streamKey", "alias/aws/kinesis"); 138 | 139 | const qldbStream = new Kinesis.Stream(this, "qldbStream", { 140 | shardCount: 1, 141 | encryption: StreamEncryption.KMS, 142 | encryptionKey: streamKey, 143 | streamName: "adsStream" 144 | }); 145 | 146 | new CfnStream(this, "qldbStreamConfig", { 147 | ledgerName: "adLedger", 148 | streamName: qldbStream.streamName, 149 | roleArn: QldbToKinesisRole.roleArn, 150 | inclusiveStartTime: new Date().toISOString(), 151 | kinesisConfiguration: { 152 | aggregationEnabled: true, 153 | streamArn: qldbStream.streamArn, 154 | }, 155 | }); 156 | 157 | // LAMBDA ROLE WITH PERMISSIONS TO WRITE TO ELASTIC SEARCH 158 | const processorLambdaRole = new Role(this, "adstack-lambda-es-access", { 159 | assumedBy: new ServicePrincipal("lambda.amazonaws.com"), 160 | roleName: "adstack-lambda-es-access-role", 161 | }); 162 | 163 | processorLambdaRole.addManagedPolicy( 164 | ManagedPolicy.fromAwsManagedPolicyName("AWSLambdaFullAccess") 165 | ); 166 | processorLambdaRole.addManagedPolicy( 167 | ManagedPolicy.fromAwsManagedPolicyName("AmazonKinesisFullAccess") 168 | ); 169 | 170 | // PROCESSOR LAMBDA 171 | const processorHandler = new Function(this, "ProcessorHandler", { 172 | runtime: Runtime.NODEJS_12_X, 173 | code: Code.fromAsset( 174 | "../backend/src/ESProcessor/.serverless/esprocessor.zip" 175 | ), 176 | handler: "handler.handler", 177 | role: processorLambdaRole, 178 | timeout: Duration.minutes(3), 179 | environment: { 180 | masterUserName: esUserName, 181 | masterUserPassword: esPassword, 182 | esUrl: es.attrDomainEndpoint 183 | }, 184 | }); 185 | 186 | processorHandler.addEventSource(new KinesisEventSource(qldbStream, { 187 | retryAttempts: 10, 188 | startingPosition: StartingPosition.TRIM_HORIZON, 189 | })); 190 | 191 | // DynamoDB 192 | const table = new dynamodb.Table(this, 'Table', { 193 | partitionKey: { name: 'id', type: AttributeType.STRING }, 194 | removalPolicy: RemovalPolicy.DESTROY 195 | }); 196 | 197 | table.addGlobalSecondaryIndex({ 198 | indexName: "publisherId-adId-index", 199 | partitionKey: { 200 | name: "publisherId", 201 | type: AttributeType.STRING 202 | }, 203 | sortKey: { 204 | name: "adId", 205 | type: AttributeType.STRING 206 | } 207 | }); 208 | 209 | // Dynamo Processor Lambda 210 | const dynamoProcessorRole = new Role(this, "adstack-lambda-dynamo-access", { 211 | assumedBy: new ServicePrincipal("lambda.amazonaws.com"), 212 | roleName: "adstack-lambda-dynamo-access-role", 213 | }); 214 | 215 | dynamoProcessorRole.addManagedPolicy( 216 | ManagedPolicy.fromAwsManagedPolicyName("AWSLambdaFullAccess") 217 | ); 218 | dynamoProcessorRole.addManagedPolicy( 219 | ManagedPolicy.fromAwsManagedPolicyName("AmazonDynamoDBFullAccess") 220 | ); 221 | dynamoProcessorRole.addManagedPolicy( 222 | ManagedPolicy.fromAwsManagedPolicyName("AmazonKinesisFullAccess") 223 | ); 224 | 225 | // PROCESSOR LAMBDA 226 | const dynamoProcessorHandler = new Function(this, "DynamoProcessorHandler", { 227 | runtime: Runtime.NODEJS_12_X, 228 | code: Code.fromAsset( 229 | "../backend/src/DynamoProcessor/.serverless/dynamoprocessor.zip" 230 | ), 231 | handler: "handler.handler", 232 | role: dynamoProcessorRole, 233 | timeout: Duration.minutes(3), 234 | environment: { 235 | tableName: table.tableName 236 | } 237 | }); 238 | 239 | dynamoProcessorHandler.addEventSource(new KinesisEventSource(qldbStream, { 240 | retryAttempts: 10, 241 | startingPosition: StartingPosition.TRIM_HORIZON, 242 | })); 243 | 244 | const publisherHandler = new Function(this, "PublisherHandler", { 245 | runtime: Runtime.NODEJS_12_X, 246 | role: lambdaRole, 247 | timeout: Duration.seconds(15), 248 | code: Code.fromAsset( 249 | "../backend/src/AdPublisherAPI/.serverless/adpublisherapi.zip" 250 | ), 251 | handler: "handler.handler", 252 | environment: { 253 | ledgerName: `${ledger.name}` 254 | } 255 | }); 256 | 257 | const readerHandler = new Function(this, "ReaderHandler", { 258 | runtime: Runtime.NODEJS_12_X, 259 | role: lambdaRole, 260 | timeout: Duration.seconds(15), 261 | code: Code.fromAsset( 262 | "../backend/src/AdReaderAPI/.serverless/adreaderapi.zip" 263 | ), 264 | handler: "handler.handler", 265 | environment: { 266 | ledgerName: `${ledger.name}`, 267 | tableName: table.tableName 268 | } 269 | }); 270 | 271 | const api = new RestApi(this, "AdsAPI", { 272 | restApiName: "Ads API", 273 | description: "This API allows users to manage Ads", 274 | }); 275 | 276 | const publisherRoot = api.root.addResource("publisher"); 277 | const publisherResource = publisherRoot.addResource("{publisher}"); 278 | const adRoot = publisherResource.addResource("ad"); 279 | const adOperationsResource = adRoot.addResource("{adId}"); 280 | 281 | // POST /publisher/{publisher}/ad 282 | const createAdIntegration = new LambdaIntegration(publisherHandler); 283 | adRoot.addMethod("POST", createAdIntegration); 284 | 285 | // PATCH /publisher/{publisher}/ad/{adId} 286 | const updateAdIntegration = new LambdaIntegration(publisherHandler); 287 | adOperationsResource.addMethod("PATCH", updateAdIntegration); 288 | 289 | // DELETE /publisher/{publisher}/ad/{adId} 290 | const deleteAdIntegration = new LambdaIntegration(publisherHandler); 291 | adOperationsResource.addMethod("DELETE", deleteAdIntegration); 292 | 293 | // GET /publisher/{publisher}/ad/* 294 | const getAllIntegration = new LambdaIntegration(readerHandler); 295 | adRoot.addMethod("GET", getAllIntegration); 296 | 297 | // GET /publisher/{publisher}/ad/{adId}?versions=true 298 | const getAdIntegration = new LambdaIntegration(readerHandler); 299 | adOperationsResource.addMethod("GET", getAdIntegration, { 300 | requestParameters: { 301 | "method.request.querystring.versions": false, 302 | }, 303 | }); 304 | 305 | // 306 | // OPERATIONS COMPONENTS 307 | // 308 | 309 | // Storage for Snapshots 310 | const snapshotBucket = new Bucket(this, "snapshotBucket", { 311 | blockPublicAccess: BlockPublicAccess.BLOCK_ALL, 312 | removalPolicy: RemovalPolicy.RETAIN, 313 | lifecycleRules: [ 314 | { 315 | enabled: true, 316 | transitions: [{ 317 | storageClass: StorageClass.GLACIER, 318 | transitionAfter: Duration.days(90) 319 | }] 320 | } 321 | ] 322 | }); 323 | 324 | // Role for lambda execution - permissions to export from QLDB and default permissions to write to cloudwatch 325 | const exportRole = new Role(this, "LambdaQLDBExportRole", { 326 | assumedBy: new ServicePrincipal("lambda.amazonaws.com"), 327 | roleName: "LambdaQLDBExportRole", 328 | }); 329 | 330 | exportRole.addToPrincipalPolicy(new PolicyStatement({ 331 | effect: Effect.ALLOW, 332 | actions: [ 333 | "qldb:ExportJournalToS3" 334 | ], 335 | resources: [ 336 | `arn:aws:qldb:${this.region}:${this.account}:ledger/${ledger.name}` 337 | ] 338 | })); 339 | 340 | exportRole.addToPrincipalPolicy(new PolicyStatement({ 341 | effect: Effect.ALLOW, 342 | actions: [ 343 | "logs:CreateLogStream", 344 | "logs:CreateLogGroup", 345 | "logs:PutLogEvents" 346 | ], 347 | resources: [ 348 | "*" 349 | ] 350 | })); 351 | 352 | // Role for export job execution - permissions to write to S3 353 | const exportJobRole = new Role(this, "QLDBExportJobRole", { 354 | assumedBy: new ServicePrincipal("qldb.amazonaws.com"), 355 | roleName: "QLDBExportJobRole", 356 | }); 357 | 358 | exportJobRole.addToPrincipalPolicy(new PolicyStatement({ 359 | effect: Effect.ALLOW, 360 | actions: [ 361 | "s3:PutObjectAcl", 362 | "s3:PutObject" 363 | ], 364 | resources: [ 365 | `${snapshotBucket.bucketArn}/*` 366 | ] 367 | })); 368 | 369 | // Snapshot Lambda 370 | const SnapshotHandler = new Function(this, "SnapshotHandler", { 371 | runtime: Runtime.NODEJS_12_X, 372 | code: Code.fromAsset( 373 | "../backend/src/PeriodicSnapshot/.serverless/opsapi.zip" 374 | ), 375 | handler: "handler.handler", 376 | role: exportRole, 377 | timeout: Duration.minutes(3), 378 | environment: { 379 | ledgerName: `${ledger.name}`, 380 | roleArn: exportJobRole.roleArn, 381 | bucketName: snapshotBucket.bucketName, 382 | } 383 | }); 384 | 385 | // Cloudwatch Event Rule to trigger on the first day of each month 386 | const lambdaTarget = new LambdaFunction(SnapshotHandler); 387 | 388 | new Rule(this, "monthlyTrigger", { 389 | ruleName: "CreateQLDBSnapshot", 390 | enabled: true, 391 | schedule: Schedule.cron({ 392 | minute: "1", 393 | hour: "0", 394 | day: "1", 395 | month: "*", 396 | year: "*" 397 | }), 398 | targets: [lambdaTarget] 399 | }); 400 | 401 | // Replay DynamoDB 402 | const replayTable = new dynamodb.Table(this, 'ReplayTable', { 403 | partitionKey: { name: 'id', type: AttributeType.STRING }, 404 | removalPolicy: RemovalPolicy.DESTROY 405 | }); 406 | 407 | replayTable.addGlobalSecondaryIndex({ 408 | indexName: "publisherId-adId-index", 409 | partitionKey: { 410 | name: "publisherId", 411 | type: AttributeType.STRING 412 | }, 413 | sortKey: { 414 | name: "adId", 415 | type: AttributeType.STRING 416 | } 417 | }); 418 | 419 | // replay Lambda Role 420 | const replayRole = new Role(this, "adstack-lambda-replay-dynamo-access", { 421 | assumedBy: new ServicePrincipal("lambda.amazonaws.com"), 422 | roleName: "adstack-lambda-replay-dynamo-access", 423 | }); 424 | 425 | replayRole.addManagedPolicy( 426 | ManagedPolicy.fromAwsManagedPolicyName("AWSLambdaFullAccess") 427 | ); 428 | replayRole.addManagedPolicy( 429 | ManagedPolicy.fromAwsManagedPolicyName("AmazonDynamoDBFullAccess") 430 | ); 431 | replayRole.addManagedPolicy( 432 | ManagedPolicy.fromAwsManagedPolicyName("AmazonQLDBFullAccess") 433 | ); 434 | 435 | // Replay LAMBDA 436 | const replayHandler = new Function(this, "ReplayHandler", { 437 | runtime: Runtime.NODEJS_12_X, 438 | code: Code.fromAsset( 439 | "../backend/src/ReplayAPI/.serverless/replayapi.zip" 440 | ), 441 | handler: "handler.handler", 442 | role: replayRole, 443 | timeout: Duration.minutes(3), 444 | environment: { 445 | tableName: replayTable.tableName 446 | } 447 | }); 448 | 449 | // Replay APIs 450 | const replayApi = new RestApi(this, "ReplayAPI", { 451 | restApiName: "Ads Replay API", 452 | description: "This API allows operations team to Ads to another state store - dynamodb - for troubleshooting or test purposes.", 453 | }); 454 | 455 | const replayIntegration = new LambdaIntegration(replayHandler); 456 | replayApi.root.addMethod("GET", replayIntegration, { 457 | requestParameters: { 458 | "method.request.querystring.startDateTime": false, 459 | "method.request.querystring.endDateTime": false, 460 | }, 461 | }); 462 | 463 | const replayEntityResource = replayApi.root.addResource("id").addResource("{id}"); 464 | 465 | replayEntityResource.addMethod("GET", replayIntegration, { 466 | requestParameters: { 467 | "method.request.querystring.startDateTime": false, 468 | "method.request.querystring.endDateTime": false, 469 | }, 470 | }); 471 | } 472 | } 473 | --------------------------------------------------------------------------------