├── .gitignore ├── LICENSE ├── README.md ├── docs └── stack.drawio ├── modules └── generatePassword.js ├── package-lock.json ├── package.json ├── resources ├── aurora-cluster.yml ├── elasticache.yml ├── s3.yml ├── security-groups.yml ├── ssm-parameters.yml └── vpc.yml ├── schema └── MyCube.js ├── serverless.yml └── serverlessCube.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | lerna-debug.log* 8 | 9 | # Diagnostic reports (https://nodejs.org/api/report.html) 10 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 11 | 12 | # Runtime data 13 | pids 14 | *.pid 15 | *.seed 16 | *.pid.lock 17 | 18 | # Directory for instrumented libs generated by jscoverage/JSCover 19 | lib-cov 20 | 21 | # Coverage directory used by tools like istanbul 22 | coverage 23 | *.lcov 24 | 25 | # nyc test coverage 26 | .nyc_output 27 | 28 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 29 | .grunt 30 | 31 | # Bower dependency directory (https://bower.io/) 32 | bower_components 33 | 34 | # node-waf configuration 35 | .lock-wscript 36 | 37 | # Compiled binary addons (https://nodejs.org/api/addons.html) 38 | build/Release 39 | 40 | # Dependency directories 41 | node_modules/ 42 | jspm_packages/ 43 | 44 | # TypeScript v1 declaration files 45 | typings/ 46 | 47 | # TypeScript cache 48 | *.tsbuildinfo 49 | 50 | # Optional npm cache directory 51 | .npm 52 | 53 | # Optional eslint cache 54 | .eslintcache 55 | 56 | # Microbundle cache 57 | .rpt2_cache/ 58 | .rts2_cache_cjs/ 59 | .rts2_cache_es/ 60 | .rts2_cache_umd/ 61 | 62 | # Optional REPL history 63 | .node_repl_history 64 | 65 | # Output of 'npm pack' 66 | *.tgz 67 | 68 | # Yarn Integrity file 69 | .yarn-integrity 70 | 71 | # dotenv environment variables file 72 | .env 73 | .env.test 74 | 75 | # parcel-bundler cache (https://parceljs.org/) 76 | .cache 77 | 78 | # Next.js build output 79 | .next 80 | 81 | # Nuxt.js build / generate output 82 | .nuxt 83 | dist 84 | 85 | # Gatsby files 86 | .cache/ 87 | # Comment in the public line in if your project uses Gatsby and *not* Next.js 88 | # https://nextjs.org/blog/next-9-1#public-directory-support 89 | # public 90 | 91 | # vuepress build output 92 | .vuepress/dist 93 | 94 | # Serverless directories 95 | .serverless/ 96 | 97 | # FuseBox cache 98 | .fusebox/ 99 | 100 | # DynamoDB Local files 101 | .dynamodb/ 102 | 103 | # TernJS port file 104 | .tern-port 105 | 106 | .DS_Store 107 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Tobi 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # serverless-cubejs 2 | Serverless Cube.js backend infrastructure on AWS. 3 | 4 | ## Architecture 5 | 6 | This project uses 7 | 8 | * API Gateway HTTP APIs for hosting the Cube.js backend API 9 | * Athena for querying data in S3 10 | * Aurora Serverless for pre-aggregations 11 | * ElastiCache for query result caching 12 | 13 | All of that runs in a preconfigured VPC. You can also have a look at the stack diagram in [docs/stack.drawio](docs/stack.drawio). 14 | 15 | AWS pricing applies, please make yourself familiar with the pricing of the deployed resources **BEFORE** you deploy the stack. This will not be covered by the AWS Free Tier! 16 | 17 | ## Usage 18 | 19 | ### Prerequisites 20 | 21 | You'll need an AWS account (obviously), and a readily installed [Serverless framework](https://www.serverless.com), along with local AWS credentials. 22 | 23 | ### Cube configuration 24 | 25 | You need to adapt (or add) the schemas for your cube in the [schema](schema/) folder appropriately (see [Cube.js docs](https://cube.dev/docs/cube)). 26 | 27 | ### Deployment 28 | 29 | ```bash 30 | $ sls deploy --aurora-password YOUR_AURORA_PASSWORD --s3-data-bucket YOUR_S3_BUCKET_NAME --cube-secret YOUR_CUBE_SECRET 31 | ``` 32 | 33 | The default stage that will be used is `dev`. You can also specify if via the `--stage` commandline option. 34 | 35 | ### Removal 36 | 37 | ```bash 38 | $ sls remove 39 | ``` 40 | 41 | It's possible that you have to manually clean some resources, e.g. the S3 bucket with the Athena query results. 42 | 43 | -------------------------------------------------------------------------------- /docs/stack.drawio: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 255 | 256 | 257 | 258 | 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 | 272 | 273 | 274 | 275 | 276 | 277 | 278 | 279 | 280 | 281 | 282 | 283 | 284 | 285 | 286 | 287 | 288 | 289 | 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | 298 | 299 | 300 | 301 | 302 | 303 | 304 | 305 | 306 | 307 | 308 | 309 | 310 | 311 | 312 | 313 | 314 | 315 | 316 | 317 | 318 | 319 | 320 | 321 | 322 | 323 | 324 | 325 | 326 | 327 | 328 | 329 | 330 | 331 | 332 | 333 | 334 | 335 | 336 | 337 | 338 | 339 | 340 | 341 | 342 | 343 | 344 | 345 | 346 | 347 | 348 | 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 | 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | 373 | 374 | 375 | 376 | 377 | 378 | 379 | 380 | 381 | 382 | 383 | 384 | 385 | 386 | 387 | 388 | 389 | 390 | 391 | 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | 408 | 409 | 410 | 411 | 412 | 413 | 414 | 415 | 416 | 417 | 418 | -------------------------------------------------------------------------------- /modules/generatePassword.js: -------------------------------------------------------------------------------- 1 | module.exports.get = () => { 2 | return `${[...Array(40)].map(() => Math.random().toString(36)[2]).join('')}`; 3 | }; 4 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-cubejs", 3 | "version": "0.1.1", 4 | "scripts": { 5 | "dev": "./node_modules/.bin/cubejs-dev-server", 6 | "cfn-lint": "cfn-lint .serverless/cloudformation-template-update-stack.json", 7 | "cfn-dia": "cfn-dia generate -t .serverless/cloudformation-template-update-stack.json -o docs/stack.drawio" 8 | }, 9 | "dependencies": { 10 | "@cubejs-backend/athena-driver": "^0.24.5", 11 | "@cubejs-backend/mysql-driver": "^0.24.5", 12 | "@cubejs-backend/server": "^0.24.5", 13 | "@cubejs-backend/serverless": "^0.24.5", 14 | "@cubejs-backend/serverless-aws": "^0.24.5", 15 | "aws-sdk": "^2.809.0", 16 | "jsonwebtoken": "^8.5.1", 17 | "jwk-to-pem": "^2.0.4", 18 | "lodash": "^4.17.20" 19 | }, 20 | "devDependencies": { 21 | "serverless-express": "^2.0.11", 22 | "serverless-iam-roles-per-function": "^3.0.1", 23 | "serverless-pseudo-parameters": "^2.5.0" 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /resources/aurora-cluster.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-rds-dbcluster.html 2 | Resources: 3 | AuroraCluster: 4 | Type: AWS::RDS::DBCluster 5 | Properties: 6 | Engine: aurora 7 | EngineMode: serverless 8 | EngineVersion: '5.6.10a' 9 | DBClusterIdentifier: ${self:custom.aurora.cluster.name} 10 | DBSubnetGroupName: 11 | Ref: AuroraClusterSubnetGroup 12 | DatabaseName: ${self:custom.aurora.database.name} 13 | MasterUsername: ${self:custom.aurora.masterUser.name} 14 | MasterUserPassword: ${self:custom.aurora.masterUser.password} 15 | BackupRetentionPeriod: ${self:custom.aurora.backup.retentionInDays} 16 | DeletionProtection: ${self:custom.aurora.cluster.deletionProtection} 17 | StorageEncrypted: true 18 | KmsKeyId: 19 | Fn::GetAtt: [AuroraClusterKMSKey, Arn] 20 | AvailabilityZones: 21 | - 'Fn::Select': [0, 'Fn::GetAZs': ''] 22 | - 'Fn::Select': [1, 'Fn::GetAZs': ''] 23 | VpcSecurityGroupIds: 24 | - Ref: AuroraClusterSecurityGroup 25 | ScalingConfiguration: 26 | AutoPause: ${self:custom.aurora.scaling.pause.autoPause} 27 | SecondsUntilAutoPause: ${self:custom.aurora.scaling.pause.secondsUntilAutoPause} 28 | MinCapacity: ${self:custom.aurora.scaling.capacity.min} 29 | MaxCapacity: ${self:custom.aurora.scaling.capacity.max} 30 | 31 | AuroraClusterSubnetGroup: 32 | Type: AWS::RDS::DBSubnetGroup 33 | Properties: 34 | DBSubnetGroupName: ${self:custom.aurora.cluster.name}-subnet-group 35 | DBSubnetGroupDescription: RDS Subnet Group for the ${self:custom.aurora.cluster.name} Aurora Cluster instance 36 | SubnetIds: 37 | - '#{PrivateASubnet}' 38 | - '#{PrivateBSubnet}' 39 | 40 | AuroraClusterKMSKey: 41 | Type: AWS::KMS::Key 42 | Properties: 43 | Description: KMS Key for the ${self:custom.aurora.cluster.name} Aurora Cluster instance 44 | EnableKeyRotation: false 45 | Enabled: true 46 | KeyPolicy: 47 | Version: '2012-10-17' 48 | Statement: 49 | - Sid: Administration 50 | Action: 51 | - kms:* 52 | Effect: Allow 53 | Principal: 54 | AWS: 55 | - Fn::Join: 56 | - '' 57 | - - 'arn:aws:iam::' 58 | - Ref: AWS::AccountId 59 | - :root 60 | Resource: '*' 61 | - Sid: Principals 62 | Effect: Allow 63 | Action: 64 | - kms:Encrypt 65 | - kms:Decrypt 66 | - kms:ReEncrypt* 67 | - kms:GenerateDataKey* 68 | - kms:DescribeKey 69 | Resource: '*' 70 | Principal: 71 | AWS: '*' 72 | Condition: 73 | StringEquals: 74 | kms:CallerAccount: 75 | Ref: AWS::AccountId 76 | kms:ViaService: 77 | Fn::Join: 78 | - '' 79 | - - ssm. 80 | - Ref: AWS::Region 81 | - .amazonaws.com 82 | 83 | AuroraClusterKMSKeyAlias: 84 | Type: AWS::KMS::Alias 85 | Properties: 86 | AliasName: alias/aurora-cluster-kms-key 87 | TargetKeyId: 88 | Ref: AuroraClusterKMSKey 89 | -------------------------------------------------------------------------------- /resources/elasticache.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | SubnetGroup: 3 | Type: AWS::ElastiCache::SubnetGroup 4 | Properties: 5 | Description: Cache Subnet Group 6 | SubnetIds: 7 | - '#{PrivateASubnet}' 8 | - '#{PrivateBSubnet}' 9 | 10 | ElastiCacheCluster: 11 | Type: AWS::ElastiCache::CacheCluster 12 | Properties: 13 | AutoMinorVersionUpgrade: true 14 | Engine: redis 15 | EngineVersion: '5.0.6' 16 | CacheNodeType: ${self:custom.elasticache.nodeType} 17 | ClusterName: elasticache-${self:service.name}-${self:provider.stage} 18 | NumCacheNodes: 1 19 | CacheSubnetGroupName: '#{SubnetGroup}' 20 | VpcSecurityGroupIds: 21 | - '#{CacheSecurityGroup.GroupId}' 22 | -------------------------------------------------------------------------------- /resources/s3.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | AthenaQueryResultsBucket: 3 | Type: 'AWS::S3::Bucket' 4 | Properties: 5 | BucketName: ${self:custom.s3.buckets.athenaQueryResults} 6 | BucketEncryption: 7 | ServerSideEncryptionConfiguration: 8 | - ServerSideEncryptionByDefault: 9 | SSEAlgorithm: aws:kms 10 | -------------------------------------------------------------------------------- /resources/security-groups.yml: -------------------------------------------------------------------------------- 1 | Resources: 2 | CacheSecurityGroup: 3 | Type: AWS::EC2::SecurityGroup 4 | Properties: 5 | GroupName: ${self:service.name}-elasticache-securitygroup-${self:provider.stage} 6 | GroupDescription: The VPC SecurityGroup for the Elasticache cluster 7 | VpcId: '#{VPC}' 8 | SecurityGroupIngress: 9 | - IpProtocol: tcp 10 | FromPort: 6379 11 | ToPort: 6379 12 | CidrIp: 10.0.0.0/20 13 | 14 | LambdaSecurityGroup: 15 | Type: AWS::EC2::SecurityGroup 16 | Properties: 17 | GroupName: ${self:service.name}-lambda-securitygroup-${self:provider.stage} 18 | GroupDescription: The VPC SecurityGroup for the Cube.js Lambda functions 19 | VpcId: '#{VPC}' 20 | 21 | AuroraClusterSecurityGroup: 22 | Type: AWS::EC2::SecurityGroup 23 | Properties: 24 | GroupName: ${self:service.name}-aurora-securitygroup-${self:provider.stage} 25 | GroupDescription: The VPC SecurityGroup for the ${self:custom.aurora.cluster.name} Aurora Cluster 26 | SecurityGroupEgress: 27 | - Description: Egress for the ${self:custom.aurora.cluster.name} Aurora Cluster 28 | CidrIp: 0.0.0.0/0 29 | IpProtocol: '-1' 30 | SecurityGroupIngress: 31 | - Description: TCP ingress for the ${self:custom.aurora.cluster.name} Aurora Cluster 32 | CidrIp: 10.0.0.0/22 33 | FromPort: 3306 34 | ToPort: 3306 35 | IpProtocol: tcp 36 | - Description: ICMP ingress for the ${self:custom.aurora.cluster.name} Aurora Cluster 37 | CidrIp: 10.0.0.0/22 38 | FromPort: -1 39 | ToPort: -1 40 | IpProtocol: icmp 41 | VpcId: '#{VPC}' 42 | -------------------------------------------------------------------------------- /resources/ssm-parameters.yml: -------------------------------------------------------------------------------- 1 | # See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ssm-parameter.html 2 | Resources: 3 | AuroraClusterUsername: 4 | Type: AWS::SSM::Parameter 5 | Properties: 6 | Name: /${self:provider.stage}/service/aurora/username 7 | Description: The master username of the Aurora Cluster instance 8 | Type: String 9 | Value: ${self:custom.aurora.masterUser.name} 10 | AuroraClusterPassword: 11 | Type: AWS::SSM::Parameter 12 | Properties: 13 | Name: /${self:provider.stage}/service/aurora/password 14 | Description: The password of the Aurora Cluster instance 15 | Type: String 16 | Value: ${self:custom.aurora.masterUser.password} 17 | AuroraClusterDatabaseName: 18 | Type: AWS::SSM::Parameter 19 | Properties: 20 | Name: /${self:provider.stage}/service/aurora/databasename 21 | Description: The database name of the Aurora Cluster instance 22 | Type: String 23 | Value: ${self:custom.aurora.database.name} 24 | AuroraClusterEndpointUrl: 25 | Type: AWS::SSM::Parameter 26 | Properties: 27 | Name: /${self:provider.stage}/service/aurora/endpoint 28 | Description: The endpoint of the Aurora Cluster instance 29 | Type: String 30 | Value: 31 | Fn::GetAtt: [AuroraCluster, 'Endpoint.Address'] 32 | -------------------------------------------------------------------------------- /resources/vpc.yml: -------------------------------------------------------------------------------- 1 | # See: 2 | # * https://www.infoq.com/articles/aws-vpc-cloudformation/ 3 | # * https://www.infoq.com/articles/aws-vpc-cloudformation-part2/ 4 | # * https://templates.cloudonaut.io/en/stable/vpc/ 5 | 6 | Resources: 7 | 8 | VPC: 9 | Type: AWS::EC2::VPC 10 | Properties: 11 | CidrBlock: 10.0.0.0/20 12 | EnableDnsSupport: True 13 | EnableDnsHostnames: True 14 | InstanceTenancy: default 15 | 16 | InternetGateway: 17 | Type: AWS::EC2::InternetGateway 18 | 19 | GatewayAttachment: 20 | Type: AWS::EC2::VPCGatewayAttachment 21 | Properties: 22 | VpcId: '#{VPC}' 23 | InternetGatewayId: '#{InternetGateway}' 24 | 25 | PrivateASubnet: 26 | Type: AWS::EC2::Subnet 27 | Properties: 28 | AvailabilityZone: 29 | 'Fn::Select': [0, 'Fn::GetAZs': ''] 30 | CidrBlock: 10.0.0.0/23 31 | VpcId: '#{VPC}' 32 | Tags: 33 | - Key: Name 34 | Value: 'Private Subnet A' 35 | - Key: Reach 36 | Value: private 37 | 38 | PrivateBSubnet: 39 | Type: AWS::EC2::Subnet 40 | Properties: 41 | AvailabilityZone: 42 | 'Fn::Select': [1, 'Fn::GetAZs': ''] 43 | CidrBlock: 10.0.2.0/23 44 | VpcId: '#{VPC}' 45 | Tags: 46 | - Key: Name 47 | Value: 'Private Subnet B' 48 | - Key: Reach 49 | Value: private 50 | 51 | PublicASubnet: 52 | Type: AWS::EC2::Subnet 53 | Properties: 54 | AvailabilityZone: 55 | 'Fn::Select': [0, 'Fn::GetAZs': ''] 56 | CidrBlock: 10.0.8.0/23 57 | VpcId: '#{VPC}' 58 | Tags: 59 | - Key: Name 60 | Value: 'Public Subnet A' 61 | - Key: Reach 62 | Value: public 63 | 64 | PublicBSubnet: 65 | Type: AWS::EC2::Subnet 66 | Properties: 67 | AvailabilityZone: 68 | 'Fn::Select': [1, 'Fn::GetAZs': ''] 69 | CidrBlock: 10.0.10.0/23 70 | VpcId: '#{VPC}' 71 | Tags: 72 | - Key: Name 73 | Value: 'Public Subnet B' 74 | - Key: Reach 75 | Value: public 76 | 77 | PublicARouteTable: 78 | Type: AWS::EC2::RouteTable 79 | Properties: 80 | VpcId: '#{VPC}' 81 | Tags: 82 | - Key: Name 83 | Value: 'Public A' 84 | 85 | PublicBRouteTable: 86 | Type: AWS::EC2::RouteTable 87 | Properties: 88 | VpcId: '#{VPC}' 89 | Tags: 90 | - Key: Name 91 | Value: 'Public B' 92 | 93 | PrivateARouteTable: 94 | Type: AWS::EC2::RouteTable 95 | Properties: 96 | VpcId: '#{VPC}' 97 | Tags: 98 | - Key: Name 99 | Value: 'Private A' 100 | 101 | PrivateBRouteTable: 102 | Type: AWS::EC2::RouteTable 103 | Properties: 104 | VpcId: '#{VPC}' 105 | Tags: 106 | - Key: Name 107 | Value: 'Private B' 108 | 109 | PrivateASubnetRouteTableAssociation: 110 | Type: AWS::EC2::SubnetRouteTableAssociation 111 | Properties: 112 | RouteTableId: '#{PrivateARouteTable}' 113 | SubnetId: '#{PrivateASubnet}' 114 | 115 | PrivateBSubnetRouteTableAssociation: 116 | Type: AWS::EC2::SubnetRouteTableAssociation 117 | Properties: 118 | RouteTableId: '#{PrivateBRouteTable}' 119 | SubnetId: '#{PrivateBSubnet}' 120 | 121 | PublicASubnetRouteTableAssociation: 122 | Type: AWS::EC2::SubnetRouteTableAssociation 123 | Properties: 124 | RouteTableId: '#{PublicARouteTable}' 125 | SubnetId: '#{PublicASubnet}' 126 | 127 | PublicBSubnetRouteTableAssociation: 128 | Type: AWS::EC2::SubnetRouteTableAssociation 129 | Properties: 130 | RouteTableId: '#{PublicBRouteTable}' 131 | SubnetId: '#{PublicBSubnet}' 132 | 133 | PublicAInternetRoute: 134 | Type: AWS::EC2::Route 135 | DependsOn: 136 | - GatewayAttachment 137 | Properties: 138 | DestinationCidrBlock: 0.0.0.0/0 139 | GatewayId: '#{InternetGateway}' 140 | RouteTableId: '#{PublicARouteTable}' 141 | 142 | PublicBInternetRoute: 143 | Type: AWS::EC2::Route 144 | DependsOn: 145 | - GatewayAttachment 146 | Properties: 147 | DestinationCidrBlock: 0.0.0.0/0 148 | GatewayId: '#{InternetGateway}' 149 | RouteTableId: '#{PublicBRouteTable}' 150 | 151 | NetworkAclPublic: 152 | Type: 'AWS::EC2::NetworkAcl' 153 | Properties: 154 | VpcId: '#{VPC}' 155 | Tags: 156 | - Key: Name 157 | Value: Public 158 | 159 | NetworkAclPrivate: 160 | Type: 'AWS::EC2::NetworkAcl' 161 | Properties: 162 | VpcId: '#{VPC}' 163 | Tags: 164 | - Key: Name 165 | Value: Private 166 | 167 | PrivateASubnetNetworkAclAssociation: 168 | Type: AWS::EC2::SubnetNetworkAclAssociation 169 | Properties: 170 | NetworkAclId: '#{NetworkAclPrivate}' 171 | SubnetId: '#{PrivateASubnet}' 172 | 173 | PrivateBSubnetNetworkAclAssociation: 174 | Type: AWS::EC2::SubnetNetworkAclAssociation 175 | Properties: 176 | NetworkAclId: '#{NetworkAclPrivate}' 177 | SubnetId: '#{PrivateBSubnet}' 178 | 179 | PublicASubnetNetworkAclAssociation: 180 | Type: AWS::EC2::SubnetNetworkAclAssociation 181 | Properties: 182 | NetworkAclId: '#{NetworkAclPublic}' 183 | SubnetId: '#{PublicASubnet}' 184 | 185 | PublicBSubnetNetworkAclAssociation: 186 | Type: AWS::EC2::SubnetNetworkAclAssociation 187 | Properties: 188 | NetworkAclId: '#{NetworkAclPublic}' 189 | SubnetId: '#{PublicBSubnet}' 190 | 191 | NetworkAclEntryInPublicAllowAll: 192 | Type: 'AWS::EC2::NetworkAclEntry' 193 | Properties: 194 | NetworkAclId: !Ref NetworkAclPublic 195 | RuleNumber: 99 196 | Protocol: -1 197 | RuleAction: allow 198 | Egress: false 199 | CidrBlock: '0.0.0.0/0' 200 | 201 | NetworkAclEntryOutPublicAllowAll: 202 | Type: 'AWS::EC2::NetworkAclEntry' 203 | Properties: 204 | NetworkAclId: !Ref NetworkAclPublic 205 | RuleNumber: 99 206 | Protocol: -1 207 | RuleAction: allow 208 | Egress: true 209 | CidrBlock: '0.0.0.0/0' 210 | 211 | NetworkAclEntryInPrivateAllowAll: 212 | Type: 'AWS::EC2::NetworkAclEntry' 213 | Properties: 214 | NetworkAclId: !Ref NetworkAclPrivate 215 | RuleNumber: 99 216 | Protocol: -1 217 | RuleAction: allow 218 | Egress: false 219 | CidrBlock: '0.0.0.0/0' 220 | 221 | NetworkAclEntryOutPrivateAllowAll: 222 | Type: 'AWS::EC2::NetworkAclEntry' 223 | Properties: 224 | NetworkAclId: !Ref NetworkAclPrivate 225 | RuleNumber: 99 226 | Protocol: -1 227 | RuleAction: allow 228 | Egress: true 229 | CidrBlock: '0.0.0.0/0' 230 | 231 | SNSVPCEndpoint: 232 | Type: AWS::EC2::VPCEndpoint 233 | Properties: 234 | PrivateDnsEnabled: True 235 | SecurityGroupIds: 236 | - '#{VpcEndpointSecurityGroup.GroupId}' 237 | ServiceName: 'com.amazonaws.${self:provider.region}.sns' 238 | SubnetIds: 239 | - '#{PrivateASubnet}' 240 | - '#{PrivateBSubnet}' 241 | VpcEndpointType: Interface 242 | VpcId: '#{VPC}' 243 | 244 | AthenaVPCEndpoint: 245 | Type: AWS::EC2::VPCEndpoint 246 | Properties: 247 | PrivateDnsEnabled: True 248 | SecurityGroupIds: 249 | - '#{VpcEndpointSecurityGroup.GroupId}' 250 | ServiceName: 'com.amazonaws.${self:provider.region}.athena' 251 | SubnetIds: 252 | - '#{PrivateASubnet}' 253 | - '#{PrivateBSubnet}' 254 | VpcEndpointType: Interface 255 | VpcId: '#{VPC}' 256 | 257 | # See https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-ec2-vpcendpoint.html 258 | S3VPCEndpoint: 259 | Type: AWS::EC2::VPCEndpoint 260 | Properties: 261 | ServiceName: 'com.amazonaws.${self:provider.region}.s3' 262 | RouteTableIds: 263 | - '#{PrivateARouteTable}' 264 | - '#{PrivateBRouteTable}' 265 | VpcEndpointType: Gateway 266 | VpcId: '#{VPC}' 267 | 268 | DynamoDBVPCEndpoint: 269 | Type: AWS::EC2::VPCEndpoint 270 | Properties: 271 | ServiceName: 'com.amazonaws.${self:provider.region}.dynamodb' 272 | RouteTableIds: 273 | - '#{PrivateARouteTable}' 274 | - '#{PrivateBRouteTable}' 275 | VpcEndpointType: Gateway 276 | VpcId: '#{VPC}' 277 | 278 | VpcEndpointSecurityGroup: 279 | Type: 'AWS::EC2::SecurityGroup' 280 | Properties: 281 | VpcId: '#{VPC}' 282 | GroupDescription: 'Security group for VPC Endpoints' 283 | SecurityGroupIngress: 284 | - IpProtocol: tcp 285 | FromPort: 443 286 | ToPort: 443 287 | SourceSecurityGroupId: '#{VpcEndpointLambdaSecurityGroup.GroupId}' 288 | 289 | VpcEndpointLambdaSecurityGroup: 290 | Type: 'AWS::EC2::SecurityGroup' 291 | Properties: 292 | VpcId: '#{VPC}' 293 | GroupDescription: 'Security group for VPC Endpoint Lambda' 294 | -------------------------------------------------------------------------------- /schema/MyCube.js: -------------------------------------------------------------------------------- 1 | cube(`MyCube`, { 2 | sql: `SELECT 'helloworld' AS test`, 3 | 4 | measures: { 5 | count: { 6 | type: `count` 7 | }, 8 | }, 9 | 10 | dimensions: { 11 | test: { 12 | sql: `test`, 13 | type: `string` 14 | }, 15 | } 16 | }); 17 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: cubejs-service 2 | 3 | plugins: 4 | - serverless-pseudo-parameters 5 | - serverless-iam-roles-per-function 6 | - serverless-express 7 | 8 | custom: 9 | 10 | # SNS 11 | sns: 12 | topicName: ${self:service.name}-${self:provider.stage}-process 13 | 14 | # S3 buckets 15 | s3: 16 | buckets: 17 | athenaQueryResults: ${self:service.name}-athena-query-results-${self:provider.stage} 18 | 19 | # ElastiCache 20 | elasticache: 21 | nodeType: 'cache.t2.micro' 22 | 23 | # Aurora 24 | aurora: 25 | # Cluster configuration 26 | cluster: 27 | # Name of the Aurora Serverless cluster 28 | name: aurora-cluster-cubejs-${self:provider.stage} 29 | # Shall the cluster be protected from deletion 30 | deletionProtection: false 31 | # Database configuration 32 | database: 33 | # Name of the database instance 34 | name: cubejs 35 | # Master User configuration 36 | masterUser: 37 | # Master user name 38 | name: master 39 | # Password (max. 41 characters) 40 | password: '${opt:aurora-password}' 41 | # Scaling configuration 42 | scaling: 43 | # Pausing configuration 44 | pause: 45 | # Shall the cluster be paused 46 | autoPause: true 47 | # After how many idle seconds (without active connection) 48 | secondsUntilAutoPause: 600 49 | # Capacity 50 | capacity: 51 | # Minimal instances 52 | min: 1 53 | # Maximal instances 54 | max: 1 55 | # Backup configuration 56 | backup: 57 | # Backup retention 58 | retentionInDays: 30 59 | 60 | provider: 61 | name: aws 62 | runtime: nodejs12.x 63 | stage: ${opt:stage, 'dev'} 64 | region: ${opt:region, 'us-east-1'} 65 | logRetentionInDays: 14 66 | httpApi: 67 | # Breaking change with Serverless 1.75.0 68 | # See https://github.com/serverless/serverless/releases/tag/v1.75.0 69 | payload: '1.0' 70 | logs: 71 | httpApi: true 72 | # See https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-logging-variables.html 73 | # Can change to custom format like: 74 | # format: '{ "ip": "$context.identity.sourceIp", "requestTime":"$context.requestTime" }' 75 | vpc: 76 | securityGroupIds: 77 | # Account-level security group for being able to send SNS messages and Athena access 78 | - '#{VpcEndpointLambdaSecurityGroup.GroupId}' 79 | # Lambda security group 80 | - '#{LambdaSecurityGroup.GroupId}' 81 | subnetIds: 82 | - '#{PrivateASubnet}' 83 | - '#{PrivateBSubnet}' 84 | iamRoleStatements: 85 | - Effect: Allow 86 | Action: 87 | - sns:* 88 | Resource: arn:aws:sns:#{AWS::Region}:#{AWS::AccountId}:${self:custom.sns.topicName} 89 | - Effect: Allow 90 | Action: 91 | # TODO: Improve 92 | - athena:* 93 | Resource: '*' 94 | - Effect: Allow 95 | Action: 96 | - s3:ListBucket 97 | - s3:GetBucketLocation 98 | Resource: 'arn:aws:s3:::${self:custom.s3.buckets.athenaQueryResults}' 99 | - Effect: Allow 100 | Action: 101 | - s3:GetBucketLocation 102 | - s3:GetObject 103 | - s3:ListBucket 104 | - s3:ListBucketMultipartUploads 105 | - s3:ListMultipartUploadParts 106 | - s3:AbortMultipartUpload 107 | - s3:PutObject 108 | Resource: 'arn:aws:s3:::${self:custom.s3.buckets.athenaQueryResults}/*' 109 | - Effect: Allow 110 | Action: 111 | - s3:GetObject 112 | Resource: 'arn:aws:s3:::${opt:s3-data-bucket}/*' 113 | - Effect: Allow 114 | Action: 115 | - s3:ListBucket 116 | - s3:GetBucketLocation 117 | Resource: 'arn:aws:s3:::${opt:s3-data-bucket}' 118 | - Effect: Allow 119 | Action: 120 | # TODO: Improve 121 | - glue:* 122 | Resource: '*' 123 | - Effect: Allow 124 | Action: 125 | - ec2:CreateNetworkInterface 126 | - ec2:DescribeNetworkInterfaces 127 | - ec2:DeleteNetworkInterface 128 | Resource: '*' 129 | environment: 130 | AWS_NODEJS_CONNECTION_REUSE_ENABLED: '1' # Enable HTTP keep-alive connections for the AWS SDK 131 | AWS_ACCOUNT_ID: '#{AWS::AccountId}' 132 | CUBEJS_AWS_S3_OUTPUT_LOCATION: 's3://#{AthenaQueryResultsBucket}/' 133 | CUBEJS_JDBC_DRIVER: athena 134 | CUBEJS_DB_TYPE: athena 135 | CUBEJS_API_SECRET: '${opt:cube-secret}' 136 | CUBEJS_APP: '${self:service.name}-${self:provider.stage}' 137 | CUBEJS_LOG_LEVEL: trace 138 | CUBEJS_EXT_DB_HOST: '#{AuroraCluster.Endpoint.Address}' 139 | CUBEJS_EXT_DB_PORT: '#{AuroraCluster.Endpoint.Port}' 140 | CUBEJS_EXT_DB_NAME: '${self:custom.aurora.database.name}' 141 | CUBEJS_EXT_DB_USER: '${self:custom.aurora.masterUser.name}' 142 | CUBEJS_EXT_DB_PASS: '${self:custom.aurora.masterUser.password}' 143 | REDIS_URL: 'redis://#{ElastiCacheCluster.RedisEndpoint.Address}:6379' 144 | NODE_ENV: production 145 | STAGE: '${self:provider.stage}' 146 | 147 | functions: 148 | cubejs: 149 | handler: serverlessCube.api 150 | memorySize: 256 151 | timeout: 27 # Maximum is 29 on a API Gateway HTTP API 152 | events: 153 | - httpApi: '*' 154 | cubejsProcess: 155 | handler: serverlessCube.process 156 | memorySize: 256 157 | timeout: 630 158 | events: 159 | - sns: '${self:custom.sns.topicName}' 160 | 161 | resources: 162 | 163 | - ${file(resources/elasticache.yml)} 164 | - ${file(resources/aurora-cluster.yml)} 165 | - ${file(resources/security-groups.yml)} 166 | - ${file(resources/ssm-parameters.yml)} 167 | - ${file(resources/s3.yml)} 168 | - ${file(resources/vpc.yml)} 169 | -------------------------------------------------------------------------------- /serverlessCube.js: -------------------------------------------------------------------------------- 1 | const HandlerClass = require('@cubejs-backend/serverless-aws'); 2 | const MySQLDriver = require('@cubejs-backend/mysql-driver'); 3 | 4 | module.exports = new HandlerClass({ 5 | // See 6 | // * https://github.com/cube-js/cube.js/blob/master/packages/cubejs-server-core/core/index.js#L190 7 | // * https://github.com/cube-js/cube.js/blob/d29a483606af5fc4abfd87213b6f148db990212c/examples/web-analytics/index.js#L18 8 | externalDbType: 'mysql', 9 | externalDriverFactory: () => new MySQLDriver({ 10 | host: process.env.CUBEJS_EXT_DB_HOST, 11 | database: process.env.CUBEJS_EXT_DB_NAME, 12 | port: process.env.CUBEJS_EXT_DB_PORT, 13 | user: process.env.CUBEJS_EXT_DB_USER, 14 | password: process.env.CUBEJS_EXT_DB_PASS, 15 | // See https://stackoverflow.com/a/56524162/1603357 16 | ssl: true, 17 | }), 18 | preAggregationsSchema: `pre_aggregations_${process.env.STAGE}`, 19 | // Don't send telemetry 20 | telemetry: false, 21 | // Set base path to hide it's Cube.js 22 | basePath: '/api', 23 | }); 24 | --------------------------------------------------------------------------------