├── lib ├── stacks │ ├── garnet-common │ │ ├── utils │ │ │ ├── lambda │ │ │ │ ├── cleanLogs │ │ │ │ │ └── index.js │ │ │ │ ├── scorpioSqs │ │ │ │ │ └── index.js │ │ │ │ ├── cleanTasks │ │ │ │ │ └── index.js │ │ │ │ └── getAzs │ │ │ │ │ └── index.js │ │ │ └── utils-construct.ts │ │ ├── networking │ │ │ └── networking-construct.ts │ │ ├── secret │ │ │ └── secret-construct.ts │ │ └── garnet-common-stack.ts │ ├── garnet-api │ │ ├── apiauth │ │ │ ├── lambda │ │ │ │ ├── apiAuthJwt │ │ │ │ │ └── index.js │ │ │ │ └── apiAuthorizer │ │ │ │ │ └── index.js │ │ │ └── api-auth-construct.ts │ │ ├── garnet-api-stack.ts │ │ ├── apicommon │ │ │ ├── lambda │ │ │ │ └── garnetVersion │ │ │ │ │ └── index.js │ │ │ └── api-common-construct.ts │ │ └── apigateway │ │ │ └── api-gateway-construct.ts │ ├── garnet-privatesub │ │ ├── lambda │ │ │ ├── sqsCheck │ │ │ │ └── index.js │ │ │ ├── garnetSub │ │ │ │ └── index.js │ │ │ ├── sqsCreate │ │ │ │ └── index.js │ │ │ └── garnetSubSqs │ │ │ │ └── index.js │ │ └── garnet-privatesub-stack.ts │ ├── garnet-lake │ │ ├── bucket │ │ │ ├── lambda │ │ │ │ ├── bucketCheck │ │ │ │ │ └── index.js │ │ │ │ └── bucketCreate │ │ │ │ │ └── index.js │ │ │ └── bucket-construct.ts │ │ ├── stream │ │ │ ├── lambda │ │ │ │ └── transform │ │ │ │ │ └── index.js │ │ │ └── firehose-stream-construct.ts │ │ ├── garnet-lake-stack.ts │ │ └── athena │ │ │ ├── lambda │ │ │ └── athena │ │ │ │ └── index.js │ │ │ └── athena-construct.ts │ ├── garnet-ingestion │ │ ├── lambda │ │ │ └── updateContextBroker │ │ │ │ └── index.js │ │ └── garnet-ingestion-stack.ts │ ├── garnet-iot │ │ ├── iot-group │ │ │ ├── lambda │ │ │ │ ├── groupMembership │ │ │ │ │ └── index.js │ │ │ │ └── groupLifecycle │ │ │ │ │ └── index.js │ │ │ └── iot-group-construct.ts │ │ ├── iot-thing │ │ │ ├── lambda │ │ │ │ ├── thingLifecycle │ │ │ │ │ └── index.js │ │ │ │ └── presence │ │ │ │ │ └── index.js │ │ │ └── iot-thing-construct.ts │ │ └── garnet-iot-stack.ts │ ├── garnet-scorpio │ │ ├── garnet-scorpio-stack.ts │ │ └── database │ │ │ └── database-construct.ts │ └── garnet-ops │ │ └── garnet-ops-stack.ts ├── layers │ └── nodejs │ │ ├── package.json │ │ ├── utils.js │ │ └── package-lock.json └── garnet-stack.ts ├── .gitignore ├── jest.config.js ├── CODE_OF_CONDUCT.md ├── bin └── garnet.ts ├── package.json ├── tsconfig.json ├── configuration.ts ├── LICENSE ├── README.md ├── install.js ├── cdk.json ├── CONTRIBUTING.md ├── architecture.ts ├── CHANGELOG.md └── constants.ts /lib/stacks/garnet-common/utils/lambda/cleanLogs/index.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | !jest.config.js 2 | *.d.ts 3 | node_modules 4 | 5 | # CDK asset staging directory 6 | .cdk.staging 7 | cdk.out 8 | 9 | .DS_Store 10 | 11 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | testEnvironment: 'node', 3 | roots: ['/test'], 4 | testMatch: ['**/*.test.ts'], 5 | transform: { 6 | '^.+\\.tsx?$': 'ts-jest' 7 | } 8 | }; 9 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | ## Code of Conduct 2 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 3 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 4 | opensource-codeofconduct@amazon.com with any additional questions or comments. 5 | -------------------------------------------------------------------------------- /lib/layers/nodejs/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "keywords": [], 10 | "author": "", 11 | "license": "ISC", 12 | "dependencies": { 13 | "axios": "^1.12.2", 14 | "jsonwebtoken": "^9.0.2" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /bin/garnet.ts: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import { App } from 'aws-cdk-lib'; 3 | import { GarnetStack } from '../lib/garnet-stack'; 4 | import { Parameters } from '../configuration'; 5 | 6 | const app = new App(); 7 | 8 | new GarnetStack(app, 'Garnet', { 9 | stackName: 'Garnet', 10 | description: 'Garnet Framework is an open-source framework for building scalable, reliable and interoperable solutions and platforms - (uksb-1tupboc26)', 11 | env: { region: Parameters.aws_region } 12 | }) -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "garnet", 3 | "version": "1.5.1", 4 | "bin": { 5 | "garnet": "bin/garnet.js" 6 | }, 7 | "scripts": { 8 | "build": "tsc", 9 | "watch": "tsc -w", 10 | "test": "jest", 11 | "cdk": "cdk", 12 | "preinstall": "node install.js" 13 | }, 14 | "devDependencies": { 15 | "@types/jest": "^30.0.0", 16 | "@types/node": "24.5.0", 17 | "aws-cdk": "^2.1029.1", 18 | "constructs": "^10.4.2", 19 | "jest": "^30.1.3", 20 | "source-map-support": "^0.5.21", 21 | "ts-jest": "^29.4.2", 22 | "ts-node": "^10.9.2", 23 | "typescript": "~5.9.2" 24 | }, 25 | "dependencies": { 26 | "aws-cdk-lib": "2.215.0", 27 | "constructs": "^10.4.2", 28 | "source-map-support": "^0.5.21" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /lib/stacks/garnet-common/utils/lambda/scorpioSqs/index.js: -------------------------------------------------------------------------------- 1 | const { SQSClient, DeleteQueueCommand } = require("@aws-sdk/client-sqs") 2 | const sqs = new SQSClient({}) 3 | const SQS_QUEUES = JSON.parse(process.env.SQS_QUEUES) 4 | 5 | exports.handler = async (event) => { 6 | console.log(event) 7 | let request_type = event['RequestType'].toLowerCase() 8 | if (request_type=='delete') { 9 | 10 | try { 11 | for await (let queue of Object.values(SQS_QUEUES)){ 12 | await sqs.send( 13 | new DeleteQueueCommand({ 14 | QueueUrl: queue 15 | }) 16 | ) 17 | 18 | } 19 | 20 | } catch (e) { 21 | console.log(e) 22 | } 23 | return true 24 | } 25 | } -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES2020", 4 | "module": "commonjs", 5 | "lib": [ 6 | "es2020", 7 | "ES2021.String", 8 | "dom" 9 | ], 10 | "declaration": true, 11 | "strict": true, 12 | "noImplicitAny": true, 13 | "strictNullChecks": true, 14 | "noImplicitThis": true, 15 | "alwaysStrict": true, 16 | "noUnusedLocals": false, 17 | "noUnusedParameters": false, 18 | "noImplicitReturns": true, 19 | "noFallthroughCasesInSwitch": false, 20 | "inlineSourceMap": true, 21 | "inlineSources": true, 22 | "experimentalDecorators": true, 23 | "strictPropertyInitialization": false, 24 | "typeRoots": [ 25 | "./node_modules/@types" 26 | ] 27 | }, 28 | "exclude": [ 29 | "node_modules", 30 | "cdk.out" 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /configuration.ts: -------------------------------------------------------------------------------- 1 | import { ARCHITECTURE } from "./architecture" 2 | 3 | // GARNET PARAMETERS 4 | export const Parameters = { 5 | /** 6 | * See regions in which you can deploy Garnet: 7 | * https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-vpc-links.html#http-api-vpc-link-availability 8 | */ 9 | aws_region: "us-east-1", 10 | 11 | /** 12 | * Choose between Concentrated (single container) or Distributed (microservices) architecture. 13 | * You can fine-tune the deployment parameters in architecture.ts 14 | * - Concentrated: All services in one container, suitable for development and testing 15 | * - Distributed: 8 specialized microservices, recommended for production deployments 16 | */ 17 | architecture: ARCHITECTURE.Concentrated, 18 | 19 | 20 | // API Authorization 21 | authorization: true 22 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apiauth/lambda/apiAuthJwt/index.js: -------------------------------------------------------------------------------- 1 | const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager") 2 | const jwt = require('jsonwebtoken') 3 | 4 | exports.handler = async (event, context) => { 5 | 6 | try { 7 | const client = new SecretsManagerClient({}); 8 | const command = new GetSecretValueCommand({ 9 | SecretId: process.env.SECRET_ARN 10 | }) 11 | 12 | const response = await client.send(command); 13 | const secret = response.SecretString; 14 | 15 | const token = jwt.sign({ 16 | sub: process.env.JWT_SUB, 17 | iss: process.env.JWT_ISS, 18 | aud: process.env.JWT_AUD 19 | }, secret); 20 | 21 | return { 22 | Status: 'SUCCESS', 23 | Data: { 24 | token: token 25 | } 26 | }; 27 | } 28 | catch (e) { 29 | console.log(e) 30 | return { 31 | Status: 'FAILED', 32 | Reason: error.message 33 | } 34 | } 35 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT No Attribution 2 | 3 | Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so. 10 | 11 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 12 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 13 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 14 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 15 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 16 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 17 | 18 | -------------------------------------------------------------------------------- /lib/stacks/garnet-privatesub/lambda/sqsCheck/index.js: -------------------------------------------------------------------------------- 1 | const { SQSClient, GetQueueUrlCommand } = require("@aws-sdk/client-sqs"); 2 | const sqs = new SQSClient() 3 | const QUEUE_NAME = process.env.QUEUE_NAME 4 | 5 | 6 | const checkQueueExists = async (queueName) => { 7 | try { 8 | await sqs.send(new GetQueueUrlCommand({ QueueName: queueName })) 9 | return true 10 | } catch (error) { 11 | if (error.name === 'QueueDoesNotExist') { 12 | return false 13 | } 14 | throw error 15 | } 16 | }; 17 | 18 | exports.handler = async (event) => { 19 | console.log('IsComplete Check Event:', JSON.stringify(event, null, 2)) 20 | 21 | const requestType = event.RequestType.toLowerCase() 22 | 23 | if (requestType === 'delete') { 24 | return { IsComplete: true } 25 | } 26 | 27 | try { 28 | const queueExists = await checkQueueExists(QUEUE_NAME) 29 | return { 30 | IsComplete: queueExists 31 | } 32 | } catch (error) { 33 | console.error('Error in isComplete handler:', error); 34 | throw error 35 | } 36 | }; 37 | -------------------------------------------------------------------------------- /lib/stacks/garnet-common/utils/lambda/cleanTasks/index.js: -------------------------------------------------------------------------------- 1 | const { ECSClient, ListTaskDefinitionsCommand, DeleteTaskDefinitionsCommand } = require("@aws-sdk/client-ecs") 2 | const ecs = new ECSClient({}) 3 | 4 | 5 | exports.handler = async (event) => { 6 | console.log(event) 7 | let request_type = event['RequestType'].toLowerCase() 8 | if (request_type=='delete') { 9 | 10 | try { 11 | 12 | const {taskDefinitionArns} = await ecs.send( 13 | new ListTaskDefinitionsCommand({ 14 | status: "INACTIVE" 15 | }) 16 | ) 17 | const inactive_garnet_tasks = chunk(taskDefinitionArns.filter((task) => task.includes('Garnet')), 10) 18 | for await (let task of inactive_garnet_tasks) { 19 | await ecs.send( 20 | new DeleteTaskDefinitionsCommand({ 21 | taskDefinitions: task 22 | }) 23 | ) 24 | 25 | } 26 | } catch (e) { 27 | console.log(e) 28 | } 29 | return true 30 | } 31 | } 32 | 33 | const chunk = (arr, size) => Array.from({ length: Math.ceil(arr.length / size) }, (v, i) => arr.slice(i * size, i * size + size)) 34 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/bucket/lambda/bucketCheck/index.js: -------------------------------------------------------------------------------- 1 | const { S3Client, HeadBucketCommand } = require("@aws-sdk/client-s3"); 2 | const s3 = new S3Client() 3 | const BUCKET_NAME = process.env.BUCKET_NAME 4 | const BUCKET_ATHENA_NAME = process.env.BUCKET_ATHENA_NAME 5 | 6 | const checkBucketExists = async (bucketName) => { 7 | try { 8 | await s3.send(new HeadBucketCommand({ Bucket: bucketName })) 9 | return true 10 | } catch (error) { 11 | if (error.$metadata && error.$metadata.httpStatusCode === 404) { 12 | return false 13 | } 14 | throw error 15 | } 16 | }; 17 | 18 | exports.handler = async (event) => { 19 | console.log('IsComplete Check Event:', JSON.stringify(event, null, 2)) 20 | 21 | const requestType = event.RequestType.toLowerCase() 22 | 23 | if (requestType === 'delete') { 24 | return { IsComplete: true } 25 | } 26 | 27 | try { 28 | const mainBucketExists = await checkBucketExists(BUCKET_NAME) 29 | const athenaBucketExists = await checkBucketExists(BUCKET_ATHENA_NAME) 30 | return { 31 | IsComplete: mainBucketExists && athenaBucketExists 32 | } 33 | } catch (error) { 34 | console.error('Error in isComplete handler:', error); 35 | throw error 36 | } 37 | }; -------------------------------------------------------------------------------- /lib/stacks/garnet-common/networking/networking-construct.ts: -------------------------------------------------------------------------------- 1 | import { SubnetType, Vpc } from "aws-cdk-lib/aws-ec2" 2 | import { Construct } from "constructs" 3 | import { garnet_broker } from "../../../../constants" 4 | 5 | 6 | export interface GarnetNetworkingProps { 7 | az1: string, 8 | az2: string 9 | } 10 | 11 | export class GarnetNetworking extends Construct { 12 | public readonly vpc: Vpc 13 | constructor(scope: Construct, id: string, props: GarnetNetworkingProps) { 14 | super(scope, id) 15 | 16 | let broker_id = garnet_broker 17 | 18 | // VPC 19 | const vpc = new Vpc(this, `VpcGarnet${broker_id}`, { 20 | natGateways: 1, 21 | availabilityZones: [`${props.az1}`,`${props.az2}`], 22 | vpcName: `garnet-vpc-${broker_id.toLowerCase()}`, 23 | subnetConfiguration: [ 24 | { 25 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 26 | name: `garnet-subnet-egress-${broker_id.toLowerCase()}`, 27 | }, 28 | { 29 | subnetType: SubnetType.PRIVATE_ISOLATED, 30 | name: `garnet-subnet-isolated-${broker_id.toLowerCase()}`, 31 | }, 32 | { 33 | subnetType: SubnetType.PUBLIC, 34 | name: `garnet-subnet-public-${broker_id.toLowerCase()}`, 35 | } 36 | ] 37 | }) 38 | 39 | this.vpc = vpc; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /lib/stacks/garnet-common/utils/lambda/getAzs/index.js: -------------------------------------------------------------------------------- 1 | const { EC2Client, DescribeAvailabilityZonesCommand, DescribeVpcEndpointServicesCommand } = require("@aws-sdk/client-ec2") 2 | const ec2 = new EC2Client({apiVersion: '2016-11-15'}) 3 | const compatible_azs = JSON.parse(process.env.COMPATIBLE_AZS) 4 | let region = process.env.AWS_REGION 5 | 6 | exports.handler = async (event) => { 7 | console.log(event) 8 | let request_type = event['RequestType'].toLowerCase() 9 | if (request_type=='create' || request_type == 'update') { 10 | 11 | 12 | const {AvailabilityZones} = await ec2.send( 13 | new DescribeAvailabilityZonesCommand({}) 14 | ) 15 | 16 | const {ServiceDetails} = await ec2.send( 17 | new DescribeVpcEndpointServicesCommand({ 18 | ServiceNames: [`com.amazonaws.${region}.iot.data`] 19 | }) 20 | ) 21 | 22 | let vpc_link_az = AvailabilityZones.filter((az) => compatible_azs.includes(az.ZoneId)).map((az) => az.ZoneName) 23 | let vpc_endpoint_iot_az = ServiceDetails[0]["AvailabilityZones"] 24 | 25 | let final_azs = vpc_link_az.filter((arr) => vpc_endpoint_iot_az.indexOf(arr) !== -1) 26 | console.log({final_azs}) 27 | 28 | return { 29 | Data: { 30 | az1: final_azs[0], 31 | az2: final_azs[final_azs.length - 1] 32 | } 33 | } 34 | } 35 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/stream/lambda/transform/index.js: -------------------------------------------------------------------------------- 1 | const {get_type, log_error, recursive_concise, transform, compactKeys} = require('/opt/nodejs/utils.js') 2 | 3 | 4 | exports.handler = async (event, context) => { 5 | let output = [] 6 | 7 | event.records.forEach(record => { 8 | try { 9 | let payload = JSON.parse(Buffer.from(record.data, 'base64').toString('utf8')) 10 | 11 | 12 | if(!payload.type) { 13 | console.log(`Type must be present - Record ${record.recordId} dropped`) 14 | console.log(record) 15 | output.push({ 16 | recordId: record.recordId, 17 | result: 'Dropped', 18 | data: record.data 19 | }) 20 | 21 | } else { 22 | output.push({ 23 | recordId: record.recordId, 24 | result: 'Ok', 25 | data: Buffer.from(JSON.stringify(transform(compactKeys(payload))), 'utf-8').toString('base64'), 26 | metadata: {"partitionKeys": {"type": get_type(payload)}} 27 | }) 28 | } 29 | } catch (e) { 30 | console.log(e) 31 | output.push({ 32 | recordId: record.recordId, 33 | result: 'Dropped', 34 | data: record.data 35 | }) 36 | } 37 | }) 38 | 39 | return { records: output } 40 | } 41 | 42 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/garnet-lake-stack.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CustomResource, Duration,NestedStack, NestedStackProps, RemovalPolicy, Stack } from "aws-cdk-lib" 2 | 3 | 4 | 5 | 6 | import { Provider } from "aws-cdk-lib/custom-resources" 7 | import { garnet_bucket, garnet_constant, garnet_nomenclature } from "../../../constants" 8 | import { GarnetBucket } from "./bucket/bucket-construct" 9 | import { GarnetDataLakeAthena } from "./athena/athena-construct" 10 | import { GarnetDataLakeStream } from "./stream/firehose-stream-construct" 11 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose" 12 | 13 | 14 | 15 | export interface GarnetLakeProps extends NestedStackProps{ 16 | 17 | } 18 | 19 | 20 | export class GarnetLake extends NestedStack { 21 | public readonly delivery_stream: CfnDeliveryStream 22 | public readonly bucket_name: string 23 | 24 | constructor(scope: Stack, id: string, props: GarnetLakeProps) { 25 | super(scope, id) 26 | 27 | 28 | const bucket = new GarnetBucket(this, 'GarnetBucket', {}) 29 | 30 | 31 | 32 | const athena = new GarnetDataLakeAthena(this, 'LakeAthena', {}) 33 | const lake_stream = new GarnetDataLakeStream(this, 'LakeStream', { 34 | bucket_name: bucket.bucket_name 35 | }) 36 | 37 | lake_stream.node.addDependency(bucket) 38 | 39 | this.bucket_name = bucket.bucket_name 40 | this.delivery_stream = lake_stream.datalake_kinesis_firehose_delivery_stream 41 | } 42 | 43 | 44 | 45 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-common/secret/secret-construct.ts: -------------------------------------------------------------------------------- 1 | import { NestedStack, NestedStackProps } from "aws-cdk-lib" 2 | import { Secret } from "aws-cdk-lib/aws-secretsmanager" 3 | import { Construct } from "constructs" 4 | import { garnet_broker, garnet_nomenclature } from "../../../../constants" 5 | 6 | 7 | export interface GarnetSecretProps { 8 | } 9 | 10 | export class GarnetSecret extends Construct { 11 | public readonly secret: Secret 12 | public readonly secret_api_jwt : Secret 13 | 14 | constructor(scope: Construct, id: string, props: GarnetSecretProps) { 15 | super(scope, id) 16 | 17 | this.secret = new Secret(this, 'Secret', { 18 | secretName: garnet_nomenclature.garnet_secret, 19 | generateSecretString: { 20 | secretStringTemplate: JSON.stringify({ 21 | username: 'garnetadmin', 22 | }), 23 | excludePunctuation: true, 24 | excludeCharacters: "/¥'%:;{}", 25 | includeSpace: false, 26 | generateStringKey: 'password' 27 | } 28 | }) 29 | 30 | this.secret_api_jwt = new Secret(this, 'SecretApiJwt', { 31 | secretName: garnet_nomenclature.garnet_api_jwt_secret, 32 | generateSecretString: { 33 | excludePunctuation: true, 34 | excludeCharacters: "/¥'%:;{}", 35 | includeSpace: false, 36 | } 37 | }) 38 | 39 | } 40 | 41 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-common/garnet-common-stack.ts: -------------------------------------------------------------------------------- 1 | import { NestedStack, NestedStackProps } from "aws-cdk-lib"; 2 | import { Construct } from "constructs"; 3 | import { GarnetSecret } from "./secret/secret-construct"; 4 | import { GarnetNetworking } from "./networking/networking-construct"; 5 | import { Utils } from "./utils/utils-construct"; 6 | import { Vpc } from "aws-cdk-lib/aws-ec2"; 7 | import { Secret } from "aws-cdk-lib/aws-secretsmanager"; 8 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose"; 9 | 10 | 11 | export class GarnetCommon extends NestedStack { 12 | public readonly vpc: Vpc 13 | public readonly secret: Secret 14 | public readonly secret_api_jwt : Secret 15 | public readonly bucket_name: string 16 | public readonly az1: string 17 | public readonly az2: string 18 | 19 | constructor(scope: Construct, id: string, props?: NestedStackProps) { 20 | super(scope, id, props); 21 | 22 | const utils_construct = new Utils(this, "Utils") 23 | const secret_construct = new GarnetSecret(this, "Secret", {}) 24 | const networking_construct = new GarnetNetworking(this, "Networking", { 25 | az1: utils_construct.az1, 26 | az2: utils_construct.az2 27 | }) 28 | 29 | networking_construct.node.addDependency(utils_construct) 30 | 31 | 32 | this.az1 = utils_construct.az1, 33 | this.az2 = utils_construct.az2 34 | this.vpc = networking_construct.vpc 35 | this.secret = secret_construct.secret 36 | this.secret_api_jwt = secret_construct.secret_api_jwt 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/athena/lambda/athena/index.js: -------------------------------------------------------------------------------- 1 | const { AthenaClient, CreateWorkGroupCommand } = require("@aws-sdk/client-athena") 2 | const { GlueClient, CreateDatabaseCommand } = require("@aws-sdk/client-glue") 3 | const athena = new AthenaClient() 4 | const glue = new GlueClient() 5 | const BUCKET_NAME_ATHENA = process.env.BUCKET_NAME_ATHENA 6 | const CATALOG_ID = process.env.CATALOG_ID 7 | const GLUEDB_NAME = process.env.GLUEDB_NAME 8 | 9 | exports.handler = async (event) => { 10 | console.log(event) 11 | let request_type = event['RequestType'].toLowerCase() 12 | if (request_type=='create' || request_type == 'update') { 13 | 14 | try { 15 | await athena.send( 16 | new CreateWorkGroupCommand({ 17 | Name: 'garnet', 18 | Configuration: { 19 | ResultConfiguration: { 20 | OutputLocation: `s3://${BUCKET_NAME_ATHENA}` 21 | } 22 | } 23 | }) 24 | ) 25 | } catch (e) { 26 | console.log(e.message) 27 | } 28 | 29 | try { 30 | await glue.send( 31 | new CreateDatabaseCommand({ 32 | CatalogId: CATALOG_ID, 33 | DatabaseInput: { 34 | Name: GLUEDB_NAME 35 | } 36 | }) 37 | ) 38 | } catch (e) { 39 | console.log(e.message) 40 | } 41 | 42 | return true 43 | } 44 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Garnet Framework 2 | 3 | #### [Version 1.5.1](./CHANGELOG.md#151---2025-09-17) 4 | 5 | __Explore the [documentation website of Garnet Framework](https://garnet-framework.tech/docs) to get started.__ 6 | 7 | ### Overview 8 | 9 | The Garnet Framework is an open-source framework that enables you to build living digital twins and context-aware solutions through dynamic knowledge graphs leveraging open standards. 10 | 11 | Garnet Framework provides real-time context management, temporal data capabilities, geospatial queries, subscription-based notifications, and automated data lake integration for comprehensive analytics and AI-powered decision-making. 12 | By creating unified, continuously updating digital representations of your physical environments and processes, Garnet Framework delivers the contextual intelligence that powers smart decision-making across domains including Smart Cities, Energy, Manufacturing, Supply Chain, Agriculture, Buildings, and Transportation. 13 | 14 | Garnet Framework is built on the [NGSI-LD](https://ngsi-ld.org/) open standard and leverages the open-source NGSI-LD Context Broker technology. 15 | It is designed to be easily deployable on the AWS infrastructure using the [AWS Cloud Development Kit](https://aws.amazon.com/cdk/) (CDK), allowing for streamlined deployment and management. 16 | 17 | Through its knowledge graph capabilities, Garnet transforms fragmented data into interconnected knowledge that evolves in near real-time with your operations—providing the essential context your applications and AI systems need. 18 | 19 | ## Getting Started 20 | 21 | Explore the [documentation website of Garnet Framework](https://garnet-framework.tech/docs) to get started. 22 | 23 | ## Security 24 | 25 | See [CONTRIBUTING](CONTRIBUTING.md#security-issue-notifications) for more information. 26 | 27 | ## License 28 | 29 | This library is licensed under the MIT-0 License. See the LICENSE file. 30 | 31 | -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apiauth/lambda/apiAuthorizer/index.js: -------------------------------------------------------------------------------- 1 | const { SecretsManagerClient, GetSecretValueCommand } = require("@aws-sdk/client-secrets-manager") 2 | const jwt = require('jsonwebtoken') 3 | 4 | // Cache variable outside the handler to persist between invocations 5 | let cachedSecret = null; 6 | let lastFetchTime = null; 7 | const CACHE_TTL = 1000 * 60 * 60; // 1 hour in milliseconds 8 | 9 | exports.handler = async (event) => { 10 | try { 11 | // Check for token in query parameters 12 | const authParam = event.queryStringParameters?.token; 13 | 14 | // Fallback to Authorization header if not in query parameters 15 | const authHeader = event.headers?.authorization; 16 | 17 | if (!authParam && !authHeader) { 18 | console.log('No token provided'); 19 | return { isAuthorized: false }; 20 | } 21 | 22 | // Use token from query parameter if available, otherwise use from header 23 | const token = authParam || authHeader; 24 | 25 | 26 | // Get secret from cache or fetch new 27 | const currentTime = Date.now(); 28 | if (!cachedSecret || !lastFetchTime || (currentTime - lastFetchTime) > CACHE_TTL) { 29 | const client = new SecretsManagerClient({}) 30 | const command = new GetSecretValueCommand({ 31 | SecretId: process.env.SECRET_ARN 32 | }) 33 | 34 | const response = await client.send(command); 35 | cachedSecret = response.SecretString; 36 | lastFetchTime = currentTime 37 | } 38 | 39 | // Verify token 40 | const decoded = jwt.verify(token, cachedSecret, { 41 | issuer: process.env.JWT_ISS, 42 | audience: process.env.JWT_AUD 43 | }); 44 | 45 | console.log(decoded) 46 | return { 47 | isAuthorized: true, 48 | context: { 49 | sub: decoded.sub, 50 | iss: decoded.iss, 51 | aud: decoded.aud 52 | } 53 | }; 54 | 55 | } catch (error) { 56 | console.error('Authorization error:', error); 57 | return { isAuthorized: false }; 58 | } 59 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-ingestion/lambda/updateContextBroker/index.js: -------------------------------------------------------------------------------- 1 | const dns_broker = `http://${process.env.DNS_CONTEXT_BROKER}/ngsi-ld/v1` 2 | const axios = require('axios') 3 | const {get_type, log_error, normalize} = require('/opt/nodejs/utils.js') 4 | 5 | exports.handler = async (event, context) => { 6 | 7 | try { 8 | 9 | let entities_with_context = [] 10 | let entities_without_context = [] 11 | 12 | for await (let msg of event.Records){ 13 | let payload = JSON.parse(msg.body) 14 | if(!payload.id || !payload.type){ 15 | throw new Error('Invalid entity: id or type is missing') 16 | } 17 | payload = normalize(payload) 18 | 19 | if(payload["@context"]){ 20 | entities_with_context = entities_with_context.concat(payload) 21 | } else { 22 | entities_without_context = entities_without_context.concat(payload) 23 | } 24 | 25 | } 26 | 27 | try { 28 | if(entities_with_context.length > 0){ 29 | const headers= { 30 | 'Content-Type': 'application/ld+json' 31 | } 32 | let {data: res} = await axios.post(`${dns_broker}/entityOperations/upsert?options=update`, entities_with_context, {headers: headers}) 33 | console.log(res) 34 | } 35 | } catch (e) { 36 | log_error(event,context, e.message, e) 37 | } 38 | 39 | try { 40 | 41 | if(entities_without_context.length > 0){ 42 | const headers= { 43 | 'Content-Type': 'application/json' 44 | } 45 | let {data: res} = await axios.post(`${dns_broker}/entityOperations/upsert?options=update`, entities_without_context, {headers: headers}) 46 | console.log(res) 47 | } 48 | 49 | } catch (e) { 50 | log_error(event,context, e.message, e) 51 | } 52 | } catch (e) { 53 | log_error(event,context, e.message, e) 54 | } 55 | } 56 | 57 | 58 | -------------------------------------------------------------------------------- /install.js: -------------------------------------------------------------------------------- 1 | const path = require('path') 2 | const fs = require('fs') 3 | const child_process = require('child_process') 4 | 5 | const root = process.cwd() 6 | npm_install_recursive(root) 7 | 8 | // Since this script is intended to be run as a "preinstall" command, 9 | // it will do `npm install` automatically inside the root folder in the end. 10 | console.log('===================================================================') 11 | console.log(`Performing "npm install" inside root folder`) 12 | console.log('===================================================================') 13 | 14 | // Recurses into a folder 15 | function npm_install_recursive(folder) 16 | { 17 | const has_package_json = fs.existsSync(path.join(folder, 'package.json')) 18 | 19 | // If there is `package.json` in this folder then perform `npm install`. 20 | // 21 | // Since this script is intended to be run as a "preinstall" command, 22 | // skip the root folder, because it will be `npm install`ed in the end. 23 | // Hence the `folder !== root` condition. 24 | // 25 | if (has_package_json && folder !== root) 26 | { 27 | console.log('===================================================================') 28 | console.log(`Performing "npm install" inside ${folder === root ? 'root folder' : './' + path.relative(root, folder)}`) 29 | console.log('===================================================================') 30 | 31 | npm_install(folder) 32 | } 33 | 34 | // Recurse into subfolders 35 | for (let subfolder of subfolders(folder)) 36 | { 37 | npm_install_recursive(subfolder) 38 | } 39 | } 40 | 41 | // Performs `npm install` 42 | function npm_install(where) 43 | { 44 | child_process.execSync('npm install', { cwd: where, env: process.env, stdio: 'inherit' }) 45 | } 46 | 47 | // Lists subfolders in a folder 48 | function subfolders(folder) 49 | { 50 | return fs.readdirSync(folder) 51 | .filter(subfolder => fs.statSync(path.join(folder, subfolder)).isDirectory()) 52 | .filter(subfolder => subfolder !== 'node_modules' && subfolder[0] !== '.') 53 | .map(subfolder => path.join(folder, subfolder)) 54 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-group/lambda/groupMembership/index.js: -------------------------------------------------------------------------------- 1 | const dns_broker = `http://${process.env.DNS_CONTEXT_BROKER}/ngsi-ld/v1` 2 | const AWSIOTTHINGTYPE = process.env.AWSIOTTHINGTYPE 3 | const AWSIOTTHINGGROUPTYPE = process.env.AWSIOTTHINGGROUPTYPE 4 | const axios = require('axios') 5 | const {log_error, normalize} = require('/opt/nodejs/utils.js') 6 | 7 | const { IoTClient, ListThingGroupsForThingCommand } = require("@aws-sdk/client-iot") 8 | const iot = new IoTClient({}) 9 | const headers = { 10 | 'Content-Type': 'application/json' 11 | } 12 | 13 | exports.handler = async (event, context) => { 14 | try { 15 | console.log(JSON.stringify(event)) 16 | let {thingName} = event 17 | let tgs = [] 18 | let token = null 19 | 20 | while (token !== undefined) { 21 | 22 | let {thingGroups, nextToken} = await iot.send( 23 | new ListThingGroupsForThingCommand({ 24 | thingName, 25 | nextToken: token 26 | }) 27 | ) 28 | tgs = tgs.concat(thingGroups) 29 | token = nextToken 30 | 31 | } 32 | 33 | let payload = {} 34 | 35 | if(tgs.length > 0){ 36 | payload.thingGroups = { 37 | object: tgs.map(tg => `urn:ngsi-ld:${AWSIOTTHINGGROUPTYPE}:${tg.groupName}`), 38 | objectType: AWSIOTTHINGGROUPTYPE 39 | } 40 | } else { 41 | payload.thingGroups = 'urn:ngsi-ld:null' 42 | 43 | } 44 | 45 | try { 46 | 47 | let entity = { 48 | id: `urn:ngsi-ld:${AWSIOTTHINGTYPE}:${thingName}`, 49 | type: [`${AWSIOTTHINGTYPE}`], 50 | ...payload 51 | } 52 | entity = normalize(entity) 53 | try { 54 | 55 | let {data: res} = await axios.post(`${dns_broker}/entities/${entity.id}/attrs`, entity, {headers: headers}) 56 | 57 | console.log(res) 58 | } catch(e){ 59 | console.log(e) 60 | } 61 | 62 | 63 | } catch (e) { 64 | log_error(event,context, e.message, e) 65 | } 66 | 67 | } catch (e) { 68 | log_error(event,context, e.message, e) 69 | } 70 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-scorpio/garnet-scorpio-stack.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CfnOutput, NestedStack, NestedStackProps } from "aws-cdk-lib"; 2 | import { SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2"; 3 | import { Construct } from "constructs"; 4 | import { GarnetApiGateway } from "../garnet-api/apigateway/api-gateway-construct"; 5 | import { GarnetScorpioDatabase } from "./database/database-construct"; 6 | import { GarnetScorpioFargate } from "./fargate/container-construct"; 7 | 8 | import { Secret } from "aws-cdk-lib/aws-secretsmanager"; 9 | import { garnet_scorpio_images } from "../../../constants"; 10 | import { ApplicationLoadBalancer } from "aws-cdk-lib/aws-elasticloadbalancingv2"; 11 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose"; 12 | 13 | 14 | export interface GarnetScorpioProps extends NestedStackProps{ 15 | vpc: Vpc, 16 | secret: Secret, 17 | delivery_stream: CfnDeliveryStream 18 | } 19 | 20 | export class GarnetScorpio extends NestedStack { 21 | 22 | public readonly dns_context_broker: string; 23 | public readonly vpc: Vpc; 24 | public readonly fargate_alb: ApplicationLoadBalancer; 25 | 26 | public readonly sg_broker: SecurityGroup 27 | 28 | 29 | constructor(scope: Construct, id: string, props: GarnetScorpioProps) { 30 | super(scope, id, props); 31 | 32 | const database_construct = new GarnetScorpioDatabase(this, "Database", { 33 | vpc: props.vpc, 34 | secret_arn: props.secret.secretArn 35 | }) 36 | 37 | const fargate_construct = new GarnetScorpioFargate( this, "Fargate", { 38 | vpc: props.vpc, 39 | sg_proxy: database_construct.sg_proxy, 40 | secret_arn: props.secret.secretArn, 41 | db_endpoint: database_construct.database_endpoint, 42 | // db_reader_endpoint: database_construct.database_reader_endpoint, 43 | db_port: database_construct.database_port, 44 | image_context_broker: garnet_scorpio_images.allInOne, 45 | delivery_stream: props.delivery_stream 46 | } 47 | ) 48 | 49 | fargate_construct.node.addDependency(database_construct) 50 | 51 | this.fargate_alb = fargate_construct.fargate_alb 52 | this.dns_context_broker = fargate_construct.fargate_alb.loadBalancerDnsName 53 | this.vpc = props.vpc 54 | 55 | this.sg_broker = fargate_construct.sg_broker 56 | 57 | new CfnOutput(this, "fargate_alb", { 58 | value: fargate_construct.fargate_alb.loadBalancerDnsName, 59 | }) 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-thing/lambda/thingLifecycle/index.js: -------------------------------------------------------------------------------- 1 | const dns_broker = `http://${process.env.DNS_CONTEXT_BROKER}/ngsi-ld/v1` 2 | const AWSIOTTHINGTYPE = process.env.AWSIOTTHINGTYPE 3 | const axios = require('axios') 4 | const {log_error, normalize} = require('/opt/nodejs/utils.js') 5 | 6 | const headers = { 7 | 'Content-Type': 'application/json' 8 | } 9 | 10 | exports.handler = async (event, context) => { 11 | try { 12 | 13 | 14 | // Extract group information from the event 15 | const {operation, thingName, timestamp, thingId, versionNumber,attributes, eventId } = event 16 | 17 | if(['CREATED', 'UPDATED'].includes(operation)){ 18 | let thing = { 19 | id: `urn:ngsi-ld:${AWSIOTTHINGTYPE}:${thingName}`, 20 | type: `${AWSIOTTHINGTYPE}`, 21 | thingName: { 22 | value: thingName 23 | }, 24 | thingId: { 25 | value: thingId 26 | }, 27 | versionNumber: { 28 | value: versionNumber 29 | }, 30 | eventType: { 31 | type: "Property", 32 | value: operation, 33 | eventId: eventId, 34 | observedAt: (new Date(timestamp)).toISOString() 35 | } 36 | } 37 | 38 | if (attributes && typeof attributes === 'object') { 39 | Object.keys(attributes).forEach(key => { 40 | if (attributes[key] !== undefined && attributes[key] !== null) { 41 | thing[key] = { 42 | type: 'Property', 43 | value: attributes[key] 44 | } 45 | } 46 | }) 47 | } 48 | 49 | try { 50 | 51 | let {data: res} = await axios.post(`${dns_broker}/entityOperations/upsert?options=update `, [thing], {headers: headers}) 52 | console.log(res) 53 | 54 | } catch(e){ 55 | console.log(e) 56 | } 57 | 58 | 59 | } 60 | 61 | if(['DELETED'].includes(operation)){ 62 | let {data: res} = await axios.delete(`${dns_broker}/entities/urn:ngsi-ld:${AWSIOTTHINGTYPE}:${thing}`) 63 | console.log(res) 64 | } 65 | 66 | 67 | } catch (e) { 68 | log_error(event, context, e.message, e) 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-thing/lambda/presence/index.js: -------------------------------------------------------------------------------- 1 | const dns_broker = `http://${process.env.DNS_CONTEXT_BROKER}/ngsi-ld/v1` 2 | const AWSIOTTHINGTYPE = process.env.AWSIOTTHINGTYPE 3 | const axios = require('axios') 4 | const {log_error, normalize} = require('/opt/nodejs/utils.js') 5 | 6 | 7 | const headers = { 8 | 'Content-Type': 'application/json' 9 | } 10 | 11 | exports.handler = async (event, context) => { 12 | try { 13 | 14 | for await (let msg of event.Records){ 15 | let presence = JSON.parse(msg.body) 16 | const {clientId, timestamp, eventType, clientInitiatedDisconnect, sessionIdentifier, principalIdentifier, disconnectReason, ipAddress, versionNumber } = presence 17 | if (clientId.startsWith('iotconsole')) return 18 | let payload = { 19 | connectivityStatus: { 20 | value: eventType.toUpperCase(), 21 | sessionIdentifier, 22 | principalIdentifier, 23 | versionNumber, 24 | observedAt: (new Date(timestamp)).toISOString(), 25 | } 26 | } 27 | 28 | if(ipAddress) { 29 | payload.connectivityStatus.ipAddress = ipAddress 30 | } 31 | 32 | if(disconnectReason) { 33 | payload.connectivityStatus.disconnectReason = disconnectReason 34 | } 35 | 36 | console.log({clientInitiatedDisconnect}) 37 | 38 | if(clientInitiatedDisconnect !== undefined) { 39 | payload.connectivityStatus.clientInitiatedDisconnect = `${clientInitiatedDisconnect}` 40 | } 41 | 42 | try { 43 | let entity = { 44 | id: `urn:ngsi-ld:${AWSIOTTHINGTYPE}:${clientId}`, 45 | type: [`${AWSIOTTHINGTYPE}`], 46 | ...payload 47 | } 48 | 49 | entity = normalize(entity) 50 | 51 | try { 52 | 53 | let {data: res} = await axios.post(`${dns_broker}/entities/${entity.id}/attrs`, entity, {headers: headers}) 54 | console.log(res) 55 | } catch(e){ 56 | console.log(e) 57 | } 58 | 59 | } catch (e) { 60 | log_error(event,context, e.message, e) 61 | } 62 | 63 | } 64 | 65 | } catch (e) { 66 | log_error(event,context, e.message, e) 67 | } 68 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-api/garnet-api-stack.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CfnOutput, NestedStack, NestedStackProps} from "aws-cdk-lib" 2 | import { Construct } from "constructs" 3 | import { GarnetApiCommon } from "./apicommon/api-common-construct" 4 | import { Vpc } from "aws-cdk-lib/aws-ec2" 5 | import { GarnetApiGateway } from "./apigateway/api-gateway-construct" 6 | import { ApplicationLoadBalancer } from "aws-cdk-lib/aws-elasticloadbalancingv2" 7 | import { GarnetApiAuthJwt } from "./apiauth/api-auth-construct" 8 | import { Secret } from "aws-cdk-lib/aws-secretsmanager" 9 | import { Queue } from "aws-cdk-lib/aws-sqs" 10 | 11 | 12 | 13 | export interface GarnetApiProps extends NestedStackProps { 14 | readonly vpc: Vpc, 15 | readonly dns_context_broker: string, 16 | readonly garnet_ingestion_sqs: Queue, 17 | readonly garnet_private_endpoint: string 18 | readonly fargate_alb: ApplicationLoadBalancer 19 | readonly secret_api_jwt: Secret 20 | 21 | } 22 | 23 | export class GarnetApi extends NestedStack { 24 | 25 | public readonly private_sub_endpoint: string 26 | public readonly api_ref: string 27 | public readonly broker_api_endpoint: string 28 | public readonly garnet_api_token : string 29 | 30 | constructor(scope: Construct, id: string, props: GarnetApiProps) { 31 | super(scope, id, props) 32 | 33 | 34 | const api_auth_construct = new GarnetApiAuthJwt(this, "ApiAuth", { 35 | secret_api_jwt: props.secret_api_jwt 36 | }) 37 | 38 | const api_gateway_construct = new GarnetApiGateway(this, "Api", { 39 | vpc: props.vpc, 40 | fargate_alb: props.fargate_alb, 41 | lambda_authorizer_arn: api_auth_construct.lambda_authorizer_arn 42 | }) 43 | 44 | const api_common_construct = new GarnetApiCommon(this, 'GarnetApiCommon', { 45 | api_ref: api_gateway_construct.api_ref, 46 | vpc: props.vpc, 47 | dns_context_broker: props.dns_context_broker, 48 | garnet_ingestion_sqs: props.garnet_ingestion_sqs, 49 | garnet_private_endpoint: props.garnet_private_endpoint 50 | }) 51 | 52 | 53 | this.api_ref = api_gateway_construct.api_ref 54 | this.broker_api_endpoint = `https://${api_gateway_construct.api_ref}.execute-api.${Aws.REGION}.amazonaws.com` 55 | this.garnet_api_token = api_auth_construct.garnet_api_token 56 | 57 | new CfnOutput(this, "garnet_endpoint", { 58 | value: `https://${api_gateway_construct.api_ref}.execute-api.${Aws.REGION}.amazonaws.com`, 59 | }) 60 | 61 | 62 | } 63 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apicommon/lambda/garnetVersion/index.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios') 2 | const CONTEXT_BROKER = process.env.CONTEXT_BROKER 3 | const GARNET_VERSION = process.env.GARNET_VERSION 4 | const GARNET_PRIVATE_ENDPOINT = process.env.GARNET_PRIVATE_ENDPOINT 5 | const GARNET_INGESTION_SQS_URL = process.env.GARNET_INGESTION_SQS_URL 6 | const GARNET_INGESTION_SQS_ARN = process.env.GARNET_INGESTION_SQS_ARN 7 | const DNS_CONTEXT_BROKER = process.env.DNS_CONTEXT_BROKER 8 | const GARNET_ARCHITECTURE = process.env.GARNET_ARCHITECTURE 9 | const GARNET_CONTAINERS = JSON.parse(process.env.GARNET_CONTAINERS) 10 | 11 | exports.handler = async (event) => { 12 | try { 13 | 14 | const {headers : {Host}} = event 15 | 16 | let path = `q/info` 17 | 18 | let allTasks = GARNET_CONTAINERS.map(async (c) => { 19 | 20 | let url = `http://${DNS_CONTEXT_BROKER}/${path}` 21 | 22 | try { 23 | let { data } = await axios.get(url, { headers: { container: c}}); 24 | console.log({data}) 25 | return { [c]: { data } }; 26 | } catch (e) { 27 | console.log(e) 28 | return {[c]: 'ERROR - check the service is running' } 29 | } 30 | }) 31 | 32 | const responsesArray = await Promise.all(allTasks); 33 | const responses = responsesArray.length > 0 ? Object.fromEntries( 34 | responsesArray.map((x) => [Object.keys(x)[0], Object.values(x)[0]]) 35 | ) : null 36 | 37 | 38 | let result = { 39 | garnet_version: GARNET_VERSION, 40 | garnet_architecture: GARNET_ARCHITECTURE, 41 | context_broker: CONTEXT_BROKER, 42 | garnet_private_endpoint: GARNET_PRIVATE_ENDPOINT, 43 | garnet_ingestion_sqs_url: GARNET_INGESTION_SQS_URL, 44 | garnet_ingestion_sqs_arn: GARNET_INGESTION_SQS_ARN, 45 | context_broker_info: responses 46 | } 47 | 48 | const response = { 49 | statusCode: 200, 50 | headers: { 51 | "Content-Type": "application/json" 52 | }, 53 | body: JSON.stringify(result), 54 | } 55 | 56 | 57 | return response 58 | 59 | 60 | } catch (e) { 61 | const response = { 62 | statusCode: 500, 63 | headers: { 64 | "Content-Type": "application/json" 65 | }, 66 | body: JSON.stringify({message: e.message}), 67 | } 68 | console.log(e) 69 | return response 70 | } 71 | 72 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-privatesub/lambda/garnetSub/index.js: -------------------------------------------------------------------------------- 1 | const iot_region = process.env.AWSIOTREGION 2 | const sub_all = process.env.SUBALL 3 | const { IoTDataPlaneClient, PublishCommand } = require("@aws-sdk/client-iot-data-plane") 4 | const iotdata = new IoTDataPlaneClient({region: iot_region}) 5 | 6 | const { recursive_concise} = require('/opt/nodejs/utils.js') 7 | 8 | exports.handler = async (event) => { 9 | try { 10 | const {body} = event 11 | if(!body){ 12 | return { 13 | statusCode: 400, 14 | headers: { 15 | "Content-Type": "application/json" 16 | }, 17 | body: JSON.stringify({message: 'Bad Request. Notification is the only type valid'}) 18 | } 19 | } 20 | const payload = JSON.parse(body) 21 | 22 | if(payload?.type != "Notification") { 23 | console.log('ERROR not Notification') 24 | return { 25 | statusCode: 400, 26 | headers: { 27 | "Content-Type": "application/json" 28 | }, 29 | body: JSON.stringify({message: 'Bad Request. Notification is the only type valid'}) 30 | } 31 | } 32 | // GET THE SUBSCRIPTION NAME FROM SUBSCRIPTION ID 33 | const subName = `${payload.subscriptionId.split(':').slice(-1)}` 34 | 35 | payload.data.forEach(entity => { 36 | for (let [key, value] of Object.entries(entity)) { 37 | if(!['type', 'id', '@context'].includes(key)) { 38 | 39 | if( typeof value == 'object' && !Array.isArray(value)){ 40 | recursive_concise(key, value) 41 | } else { 42 | entity[key] = { 43 | value: value 44 | } 45 | } 46 | 47 | 48 | } 49 | } 50 | }) 51 | 52 | const publish = await iotdata.send( 53 | new PublishCommand({ 54 | topic: `garnet/subscriptions/${subName}`, 55 | payload: JSON.stringify(payload) 56 | }) 57 | ) 58 | 59 | 60 | const response = { 61 | statusCode: 200 62 | } 63 | return response 64 | 65 | } catch (e) { 66 | const response = { 67 | statusCode: 500, 68 | headers: { 69 | "Content-Type": "application/json" 70 | }, 71 | body: JSON.stringify({message: e.message}), 72 | } 73 | console.log(e) 74 | return response 75 | 76 | } 77 | 78 | 79 | } 80 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/athena/athena-construct.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CustomResource, Duration, RemovalPolicy } from "aws-cdk-lib" 2 | import { PolicyStatement } from "aws-cdk-lib/aws-iam" 3 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs" 4 | import { Provider } from "aws-cdk-lib/custom-resources" 5 | import { Construct } from "constructs" 6 | import { Runtime, Function, Code, Architecture } from "aws-cdk-lib/aws-lambda" 7 | import { garnet_bucket_athena, garnet_constant } from "../../../../constants" 8 | 9 | export interface GarnetDataLakeAthenaProps { 10 | 11 | } 12 | 13 | 14 | export class GarnetDataLakeAthena extends Construct { 15 | 16 | 17 | constructor(scope: Construct, id: string, props: GarnetDataLakeAthenaProps) { 18 | super(scope, id) 19 | 20 | // CUSTOM RESOURCE WITH A LAMBDA THAT WILL CREATE ATHENA WORKGROUP AND GLUE DB 21 | const lambda_athena_log = new LogGroup(this, 'LambdaAthenaFunctionLogs', { 22 | retention: RetentionDays.ONE_MONTH, 23 | // logGroupName: `garnet-lake-athena-lambda-cw-logs`, 24 | removalPolicy: RemovalPolicy.DESTROY 25 | }) 26 | const lambda_athena_path = `${__dirname}/lambda/athena` 27 | const lambda_athena = new Function(this, 'AthenaFunction', { 28 | functionName: `garnet-lake-athena-lambda`, 29 | description: 'Garnet Lake - Function that creates Athena resources', 30 | logGroup: lambda_athena_log, 31 | runtime: Runtime.NODEJS_22_X, 32 | code: Code.fromAsset(lambda_athena_path), 33 | handler: 'index.handler', 34 | timeout: Duration.seconds(50), 35 | architecture: Architecture.ARM_64, 36 | environment: { 37 | BUCKET_NAME_ATHENA: garnet_bucket_athena, 38 | CATALOG_ID: Aws.ACCOUNT_ID, 39 | GLUEDB_NAME: garnet_constant.gluedbName 40 | } 41 | }) 42 | lambda_athena.node.addDependency(lambda_athena_log) 43 | lambda_athena.addToRolePolicy(new PolicyStatement({ 44 | actions: [ 45 | "athena:CreateWorkGroup", 46 | "glue:CreateDatabase" 47 | ], 48 | resources: ["*"] 49 | })) 50 | 51 | const athena_provider_log = new LogGroup(this, 'LambdaAthenaProviderLogs', { 52 | retention: RetentionDays.ONE_MONTH, 53 | // logGroupName: `garnet-provider-custom-athena-lambda-cw-logs`, 54 | removalPolicy: RemovalPolicy.DESTROY 55 | }) 56 | 57 | const athena_provider = new Provider(this, 'AthenaProvider', { 58 | onEventHandler: lambda_athena, 59 | providerFunctionName: `garnet-custom-provider-athena-lambda`, 60 | logGroup: athena_provider_log 61 | }) 62 | athena_provider.node.addDependency(athena_provider_log) 63 | const athena_resource = new CustomResource(this, 'CustomBucketAthenaResource', { 64 | serviceToken: athena_provider.serviceToken, 65 | 66 | }) 67 | 68 | 69 | } 70 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-privatesub/lambda/sqsCreate/index.js: -------------------------------------------------------------------------------- 1 | const { SQSClient, CreateQueueCommand, GetQueueUrlCommand } = require("@aws-sdk/client-sqs"); 2 | const sqs = new SQSClient(); 3 | const QUEUE_NAME = process.env.QUEUE_NAME; 4 | 5 | const checkQueueExists = async (queueName) => { 6 | try { 7 | await sqs.send(new GetQueueUrlCommand({ QueueName: queueName })); 8 | console.log(`Queue ${queueName} already exists`) 9 | return true 10 | } catch (error) { 11 | console.log(error) 12 | if (error.name === 'QueueDoesNotExist') { 13 | console.log(`Queue ${queueName} does not exist`); 14 | return false; 15 | } 16 | 17 | // For other errors, we might want to throw them 18 | // throw error; 19 | } 20 | }; 21 | 22 | const createQueueIfNotExists = async (queueName) => { 23 | const exists = await checkQueueExists(queueName) 24 | if (!exists) { 25 | try { 26 | console.log(`Creating queue ${queueName}`); 27 | const response = await sqs.send(new CreateQueueCommand({ 28 | QueueName: queueName 29 | })) 30 | console.log(`Successfully created queue ${queueName} with URL: ${response.QueueUrl}`); 31 | return response.QueueUrl; 32 | } catch (error) { 33 | // QueueAlreadyExists - if queue was created between our check and create 34 | if (error.name === 'QueueAlreadyExists') { 35 | console.log(`Queue ${queueName} already exists (caught in create)`); 36 | // Get the queue URL 37 | const response = await sqs.send(new GetQueueUrlCommand({ QueueName: queueName })); 38 | return response.QueueUrl; 39 | } 40 | throw error; 41 | } 42 | } else { 43 | // Queue exists, get its URL 44 | const response = await sqs.send(new GetQueueUrlCommand({ QueueName: queueName })); 45 | return response.QueueUrl; 46 | } 47 | } 48 | 49 | exports.handler = async (event) => { 50 | console.log('Event:', JSON.stringify(event, null, 2)); 51 | const requestType = event['RequestType'].toLowerCase(); 52 | 53 | if (requestType === 'create' || requestType === 'update') { 54 | try { 55 | // Create main queue if it doesn't exist 56 | const queueUrl = await createQueueIfNotExists(QUEUE_NAME); 57 | console.log(queueUrl) 58 | return { 59 | Data: { 60 | queue_name: QUEUE_NAME, 61 | queue_url: queueUrl 62 | } 63 | }; 64 | } catch (error) { 65 | console.error('Error in handler:', error); 66 | throw error; 67 | } 68 | } else if (requestType === 'delete') { 69 | // Don't delete the queue, just return success 70 | return { 71 | Data: { 72 | queue_name: QUEUE_NAME 73 | } 74 | }; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/garnet-iot-stack.ts: -------------------------------------------------------------------------------- 1 | import { NestedStack, NestedStackProps, RemovalPolicy} from "aws-cdk-lib"; 2 | import { Construct } from "constructs" 3 | import { GarnetIotGroup } from "./iot-group/iot-group-construct"; 4 | import { GarnetIotThing } from "./iot-thing/iot-thing-construct"; 5 | import { Queue } from "aws-cdk-lib/aws-sqs"; 6 | import { Vpc } from "aws-cdk-lib/aws-ec2"; 7 | import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources"; 8 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 9 | 10 | export interface GarnetIotProps extends NestedStackProps { 11 | vpc: Vpc, 12 | dns_context_broker: string 13 | } 14 | 15 | export class GarnetIot extends NestedStack { 16 | constructor(scope: Construct, id: string, props: GarnetIotProps) { 17 | super(scope, id, props) 18 | 19 | 20 | 21 | /* 22 | * EVENT CONFIGURATION 23 | */ 24 | 25 | let event_param = { 26 | eventConfigurations: { 27 | "THING": { 28 | "Enabled": true 29 | }, 30 | "THING_GROUP": { 31 | "Enabled": true 32 | }, 33 | "THING_GROUP_MEMBERSHIP": { 34 | Enabled: true 35 | }, 36 | "THING_GROUP_HIERARCHY": { 37 | "Enabled": true 38 | }, 39 | "POLICY": { 40 | "Enabled": true 41 | }, 42 | "CERTIFICATE": { 43 | "Enabled": true 44 | } 45 | } 46 | } 47 | 48 | const garnet_iot_custom_thinggroup_event_log = new LogGroup(this, 'GarnetIoTEventConfigLogs', { 49 | retention: RetentionDays.ONE_MONTH, 50 | removalPolicy: RemovalPolicy.DESTROY 51 | }) 52 | 53 | const iotgroup_event = new AwsCustomResource(this, 'GarnetIoTEventConfig', { 54 | functionName: `garnet-iot-event-config`, 55 | onCreate: { 56 | service: 'Iot', 57 | action: 'UpdateEventConfigurations', 58 | physicalResourceId: PhysicalResourceId.of(Date.now().toString()), 59 | parameters: event_param 60 | }, 61 | onUpdate: { 62 | service: 'Iot', 63 | action: 'UpdateEventConfigurations', 64 | physicalResourceId: PhysicalResourceId.of(Date.now().toString()), 65 | parameters: event_param 66 | }, 67 | logGroup: garnet_iot_custom_thinggroup_event_log, 68 | policy: AwsCustomResourcePolicy.fromSdkCalls({resources: AwsCustomResourcePolicy.ANY_RESOURCE}) 69 | }) 70 | 71 | /* 72 | * END EVENT CONFIGURATION 73 | */ 74 | 75 | 76 | 77 | const iot_group_construct = new GarnetIotGroup(this, 'GarnetIotGroup', { 78 | vpc: props.vpc, 79 | dns_context_broker: props.dns_context_broker 80 | }) 81 | const iot_presence_construct = new GarnetIotThing(this, 'GarnetIotThing',{ 82 | vpc: props.vpc, 83 | dns_context_broker: props.dns_context_broker 84 | }) 85 | } 86 | } -------------------------------------------------------------------------------- /cdk.json: -------------------------------------------------------------------------------- 1 | { 2 | "app": "npx ts-node --prefer-ts-exts bin/garnet.ts", 3 | "watch": { 4 | "include": [ 5 | "**" 6 | ], 7 | "exclude": [ 8 | "README.md", 9 | "cdk*.json", 10 | "**/*.d.ts", 11 | "**/*.js", 12 | "tsconfig.json", 13 | "package*.json", 14 | "yarn.lock", 15 | "node_modules", 16 | "test" 17 | ] 18 | }, 19 | "context": { 20 | "@aws-cdk/aws-lambda:recognizeLayerVersion": true, 21 | "@aws-cdk/core:checkSecretUsage": true, 22 | "@aws-cdk/core:target-partitions": [ 23 | "aws", 24 | "aws-cn" 25 | ], 26 | "@aws-cdk-containers/ecs-service-extensions:enableDefaultLogDriver": true, 27 | "@aws-cdk/aws-ec2:uniqueImdsv2TemplateName": true, 28 | "@aws-cdk/aws-ecs:arnFormatIncludesClusterName": true, 29 | "@aws-cdk/aws-iam:minimizePolicies": true, 30 | "@aws-cdk/core:validateSnapshotRemovalPolicy": true, 31 | "@aws-cdk/aws-codepipeline:crossAccountKeyAliasStackSafeResourceName": true, 32 | "@aws-cdk/aws-s3:createDefaultLoggingPolicy": true, 33 | "@aws-cdk/aws-sns-subscriptions:restrictSqsDescryption": true, 34 | "@aws-cdk/aws-apigateway:disableCloudWatchRole": true, 35 | "@aws-cdk/core:enablePartitionLiterals": true, 36 | "@aws-cdk/aws-events:eventsTargetQueueSameAccount": true, 37 | "@aws-cdk/aws-iam:standardizedServicePrincipals": true, 38 | "@aws-cdk/aws-ecs:disableExplicitDeploymentControllerForCircuitBreaker": true, 39 | "@aws-cdk/aws-iam:importedRoleStackSafeDefaultPolicyName": true, 40 | "@aws-cdk/aws-s3:serverAccessLogsUseBucketPolicy": true, 41 | "@aws-cdk/aws-route53-patters:useCertificate": true, 42 | "@aws-cdk/customresources:installLatestAwsSdkDefault": false, 43 | "@aws-cdk/aws-rds:databaseProxyUniqueResourceName": true, 44 | "@aws-cdk/aws-codedeploy:removeAlarmsFromDeploymentGroup": true, 45 | "@aws-cdk/aws-apigateway:authorizerChangeDeploymentLogicalId": true, 46 | "@aws-cdk/aws-ec2:launchTemplateDefaultUserData": true, 47 | "@aws-cdk/aws-secretsmanager:useAttachedSecretResourcePolicyForSecretTargetAttachments": true, 48 | "@aws-cdk/aws-redshift:columnId": true, 49 | "@aws-cdk/aws-stepfunctions-tasks:enableEmrServicePolicyV2": true, 50 | "@aws-cdk/aws-ec2:restrictDefaultSecurityGroup": true, 51 | "@aws-cdk/aws-apigateway:requestValidatorUniqueId": true, 52 | "@aws-cdk/aws-kms:aliasNameRef": true, 53 | "@aws-cdk/aws-autoscaling:generateLaunchTemplateInsteadOfLaunchConfig": true, 54 | "@aws-cdk/core:includePrefixInUniqueNameGeneration": true, 55 | "@aws-cdk/aws-efs:denyAnonymousAccess": true, 56 | "@aws-cdk/aws-opensearchservice:enableOpensearchMultiAzWithStandby": true, 57 | "@aws-cdk/aws-lambda-nodejs:useLatestRuntimeVersion": true, 58 | "@aws-cdk/aws-efs:mountTargetOrderInsensitiveLogicalId": true, 59 | "@aws-cdk/aws-rds:auroraClusterChangeScopeOfInstanceParameterGroupWithEachParameters": true, 60 | "@aws-cdk/aws-appsync:useArnForSourceApiAssociationIdentifier": true, 61 | "@aws-cdk/aws-rds:preventRenderingDeprecatedCredentials": true 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /lib/stacks/garnet-privatesub/lambda/garnetSubSqs/index.js: -------------------------------------------------------------------------------- 1 | const iot_region = process.env.AWSIOTREGION 2 | const { IoTDataPlaneClient, PublishCommand } = require("@aws-sdk/client-iot-data-plane") 3 | const iotdata = new IoTDataPlaneClient({region: iot_region}) 4 | 5 | const { recursive_concise} = require('/opt/nodejs/utils.js') 6 | 7 | exports.handler = async (event) => { 8 | console.log('Processing SQS records:', event.Records.length); 9 | 10 | try { 11 | for (const msg of event.Records) { 12 | try { 13 | console.log('Processing message:', msg.messageId); 14 | 15 | const payload = JSON.parse(msg.body) 16 | 17 | // Validate payload structure 18 | if (!payload.subscriptionId) { 19 | console.error('Missing subscriptionId in payload:', JSON.stringify(payload)); 20 | continue; 21 | } 22 | 23 | if (!payload.data || !Array.isArray(payload.data)) { 24 | console.error('Missing or invalid data array in payload:', JSON.stringify(payload)); 25 | continue; 26 | } 27 | 28 | // GET THE SUBSCRIPTION NAME FROM SUBSCRIPTION ID (fix: get actual last element) 29 | const subName = payload.subscriptionId.split(':').slice(-1)[0] 30 | console.log('Subscription name:', subName); 31 | 32 | // Transform payload data 33 | payload.data.forEach(entity => { 34 | for (let [key, value] of Object.entries(entity)) { 35 | if(!['type', 'id', '@context'].includes(key)) { 36 | if( typeof value == 'object' && !Array.isArray(value)){ 37 | recursive_concise(key, value) 38 | } else { 39 | entity[key] = { 40 | value: value 41 | } 42 | } 43 | } 44 | } 45 | }) 46 | 47 | // Publish to IoT MQTT topic 48 | const topic = `garnet/subscriptions/${subName}`; 49 | console.log('Publishing to topic:', topic); 50 | 51 | const publish = await iotdata.send( 52 | new PublishCommand({ 53 | topic: topic, 54 | payload: JSON.stringify(payload) 55 | }) 56 | ) 57 | 58 | console.log('Successfully published message to topic:', topic); 59 | 60 | } catch (e) { 61 | console.error('Error processing message:', msg.messageId, e); 62 | // Continue processing other messages even if one fails 63 | } 64 | } 65 | 66 | } catch (e) { 67 | console.error('Error processing SQS event:', e); 68 | throw e; // Re-throw to mark the Lambda as failed 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing Guidelines 2 | 3 | Thank you for your interest in contributing to our project. Whether it's a bug report, new feature, correction, or additional 4 | documentation, we greatly value feedback and contributions from our community. 5 | 6 | Please read through this document before submitting any issues or pull requests to ensure we have all the necessary 7 | information to effectively respond to your bug report or contribution. 8 | 9 | 10 | ## Reporting Bugs/Feature Requests 11 | 12 | We welcome you to use the GitHub issue tracker to report bugs or suggest features. 13 | 14 | When filing an issue, please check existing open, or recently closed, issues to make sure somebody else hasn't already 15 | reported the issue. Please try to include as much information as you can. Details like these are incredibly useful: 16 | 17 | * A reproducible test case or series of steps 18 | * The version of our code being used 19 | * Any modifications you've made relevant to the bug 20 | * Anything unusual about your environment or deployment 21 | 22 | 23 | ## Contributing via Pull Requests 24 | Contributions via pull requests are much appreciated. Before sending us a pull request, please ensure that: 25 | 26 | 1. You are working against the latest source on the *main* branch. 27 | 2. You check existing open, and recently merged, pull requests to make sure someone else hasn't addressed the problem already. 28 | 3. You open an issue to discuss any significant work - we would hate for your time to be wasted. 29 | 30 | To send us a pull request, please: 31 | 32 | 1. Fork the repository. 33 | 2. Modify the source; please focus on the specific change you are contributing. If you also reformat all the code, it will be hard for us to focus on your change. 34 | 3. Ensure local tests pass. 35 | 4. Commit to your fork using clear commit messages. 36 | 5. Send us a pull request, answering any default questions in the pull request interface. 37 | 6. Pay attention to any automated CI failures reported in the pull request, and stay involved in the conversation. 38 | 39 | GitHub provides additional document on [forking a repository](https://help.github.com/articles/fork-a-repo/) and 40 | [creating a pull request](https://help.github.com/articles/creating-a-pull-request/). 41 | 42 | 43 | ## Finding contributions to work on 44 | Looking at the existing issues is a great way to find something to contribute on. As our projects, by default, use the default GitHub issue labels (enhancement/bug/duplicate/help wanted/invalid/question/wontfix), looking at any 'help wanted' issues is a great place to start. 45 | 46 | 47 | ## Code of Conduct 48 | This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct). 49 | For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact 50 | opensource-codeofconduct@amazon.com with any additional questions or comments. 51 | 52 | 53 | ## Security issue notifications 54 | If you discover a potential security issue in this project we ask that you notify AWS/Amazon Security via our [vulnerability reporting page](http://aws.amazon.com/security/vulnerability-reporting/). Please do **not** create a public github issue. 55 | 56 | 57 | ## Licensing 58 | 59 | See the [LICENSE](LICENSE) file for our project's licensing. We will ask you to confirm the licensing of your contribution. 60 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/bucket/lambda/bucketCreate/index.js: -------------------------------------------------------------------------------- 1 | const { S3Client, CreateBucketCommand, PutBucketMetricsConfigurationCommand, HeadBucketCommand } = require("@aws-sdk/client-s3"); 2 | const s3 = new S3Client(); 3 | const BUCKET_NAME = process.env.BUCKET_NAME; 4 | const BUCKET_ATHENA_NAME = process.env.BUCKET_ATHENA_NAME; 5 | 6 | const checkBucketExists = async (bucketName) => { 7 | try { 8 | await s3.send(new HeadBucketCommand({ Bucket: bucketName })); 9 | console.log(`Bucket ${bucketName} already exists`) 10 | return true 11 | } catch (error) { 12 | console.log(error) 13 | if (error.$metadata && error.$metadata.httpStatusCode === 404) { 14 | console.log(`Bucket ${bucketName} does not exist`); 15 | return false; 16 | } 17 | 18 | // throw error; 19 | } 20 | }; 21 | 22 | const createBucketIfNotExists = async (bucketName) => { 23 | const exists = await checkBucketExists(bucketName) 24 | if (!exists) { 25 | try { 26 | console.log(`Creating bucket ${bucketName}`); 27 | await s3.send(new CreateBucketCommand({ Bucket: bucketName })); 28 | console.log(`Successfully created bucket ${bucketName}`); 29 | } catch (error) { 30 | // BucketAlreadyOwnedByYou 31 | // BucketAlreadyExists 32 | if (!error.name.includes('BucketAlready')) { 33 | throw error; 34 | } 35 | console.log(`Bucket ${bucketName} already exists (caught in create)`); 36 | } 37 | } 38 | } 39 | 40 | exports.handler = async (event) => { 41 | console.log('Event:', JSON.stringify(event, null, 2)); 42 | const requestType = event['RequestType'].toLowerCase(); 43 | 44 | if (requestType === 'create' || requestType === 'update') { 45 | try { 46 | // Create main bucket if it doesn't exist 47 | await createBucketIfNotExists(BUCKET_NAME); 48 | 49 | // Create Athena bucket if it doesn't exist 50 | await createBucketIfNotExists(BUCKET_ATHENA_NAME); 51 | 52 | // Set up metrics configuration 53 | try { 54 | console.log("Setting up metrics configuration"); 55 | await s3.send( 56 | new PutBucketMetricsConfigurationCommand({ 57 | Bucket: BUCKET_NAME, 58 | Id: "GarnetBucketMetric", 59 | MetricsConfiguration: { 60 | Id: "GarnetBucketConfigmMetric" 61 | } 62 | }) 63 | ); 64 | console.log("Successfully set up metrics configuration"); 65 | } catch (error) { 66 | console.log("Error setting up metrics configuration:", error.message); 67 | } 68 | 69 | return { 70 | Data: { 71 | bucket_name: BUCKET_NAME 72 | } 73 | }; 74 | } catch (error) { 75 | console.error('Error in handler:', error); 76 | throw error; 77 | } 78 | } else if (requestType === 'delete') { 79 | // Don't delete the bucket, just return success 80 | return { 81 | Data: { 82 | bucket_name: BUCKET_NAME 83 | } 84 | }; 85 | } 86 | } 87 | 88 | 89 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-group/lambda/groupLifecycle/index.js: -------------------------------------------------------------------------------- 1 | const dns_broker = `http://${process.env.DNS_CONTEXT_BROKER}/ngsi-ld/v1` 2 | const AWSIOTTHINGGROUPTYPE = process.env.AWSIOTTHINGGROUPTYPE 3 | const axios = require('axios') 4 | const {log_error, normalize} = require('/opt/nodejs/utils.js') 5 | 6 | const headers = { 7 | 'Content-Type': 'application/json' 8 | } 9 | 10 | exports.handler = async (event, context) => { 11 | try { 12 | 13 | 14 | // Extract group information from the event 15 | const {operation, thingGroupName, timestamp, thingGroupId, versionNumber, parentGroupName, description, rootToParentThingGroups, attributes, eventId } = event 16 | 17 | if(['CREATED', 'UPDATED'].includes(operation)){ 18 | let group = { 19 | id: `urn:ngsi-ld:${AWSIOTTHINGGROUPTYPE}:${thingGroupName}`, 20 | type: `${AWSIOTTHINGGROUPTYPE}`, 21 | thingGroupName: { 22 | value: thingGroupName 23 | }, 24 | description: { 25 | value: description 26 | }, 27 | thingGroupId: { 28 | value: thingGroupId 29 | }, 30 | versionNumber: { 31 | value: versionNumber 32 | }, 33 | eventType: { 34 | type: "Property", 35 | value: operation, 36 | eventId: eventId, 37 | observedAt: (new Date(timestamp)).toISOString() 38 | } 39 | } 40 | 41 | if(parentGroupName){ 42 | group.inParentGroup = { 43 | object: `urn:ngsi-ld:${AWSIOTTHINGGROUPTYPE}:${parentGroupName}`, 44 | objectType: `${AWSIOTTHINGGROUPTYPE}` 45 | } 46 | 47 | group.scope = '/' + parentGroupName + `/` + thingGroupName 48 | } 49 | 50 | if(rootToParentThingGroups && rootToParentThingGroups.length > 0 ){ 51 | group.inGroupHierarchy = { 52 | objectList: rootToParentThingGroups.map(tg => `urn:ngsi-ld:${AWSIOTTHINGGROUPTYPE}:${tg.groupArn.split('/').pop()}`), 53 | objectType: AWSIOTTHINGGROUPTYPE 54 | } 55 | 56 | group.scope = '/' + rootToParentThingGroups.map(tg => tg.groupArn.split('/').pop()).join('/') + `/` + thingGroupName 57 | } 58 | 59 | if (attributes && typeof attributes === 'object') { 60 | Object.keys(attributes).forEach(key => { 61 | if (attributes[key] !== undefined && attributes[key] !== null) { 62 | group[key] = { 63 | type: 'Property', 64 | value: attributes[key] 65 | } 66 | } 67 | }) 68 | } 69 | 70 | try { 71 | 72 | let {data: res} = await axios.post(`${dns_broker}/entityOperations/upsert?options=update `, [group], {headers: headers}) 73 | console.log(res) 74 | 75 | } catch(e){ 76 | console.log(e) 77 | } 78 | 79 | 80 | } 81 | 82 | if(['DELETED'].includes(operation)){ 83 | let {data: res} = await axios.delete(`${dns_broker}/entities/urn:ngsi-ld:${AWSIOTTHINGGROUPTYPE}:${thingGroupName}`) 84 | console.log(res) 85 | } 86 | 87 | 88 | } catch (e) { 89 | log_error(event, context, e.message, e) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /lib/layers/nodejs/utils.js: -------------------------------------------------------------------------------- 1 | const reserved_attributes = [ 2 | "type", 3 | "id", 4 | "@context", 5 | "scope", 6 | "observedAt", 7 | "createdAt", 8 | "modifiedAt", 9 | "deletedAt" 10 | ] 11 | 12 | const reserved_types = ['Property', 'Relationship', 'GeoProperty', 'DateTime'] 13 | 14 | 15 | exports.get_type = (payload) => { 16 | 17 | if(Array.isArray(payload.type)) { 18 | let tp = `${payload?.id.split('urn:ngsi-ld:').slice(-1)}` 19 | tp = `${tp.split(':').slice(0,1)}` 20 | if(!payload.type.includes(tp)){ 21 | console.log(`${payload.type[0]} - ${tp}`) 22 | tp = payload.type[0] 23 | } 24 | return tp 25 | } 26 | if(payload.type.includes('#')){ 27 | payload.type = `${payload.type.split('#').slice(-1)}` 28 | } 29 | if(payload.type.includes('/')){ 30 | payload.type = `${payload.type.split('/').slice(-1)}` 31 | } 32 | return payload.type 33 | } 34 | 35 | exports.log_error = (event, context, message, error) => { 36 | console.error(JSON.stringify({ 37 | message: message, 38 | event: event, 39 | error: error, 40 | context: context 41 | })) 42 | } 43 | 44 | exports.recursive_concise = (key, value) => { 45 | 46 | if( typeof value == 'object' && !Array.isArray(value)){ 47 | if(reserved_types.includes(value["type"])) { 48 | delete value["type"] 49 | } 50 | for (let [skey, svalue] of Object.entries(value)){ 51 | this.recursive_concise(skey,svalue) 52 | } 53 | } 54 | } 55 | 56 | exports.transform = (payload) => { 57 | 58 | payload.type = this.get_type(payload) 59 | 60 | for (let [key, value] of Object.entries(payload)) { 61 | if(!reserved_attributes.includes(key)) { 62 | if( typeof value == "object" && !Array.isArray(value)){ 63 | this.recursive_concise(key, value) 64 | } else { 65 | payload[key] = { 66 | "value": value 67 | } 68 | } 69 | } 70 | } 71 | 72 | return payload 73 | } 74 | 75 | exports.concise = (payload) => { 76 | for (let [key, value] of Object.entries(payload)) { 77 | if(!reserved_attributes.includes(key)) { 78 | 79 | if( typeof value == "object" && !Array.isArray(value)){ 80 | this.recursive_concise(key, value) 81 | } else { 82 | payload[key] = { 83 | "value": value 84 | } 85 | } 86 | 87 | 88 | } 89 | } 90 | 91 | return payload 92 | } 93 | 94 | 95 | exports.normalize = (entity) => { 96 | const excludedKeys = ['@context', 'id', 'type'] 97 | Object.entries(entity).map(([key, value]) => { 98 | if (!excludedKeys.includes(key) && 99 | entity[key].value && !entity[key].type) { 100 | entity[key].type = "Property" 101 | } 102 | }) 103 | 104 | return entity 105 | } 106 | 107 | exports.compactKeys = (input) => { 108 | try { 109 | if (input === null || typeof input !== 'object') return input; 110 | 111 | if (Array.isArray(input)) { 112 | // Here we use compactKeys again for each array element 113 | return input.map((item) => this.compactKeys(item)); 114 | } 115 | 116 | return Object.fromEntries( 117 | Object.entries(input).map(([key, value]) => [ 118 | key === 'id' || key === 'type' ? key : key.split(/[/#:]/).pop(), 119 | this.compactKeys(value) 120 | ]) 121 | ); 122 | } catch (error) { 123 | console.error('Error while compacting keys:', error); 124 | return input; 125 | } 126 | } -------------------------------------------------------------------------------- /lib/garnet-stack.ts: -------------------------------------------------------------------------------- 1 | import { CfnElement, CfnOutput, Names, Stack, StackProps } from 'aws-cdk-lib' 2 | import { Construct } from 'constructs' 3 | import { GarnetScorpio } from './stacks/garnet-scorpio/garnet-scorpio-stack' 4 | import { GarnetIngestionStack} from './stacks/garnet-ingestion/garnet-ingestion-stack' 5 | import { garnet_constant } from '../constants' 6 | import { GarnetCommon } from './stacks/garnet-common/garnet-common-stack' 7 | import { GarnetOps } from './stacks/garnet-ops/garnet-ops-stack' 8 | import { deployment_params } from '../architecture' 9 | import { GarnetLake } from './stacks/garnet-lake/garnet-lake-stack' 10 | import { GarnetIot } from './stacks/garnet-iot/garnet-iot-stack' 11 | import { GarnetPrivateSub } from './stacks/garnet-privatesub/garnet-privatesub-stack' 12 | import { GarnetApi } from './stacks/garnet-api/garnet-api-stack' 13 | 14 | export class GarnetStack extends Stack { 15 | 16 | 17 | getLogicalId(element: CfnElement): string { 18 | if (element?.node?.id?.includes('NestedStackResource')) { 19 | let stack_name = (/([a-zA-Z0-9]+)\.NestedStackResource/.exec(element.node.id)![1]) 20 | return stack_name 21 | } 22 | return super.getLogicalId(element) 23 | } 24 | 25 | constructor(scope: Construct, id: string, props?: StackProps) { 26 | super(scope, id, props) 27 | const garnet_datalake = new GarnetLake(this, 'GarnetLake', {}) 28 | 29 | const garnet_common = new GarnetCommon(this, 'CommonContructs', {}) 30 | 31 | 32 | const garnet_broker_stack = new GarnetScorpio(this, 'ScorpioBroker', { 33 | vpc: garnet_common.vpc, 34 | secret: garnet_common.secret, 35 | delivery_stream: garnet_datalake.delivery_stream 36 | }) 37 | 38 | const garnet_ingestion_stack = new GarnetIngestionStack(this, 'GarnetIngestion', { 39 | dns_context_broker: garnet_broker_stack.dns_context_broker, 40 | vpc: garnet_common.vpc, 41 | 42 | }) 43 | 44 | const garnet_iot_stack = new GarnetIot(this, 'GarnetIoT', { 45 | vpc: garnet_common.vpc, 46 | dns_context_broker: garnet_broker_stack.dns_context_broker, 47 | }) 48 | 49 | const garnet_privatesub = new GarnetPrivateSub(this, 'GarnetPrivateSub', { 50 | vpc: garnet_common.vpc, 51 | bucket_name: garnet_datalake.bucket_name 52 | }) 53 | 54 | const garnet_api = new GarnetApi(this, 'GarnetApi', { 55 | vpc: garnet_common.vpc, 56 | garnet_ingestion_sqs: garnet_ingestion_stack.sqs_garnet_ingestion, 57 | dns_context_broker: garnet_broker_stack.dns_context_broker, 58 | garnet_private_endpoint: garnet_privatesub.private_sub_endpoint, 59 | fargate_alb: garnet_broker_stack.fargate_alb, 60 | secret_api_jwt: garnet_common.secret_api_jwt 61 | }) 62 | 63 | const garnet_ops_stack = new GarnetOps(this, 'GarnetOps', {}) 64 | 65 | new CfnOutput(this, 'GarnetVersion', { 66 | value: garnet_constant.garnet_version, 67 | description: 'Version of Garnet Framework' 68 | }) 69 | new CfnOutput(this, 'GarnetArchitecture', { 70 | value: deployment_params.architecture, 71 | description: 'Architecture deployed' 72 | }) 73 | new CfnOutput(this, 'GarnetEndpoint', { 74 | value: garnet_api.broker_api_endpoint, 75 | description: 'Garnet Unified API' 76 | }) 77 | new CfnOutput(this, 'GarnetApiToken', { 78 | value: garnet_api.garnet_api_token, 79 | description: `Authentication token for Garnet API. Use in HTTP headers as: Authorization: . 80 | Example: curl -H "Authorization: " ` 81 | }) 82 | new CfnOutput(this, 'GarnetPrivateSubEndpoint', { 83 | value: garnet_privatesub.private_sub_endpoint, 84 | description: 'Garnet Private Notification Endpoint for Secured Subscriptions. Only accessible within the Garnet VPC' 85 | }) 86 | new CfnOutput(this, 'GarnetIngestionQueue', { 87 | value: garnet_ingestion_stack.sqs_garnet_ingestion.queueUrl, 88 | description: 'Garnet SQS Queue URL to ingest data from your Data Producers' 89 | }) 90 | 91 | new CfnOutput(this, 'BucketDatalakeName', { 92 | value: garnet_datalake.bucket_name, 93 | description: 'Name of the S3 Bucket for the datalake' 94 | }) 95 | 96 | 97 | 98 | 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/bucket/bucket-construct.ts: -------------------------------------------------------------------------------- 1 | import { CustomResource, Duration, Names, RemovalPolicy } from "aws-cdk-lib"; 2 | import { Runtime, Function, Code, Architecture } from "aws-cdk-lib/aws-lambda"; 3 | import { Construct } from "constructs"; 4 | import { Parameters } from "../../../../configuration"; 5 | import { PolicyStatement} from "aws-cdk-lib/aws-iam"; 6 | import { Provider } from "aws-cdk-lib/custom-resources"; 7 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose"; 8 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 9 | import {garnet_bucket, garnet_bucket_athena, garnet_nomenclature } from '../../../../constants' 10 | 11 | export interface GarnetBucketProps { 12 | 13 | } 14 | 15 | 16 | export class GarnetBucket extends Construct { 17 | public readonly bucket_name: string 18 | constructor(scope: Construct, id: string, props: GarnetBucketProps) { 19 | super(scope, id) 20 | 21 | 22 | // CUSTOM RESOURCE WITH A LAMBDA THAT WILL CREATE GARNET BUCKET AND ATHENA RESULTS BUCKET IF NOT EXISTS 23 | const lambda_bucket_logs = new LogGroup(this, 'LambdaBucketCreateFunctionLogs', { 24 | retention: RetentionDays.ONE_MONTH, 25 | removalPolicy: RemovalPolicy.DESTROY 26 | }) 27 | 28 | const lambda_bucket_path = `${__dirname}/lambda/bucketCreate` 29 | const lambda_bucket = new Function(this, 'BucketCreateFunction', { 30 | functionName: garnet_nomenclature.garnet_utils_bucket_create_lambda, 31 | description: 'Garnet Utils - Function that creates Garnet Bucket if it does not exist', 32 | runtime: Runtime.NODEJS_22_X, 33 | code: Code.fromAsset(lambda_bucket_path), 34 | handler: 'index.handler', 35 | timeout: Duration.seconds(50), 36 | logGroup: lambda_bucket_logs, 37 | architecture: Architecture.ARM_64, 38 | environment: { 39 | BUCKET_NAME: garnet_bucket, 40 | BUCKET_ATHENA_NAME: garnet_bucket_athena 41 | } 42 | }) 43 | 44 | lambda_bucket.node.addDependency(lambda_bucket_logs) 45 | 46 | lambda_bucket.addToRolePolicy(new PolicyStatement({ 47 | actions: [ 48 | "s3:CreateBucket", 49 | "s3:PutMetricsConfiguration", 50 | "s3:HeadBucket", 51 | "s3:ListBucket" 52 | ], 53 | resources: ["arn:aws:s3:::*"] 54 | })) 55 | 56 | 57 | const lambda_bucket_check_logs = new LogGroup(this, 'LambdaBucketCheckFunctionLogs', { 58 | retention: RetentionDays.ONE_MONTH, 59 | removalPolicy: RemovalPolicy.DESTROY 60 | }) 61 | 62 | const lambda_bucket_check_path = `${__dirname}/lambda/bucketCheck` 63 | const lambda_bucket_check = new Function(this, 'BucketCheckFunction', { 64 | functionName: `garnet-utils-bucket-check-lambda`, 65 | description: 'Garnet Utils - Function that check if Garnet Bucket exists', 66 | runtime: Runtime.NODEJS_22_X, 67 | code: Code.fromAsset(lambda_bucket_check_path), 68 | handler: 'index.handler', 69 | timeout: Duration.seconds(50), 70 | logGroup: lambda_bucket_check_logs, 71 | architecture: Architecture.ARM_64, 72 | environment: { 73 | BUCKET_NAME: garnet_bucket, 74 | BUCKET_ATHENA_NAME: garnet_bucket_athena 75 | } 76 | }) 77 | 78 | lambda_bucket_check.node.addDependency(lambda_bucket_check_logs) 79 | 80 | lambda_bucket_check.addToRolePolicy(new PolicyStatement({ 81 | actions: [ 82 | "s3:HeadBucket", 83 | "s3:ListBucket" 84 | ], 85 | resources: ["arn:aws:s3:::*"] 86 | })) 87 | 88 | 89 | 90 | const bucket_provider_log = new LogGroup(this, 'LambdaCustomBucketProviderLogs', { 91 | retention: RetentionDays.ONE_MONTH, 92 | // logGroupName: `garnet-provider-custom-bucket-lambda-cw-logs`, 93 | removalPolicy: RemovalPolicy.DESTROY 94 | }) 95 | 96 | const bucket_provider = new Provider(this, 'CustomBucketProvider', { 97 | onEventHandler: lambda_bucket, 98 | isCompleteHandler: lambda_bucket_check, 99 | providerFunctionName: garnet_nomenclature.garnet_utils_bucket_provider, 100 | logGroup: bucket_provider_log 101 | }) 102 | 103 | bucket_provider.node.addDependency(bucket_provider_log) 104 | 105 | const bucket_resource = new CustomResource(this, 'CustomBucketProviderResource', { 106 | serviceToken: bucket_provider.serviceToken, 107 | 108 | }) 109 | 110 | this.bucket_name = bucket_resource.getAtt('bucket_name').toString() 111 | 112 | } 113 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-ingestion/garnet-ingestion-stack.ts: -------------------------------------------------------------------------------- 1 | import { Duration, NestedStack, NestedStackProps, RemovalPolicy, Stack } from 'aws-cdk-lib' 2 | import { SubnetType, Vpc } from 'aws-cdk-lib/aws-ec2' 3 | import { Code, LayerVersion, Runtime, Function,Architecture} from "aws-cdk-lib/aws-lambda" 4 | import { Queue } from "aws-cdk-lib/aws-sqs" 5 | import { garnet_nomenclature } from '../../../constants' 6 | import { LogGroup, RetentionDays } from 'aws-cdk-lib/aws-logs' 7 | import { Parameters } from '../../../configuration' 8 | import { PolicyStatement } from 'aws-cdk-lib/aws-iam' 9 | import { deployment_params } from '../../../architecture' 10 | import { SqsEventSource } from 'aws-cdk-lib/aws-lambda-event-sources' 11 | 12 | export interface GarnetIngestionStackProps extends NestedStackProps { 13 | dns_context_broker: string, 14 | vpc: Vpc 15 | } 16 | 17 | export class GarnetIngestionStack extends NestedStack { 18 | 19 | public readonly sqs_garnet_ingestion: Queue 20 | 21 | constructor(scope: Stack, id: string, props: GarnetIngestionStackProps) { 22 | super(scope, id, props) 23 | 24 | //CHECK PROPS 25 | if (!props.vpc) { 26 | throw new Error( 27 | "The property vpc is required to create an instance of the construct" 28 | ); 29 | } 30 | if (!props.dns_context_broker) { 31 | throw new Error( 32 | "The property dns_context_broker is required to create an instance of the construct" 33 | ); 34 | } 35 | 36 | // LAMBDA LAYER (SHARED LIBRARIES) 37 | const layer_lambda_path = `./lib/layers`; 38 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 39 | code: Code.fromAsset(layer_lambda_path), 40 | compatibleRuntimes: [Runtime.NODEJS_22_X], 41 | }) 42 | 43 | // SQS ENTRY POINT 44 | const sqs_garnet_endpoint = new Queue(this, "SqsGarnetIot", { 45 | queueName: garnet_nomenclature.garnet_ingestion_queue, 46 | visibilityTimeout: Duration.seconds(55) 47 | }) 48 | 49 | 50 | // LAMBDA THAT GETS MESSAGES FROM THE QUEUE AND UPDATES CONTEXT BROKER 51 | const lambda_to_context_broker_log = new LogGroup(this, 'LambdaIngestionUpdateContextBrokerLogs', { 52 | retention: RetentionDays.THREE_MONTHS, 53 | removalPolicy: RemovalPolicy.DESTROY 54 | }) 55 | const lambda_to_context_broker_path = `${__dirname}/lambda/updateContextBroker`; 56 | const lambda_to_context_broker = new Function(this,"LambdaIngestionUpdateContextBroker", { 57 | functionName: garnet_nomenclature.garnet_ingestion_update_broker_lambda, 58 | description: 'Garnet Ingestion- Function that updates the context broker', 59 | vpc: props.vpc, 60 | vpcSubnets: { 61 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 62 | }, 63 | runtime: Runtime.NODEJS_22_X, 64 | code: Code.fromAsset(lambda_to_context_broker_path), 65 | handler: "index.handler", 66 | timeout: Duration.seconds(50), 67 | logGroup: lambda_to_context_broker_log, 68 | layers: [layer_lambda], 69 | architecture: Architecture.ARM_64, 70 | environment: { 71 | DNS_CONTEXT_BROKER: props.dns_context_broker 72 | } 73 | } 74 | ) 75 | lambda_to_context_broker.node.addDependency(lambda_to_context_broker_log) 76 | lambda_to_context_broker.addToRolePolicy( 77 | new PolicyStatement({ 78 | actions: [ 79 | "logs:CreateLogGroup", 80 | "logs:CreateLogStream", 81 | "logs:PutLogEvents", 82 | "ec2:CreateNetworkInterface", 83 | "ec2:DescribeNetworkInterfaces", 84 | "ec2:DeleteNetworkInterface", 85 | "ec2:AssignPrivateIpAddresses", 86 | "ec2:UnassignPrivateIpAddresses", 87 | ], 88 | resources: ["*"], 89 | }) 90 | ) 91 | 92 | // ADD PERMISSION FOR LAMBDA TO ACCESS SQS 93 | lambda_to_context_broker.addToRolePolicy( 94 | new PolicyStatement({ 95 | actions: [ 96 | "sqs:ReceiveMessage", 97 | "sqs:DeleteMessage", 98 | "sqs:GetQueueAttributes", 99 | ], 100 | resources: [`${sqs_garnet_endpoint.queueArn}`], 101 | }) 102 | ) 103 | 104 | lambda_to_context_broker.addEventSource( 105 | new SqsEventSource(sqs_garnet_endpoint, { 106 | batchSize: deployment_params.lambda_broker_batch_size, 107 | maxBatchingWindow: Duration.seconds(deployment_params.lambda_broker_batch_window), 108 | maxConcurrency: deployment_params.lambda_broker_concurent_sqs 109 | }) 110 | ) 111 | 112 | this.sqs_garnet_ingestion = sqs_garnet_endpoint 113 | 114 | 115 | } 116 | } 117 | 118 | -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apicommon/api-common-construct.ts: -------------------------------------------------------------------------------- 1 | import { Aws, Duration, RemovalPolicy } from "aws-cdk-lib"; 2 | import { CfnIntegration, CfnRoute } from "aws-cdk-lib/aws-apigatewayv2"; 3 | import { Vpc } from "aws-cdk-lib/aws-ec2"; 4 | import { 5 | Runtime, 6 | Function, 7 | Code, 8 | CfnPermission, 9 | LayerVersion, 10 | Architecture, 11 | } from "aws-cdk-lib/aws-lambda"; 12 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 13 | import { Construct } from "constructs"; 14 | import { 15 | garnet_broker, 16 | garnet_constant, 17 | garnet_nomenclature, 18 | } from "../../../../constants"; 19 | import { deployment_params } from "../../../../architecture"; 20 | import { Queue } from "aws-cdk-lib/aws-sqs"; 21 | 22 | export interface GarnetApiCommonProps { 23 | readonly api_ref: string; 24 | readonly vpc: Vpc; 25 | dns_context_broker: string; 26 | garnet_ingestion_sqs: Queue; 27 | garnet_private_endpoint: string; 28 | } 29 | 30 | export class GarnetApiCommon extends Construct { 31 | constructor(scope: Construct, id: string, props: GarnetApiCommonProps) { 32 | super(scope, id); 33 | 34 | const sqs_ingestion = props.garnet_ingestion_sqs; 35 | 36 | // LAMBDA LAYER (SHARED LIBRARIES) 37 | const layer_lambda_path = `./lib/layers`; 38 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 39 | code: Code.fromAsset(layer_lambda_path), 40 | compatibleRuntimes: [Runtime.NODEJS_22_X], 41 | }); 42 | 43 | // ********************************************** 44 | 45 | /** 46 | * GARNET VERSION 47 | */ 48 | 49 | // LAMBDA GARNET API VERSION 50 | const lambda_garnet_version_log = new LogGroup( 51 | this, 52 | "LambdaGarnetVersionLogs", 53 | { 54 | retention: RetentionDays.THREE_MONTHS, 55 | removalPolicy: RemovalPolicy.DESTROY, 56 | } 57 | ); 58 | const lambda_garnet_version_path = `${__dirname}/lambda/garnetVersion`; 59 | const lambda_garnet_version = new Function(this, "LambdaGarnetVersion", { 60 | functionName: `garnet-api-version-lambda`, 61 | vpc: props.vpc, 62 | description: "Garnet API - Function that returns the Garnet Version", 63 | runtime: Runtime.NODEJS_22_X, 64 | code: Code.fromAsset(lambda_garnet_version_path), 65 | handler: "index.handler", 66 | timeout: Duration.seconds(30), 67 | logGroup: lambda_garnet_version_log, 68 | layers: [layer_lambda], 69 | architecture: Architecture.ARM_64, 70 | environment: { 71 | CONTEXT_BROKER: garnet_broker, 72 | GARNET_VERSION: garnet_constant.garnet_version, 73 | GARNET_PRIVATE_ENDPOINT: props.garnet_private_endpoint, 74 | GARNET_INGESTION_SQS_URL: sqs_ingestion.queueUrl, 75 | GARNET_INGESTION_SQS_ARN: sqs_ingestion.queueArn, 76 | DNS_CONTEXT_BROKER: props.dns_context_broker, 77 | GARNET_ARCHITECTURE: deployment_params.architecture, 78 | GARNET_CONTAINERS: 79 | deployment_params.architecture == "distributed" 80 | ? JSON.stringify([ 81 | `${garnet_nomenclature.garnet_broker_atcontextserver}`, 82 | `${garnet_nomenclature.garnet_broker_entitymanager}`, 83 | `${garnet_nomenclature.garnet_broker_historyentitymanager}`, 84 | `${garnet_nomenclature.garnet_broker_historyquerymanager}`, 85 | `${garnet_nomenclature.garnet_broker_querymanager}`, 86 | `${garnet_nomenclature.garnet_broker_registrymanager}`, 87 | `${garnet_nomenclature.garnet_broker_registrysubscriptionmanager}`, 88 | `${garnet_nomenclature.garnet_broker_subscriptionmanager}`, 89 | ]) 90 | : JSON.stringify([`${garnet_nomenclature.garnet_broker_allinone}`]), 91 | }, 92 | }); 93 | lambda_garnet_version.node.addDependency(lambda_garnet_version_log); 94 | const garnet_version_integration = new CfnIntegration( 95 | this, 96 | "GarnetVersionIntegration", 97 | { 98 | apiId: props.api_ref, 99 | integrationMethod: "GET", 100 | integrationType: "AWS_PROXY", 101 | integrationUri: lambda_garnet_version.functionArn, 102 | connectionType: "INTERNET", 103 | description: "GARNET VERSION INTEGRATION", 104 | payloadFormatVersion: "1.0", 105 | } 106 | ); 107 | 108 | const garnet_version_route = new CfnRoute(this, "GarnetVersionRoute", { 109 | apiId: props.api_ref, 110 | routeKey: "GET /", 111 | target: `integrations/${garnet_version_integration.ref}`, 112 | }); 113 | 114 | new CfnPermission(this, "ApiGatewayLambdaPermissionGarnetVersion", { 115 | principal: `apigateway.amazonaws.com`, 116 | action: "lambda:InvokeFunction", 117 | functionName: lambda_garnet_version.functionName, 118 | sourceArn: `arn:aws:execute-api:${Aws.REGION}:${Aws.ACCOUNT_ID}:${props.api_ref}/*/*/*`, 119 | }); 120 | 121 | /** 122 | * END GARNET VERSION 123 | */ 124 | 125 | // ********************************************** 126 | } 127 | } 128 | -------------------------------------------------------------------------------- /lib/stacks/garnet-lake/stream/firehose-stream-construct.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs" 2 | import { Runtime, Function, Code, Architecture,LayerVersion } from "aws-cdk-lib/aws-lambda" 3 | import { Bucket } from "aws-cdk-lib/aws-s3" 4 | import { PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam" 5 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs" 6 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose" 7 | import { Duration, RemovalPolicy } from "aws-cdk-lib" 8 | import { garnet_nomenclature } from "../../../../constants" 9 | 10 | export interface GarnetDataLakeStreamProps { 11 | readonly bucket_name: string 12 | } 13 | 14 | 15 | export class GarnetDataLakeStream extends Construct { 16 | public readonly datalake_kinesis_firehose_delivery_stream : CfnDeliveryStream 17 | 18 | constructor(scope: Construct, id: string, props: GarnetDataLakeStreamProps) { 19 | super(scope, id) 20 | 21 | 22 | // LAMBDA LAYER (SHARED LIBRARIES) 23 | const layer_lambda_path = `./lib/layers`; 24 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 25 | code: Code.fromAsset(layer_lambda_path), 26 | compatibleRuntimes: [Runtime.NODEJS_22_X], 27 | }) 28 | 29 | // KINESIS FIREHOSE TO DATALAKE BUCKET 30 | 31 | // DATALAKE BUCKET 32 | const bucket = Bucket.fromBucketName(this, "GarnetBucket", props.bucket_name) 33 | 34 | 35 | // ROLE THAT GRANTS ACCESS TO FIREHOSE TO READ/WRITE BUCKET 36 | const role_firehose = new Role(this, "FirehoseRole", { 37 | assumedBy: new ServicePrincipal("firehose.amazonaws.com") 38 | }) 39 | bucket.grantReadWrite(role_firehose) 40 | 41 | // LAMBDA THAT TRANSFORMS IN FIREHOSE STREAM 42 | const lambda_transform_logs = new LogGroup(this, 'LambdaBucketHeadFunctionLogs', { 43 | retention: RetentionDays.ONE_MONTH, 44 | // logGroupName: `${garnet_nomenclature.garnet_lake_transform_lambda}-cw-logs`, 45 | removalPolicy: RemovalPolicy.DESTROY 46 | }) 47 | const lambda_transform_path = `${__dirname}/lambda/transform` 48 | const lambda_transform = new Function(this, 'LakeTransformLambda', { 49 | functionName: garnet_nomenclature.garnet_lake_transform_lambda, 50 | description: 'Garnet Lake - Function that transforms the Kinesis Firehose records to extract entities from notifications', 51 | runtime: Runtime.NODEJS_22_X, 52 | layers: [layer_lambda], 53 | code: Code.fromAsset(lambda_transform_path), 54 | handler: 'index.handler', 55 | timeout: Duration.minutes(1), 56 | architecture: Architecture.ARM_64, 57 | logGroup: lambda_transform_logs 58 | }) 59 | 60 | lambda_transform.node.addDependency(lambda_transform_logs) 61 | 62 | // KINESIS FIREHOSE DELIVERY STREAM 63 | const kinesis_firehose = new CfnDeliveryStream( this, "GarnetFirehose", { 64 | deliveryStreamName: garnet_nomenclature.garnet_lake_firehose_stream, 65 | deliveryStreamType: "DirectPut", 66 | extendedS3DestinationConfiguration: { 67 | bucketArn: bucket.bucketArn, 68 | roleArn: role_firehose.roleArn, 69 | bufferingHints: { 70 | intervalInSeconds: garnet_nomenclature.garnet_lake_firehose_interval, 71 | sizeInMBs: garnet_nomenclature.garnet_lake_buffer_size, 72 | }, 73 | processingConfiguration: { 74 | enabled: true, 75 | processors: [ 76 | { 77 | type: "RecordDeAggregation", 78 | parameters: [ 79 | { 80 | parameterName: "SubRecordType", 81 | parameterValue: "JSON", 82 | } 83 | ] 84 | }, 85 | { 86 | type: 'Lambda', 87 | parameters: [{ 88 | parameterName: 'LambdaArn', 89 | parameterValue: lambda_transform.functionArn 90 | }] 91 | }, 92 | { 93 | type: 'AppendDelimiterToRecord', 94 | parameters: [ 95 | { 96 | parameterName: 'Delimiter', 97 | parameterValue: '\\n', 98 | }, 99 | ], 100 | } 101 | // ,{ 102 | // type: "MetadataExtraction", 103 | // parameters: [ 104 | // { 105 | // parameterName: "MetadataExtractionQuery", 106 | // parameterValue: "{type:.type}", 107 | // }, 108 | // { 109 | // parameterName: "JsonParsingEngine", 110 | // parameterValue: "JQ-1.6", 111 | // }, 112 | // ], 113 | // } 114 | ], 115 | }, 116 | dynamicPartitioningConfiguration: { 117 | enabled: true, 118 | }, 119 | prefix: `type=!{partitionKeyFromLambda:type}/dt=!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/`, 120 | errorOutputPrefix: `type=!{firehose:error-output-type}/dt=!{timestamp:yyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/`, 121 | }, 122 | } 123 | ) 124 | 125 | lambda_transform.grantInvoke(role_firehose) 126 | 127 | this.datalake_kinesis_firehose_delivery_stream = kinesis_firehose 128 | 129 | 130 | } 131 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apiauth/api-auth-construct.ts: -------------------------------------------------------------------------------- 1 | import { CustomResource, Duration, RemovalPolicy } from "aws-cdk-lib" 2 | import { Code, LayerVersion, Runtime, Function, Architecture } from "aws-cdk-lib/aws-lambda" 3 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs" 4 | import { Secret } from "aws-cdk-lib/aws-secretsmanager" 5 | import { Construct } from "constructs" 6 | import { garnet_nomenclature } from "../../../../constants" 7 | import { Provider } from "aws-cdk-lib/custom-resources" 8 | import { PolicyStatement, ServicePrincipal } from "aws-cdk-lib/aws-iam" 9 | 10 | export interface GarnetApiAuthJwtProps { 11 | secret_api_jwt: Secret 12 | } 13 | 14 | export class GarnetApiAuthJwt extends Construct { 15 | public readonly garnet_api_token: string 16 | public readonly lambda_authorizer_arn: string 17 | constructor(scope: Construct, id: string, props: GarnetApiAuthJwtProps){ 18 | super(scope, id) 19 | 20 | 21 | 22 | 23 | 24 | // LAMBDA LAYER (SHARED LIBRARIES) 25 | const layer_lambda_path = `./lib/layers`; 26 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 27 | code: Code.fromAsset(layer_lambda_path), 28 | compatibleRuntimes: [Runtime.NODEJS_22_X], 29 | }) 30 | 31 | 32 | // Logs for the lambda that generates the JWT 33 | const api_auth_jwt_generator_logs = new LogGroup(this, 'ApiAuthJwtGeneratorLogs', { 34 | retention: RetentionDays.ONE_MONTH, 35 | removalPolicy: RemovalPolicy.DESTROY 36 | }) 37 | 38 | const api_auth_jwt_generator_lambda_path = `${__dirname}/lambda/apiAuthJwt` 39 | const api_auth_jwt_generator_lambda = new Function(this, 'ApiAuthJwtGeneratorLambda', { 40 | functionName: garnet_nomenclature.garnet_api_auth_jwt_lambda, 41 | description: 'Garnet API - Function that generates a JWT token', 42 | runtime: Runtime.NODEJS_22_X, 43 | logGroup: api_auth_jwt_generator_logs, 44 | layers: [layer_lambda], 45 | code: Code.fromAsset(api_auth_jwt_generator_lambda_path), 46 | handler: 'index.handler', 47 | timeout: Duration.seconds(50), 48 | architecture: Architecture.ARM_64, 49 | environment: { 50 | SECRET_ARN: props.secret_api_jwt.secretArn, 51 | JWT_SUB: garnet_nomenclature.garnet_api_auth_sub, 52 | JWT_ISS: garnet_nomenclature.garnet_api_auth_issuer, 53 | JWT_AUD: garnet_nomenclature.garnet_api_auth_audience 54 | } 55 | }) 56 | 57 | api_auth_jwt_generator_lambda.node.addDependency(api_auth_jwt_generator_logs) 58 | 59 | props.secret_api_jwt.grantRead(api_auth_jwt_generator_lambda) 60 | 61 | const api_auth_jwt_generator_provider_logs = new LogGroup(this, 'LambdaJwtAuthProviderLogs', { 62 | retention: RetentionDays.ONE_MONTH, 63 | removalPolicy: RemovalPolicy.DESTROY 64 | }) 65 | 66 | const api_auth_jwt_generator_provider = new Provider(this, 'LambdaAuthJwtProvider', { 67 | onEventHandler: api_auth_jwt_generator_lambda, 68 | logGroup: api_auth_jwt_generator_provider_logs 69 | }) 70 | api_auth_jwt_generator_provider.node.addDependency(api_auth_jwt_generator_provider_logs) 71 | 72 | const api_auth_jwt_generator_resource = new CustomResource(this, 'ApiJwtAuthResource', { 73 | serviceToken: api_auth_jwt_generator_provider.serviceToken 74 | }) 75 | 76 | this.garnet_api_token = api_auth_jwt_generator_resource.getAttString('token') 77 | 78 | 79 | // Logs for the lambda authorizer 80 | const api_authorizer_logs = new LogGroup(this, 'ApiAuthorizerLogs', { 81 | retention: RetentionDays.ONE_MONTH, 82 | removalPolicy: RemovalPolicy.DESTROY 83 | }) 84 | 85 | const api_authorizer_lambda_path = `${__dirname}/lambda/apiAuthorizer` 86 | const api_authorizer_lambda = new Function(this, 'ApiAuthorizerLambda', { 87 | functionName: garnet_nomenclature.garnet_api_authorizer_lambda, 88 | description: 'Garnet API - Lambda Authorizer for the Garnet API', 89 | runtime: Runtime.NODEJS_22_X, 90 | logGroup: api_authorizer_logs, 91 | layers: [layer_lambda], 92 | code: Code.fromAsset(api_authorizer_lambda_path), 93 | handler: 'index.handler', 94 | timeout: Duration.seconds(50), 95 | architecture: Architecture.ARM_64, 96 | environment: { 97 | SECRET_ARN: props.secret_api_jwt.secretArn, 98 | JWT_SUB: garnet_nomenclature.garnet_api_auth_sub, 99 | JWT_ISS: garnet_nomenclature.garnet_api_auth_issuer, 100 | JWT_AUD: garnet_nomenclature.garnet_api_auth_audience 101 | } 102 | }) 103 | 104 | api_authorizer_lambda.node.addDependency(api_authorizer_logs) 105 | 106 | props.secret_api_jwt.grantRead(api_authorizer_lambda) 107 | api_authorizer_lambda.grantInvoke(new ServicePrincipal('apigateway.amazonaws.com')) 108 | 109 | this.lambda_authorizer_arn = api_authorizer_lambda.functionArn 110 | } 111 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-scorpio/database/database-construct.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CfnOutput, Duration, Names, Token } from "aws-cdk-lib" 2 | import { SecurityGroup, SubnetType, Vpc, Port } from "aws-cdk-lib/aws-ec2" 3 | import { AuroraPostgresEngineVersion, CaCertificate, CfnDBProxyEndpoint, ClusterInstance, Credentials, DatabaseCluster, DatabaseClusterEngine, DatabaseProxy, ParameterGroup, ProxyTarget } from "aws-cdk-lib/aws-rds" 4 | import { Secret } from "aws-cdk-lib/aws-secretsmanager" 5 | 6 | import { Construct } from "constructs" 7 | import { deployment_params } from "../../../../architecture" 8 | import { Role, ServicePrincipal } from "aws-cdk-lib/aws-iam" 9 | import { garnet_broker, garnet_constant, garnet_nomenclature } from "../../../../constants" 10 | import { Alarm } from "aws-cdk-lib/aws-cloudwatch" 11 | import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources" 12 | 13 | export interface GarnetScorpioDatabaseProps { 14 | vpc: Vpc 15 | secret_arn: string 16 | } 17 | 18 | export class GarnetScorpioDatabase extends Construct{ 19 | 20 | public readonly database_endpoint: string 21 | // public readonly database_reader_endpoint: string 22 | public readonly database_port: string 23 | public readonly sg_proxy: SecurityGroup 24 | 25 | constructor(scope: Construct, id: string, props: GarnetScorpioDatabaseProps) { 26 | super(scope, id) 27 | 28 | // Check props 29 | if (!props.vpc){ 30 | throw new Error('The property vpc is required to create an instance of ScorpioDatabase Construct') 31 | } 32 | if (!props.secret_arn){ 33 | throw new Error('The property secret_arn is required to create an instance of ScorpioDatabase Construct') 34 | } 35 | 36 | const secret = Secret.fromSecretCompleteArn(this, 'Secret', props.secret_arn) 37 | const sg_database = new SecurityGroup(this, 'SecurityGroupDatabase', { 38 | vpc: props.vpc, 39 | securityGroupName: garnet_nomenclature.garnet_broker_sg_database 40 | }) 41 | 42 | const engine = DatabaseClusterEngine.auroraPostgres({ version: AuroraPostgresEngineVersion.VER_16_6 }) 43 | 44 | // Parameter Group 45 | 46 | const parameterGroup = new ParameterGroup(this, 'ParameterGroup', { 47 | engine, 48 | // parameters: { 49 | // 'rds.force_ssl': '0' 50 | // } 51 | }) 52 | 53 | // Aurora Cluster 54 | const cluster = new DatabaseCluster(this, 'DatabaseAurora', { 55 | engine, 56 | parameterGroup, 57 | clusterIdentifier: garnet_nomenclature.garnet_db_cluster_id, 58 | credentials: Credentials.fromSecret(secret), 59 | vpc: props.vpc, 60 | securityGroups: [sg_database], 61 | defaultDatabaseName: garnet_constant.dbname, 62 | vpcSubnets:{ 63 | subnetType: SubnetType.PRIVATE_ISOLATED 64 | }, 65 | writer: ClusterInstance.serverlessV2('writer', { 66 | caCertificate: CaCertificate.RDS_CA_RSA4096_G1 67 | }), 68 | // readers: [ClusterInstance.serverlessV2('reader', { 69 | // scaleWithWriter: true, 70 | // caCertificate: CaCertificate.RDS_CA_RSA4096_G1 71 | // })], 72 | serverlessV2MinCapacity: deployment_params.aurora_min_capacity!, 73 | serverlessV2MaxCapacity: deployment_params.aurora_max_capacity!, 74 | storageType: deployment_params.aurora_storage_type! 75 | }) 76 | 77 | // ROLE FOR PROXY 78 | const role_proxy = new Role(this, 'RoleRdsProxy', { 79 | roleName: `garnet-rds-proxy-role-${Aws.REGION}`, 80 | assumedBy: new ServicePrincipal("rds.amazonaws.com") 81 | }) 82 | secret.grantRead(role_proxy) 83 | 84 | // SECURITY GROUP FOR PROXY 85 | const sg_proxy = new SecurityGroup(this, 'SecurityGroupProxyDatabase', { 86 | vpc: props.vpc, 87 | securityGroupName: garnet_nomenclature.garnet_broker_sg_rds, 88 | allowAllOutbound: true 89 | }) 90 | 91 | sg_database.addIngressRule(sg_proxy, Port.tcp(5432)) 92 | this.sg_proxy = sg_proxy 93 | 94 | // RDS Proxy 95 | const rds_proxy = new DatabaseProxy(this, 'RdsProxy', { 96 | dbProxyName: garnet_nomenclature.garnet_proxy_rds, 97 | proxyTarget: ProxyTarget.fromCluster(cluster), 98 | secrets: [secret], 99 | maxConnectionsPercent: 100, 100 | debugLogging: true, 101 | vpc: props.vpc, 102 | idleClientTimeout: Duration.minutes(5), 103 | requireTLS: false, 104 | role: role_proxy, 105 | securityGroups: [sg_proxy] 106 | }) 107 | 108 | rds_proxy.node.addDependency(cluster) 109 | 110 | // // RDS Read OnlyProxy 111 | // const readOnlyEndpoint = new CfnDBProxyEndpoint(this, 'RdsProxyReadOnlyEndpoint', { 112 | // dbProxyEndpointName: `${garnet_nomenclature.garnet_proxy_rds}-readonly`, 113 | // dbProxyName: rds_proxy.dbProxyName, 114 | // targetRole: 'READ_ONLY', 115 | // vpcSubnetIds: props.vpc.privateSubnets.map(subnet => subnet.subnetId), 116 | // vpcSecurityGroupIds: [sg_proxy.securityGroupId] 117 | // }) 118 | 119 | 120 | // Add CloudWatch alarms for key metrics 121 | new Alarm(this, 'DatabaseConnectionsAlarm', { 122 | metric: cluster.metricDatabaseConnections(), 123 | threshold: 900, // 90% of max connections 124 | evaluationPeriods: 3, 125 | datapointsToAlarm: 2 126 | }) 127 | 128 | this.database_endpoint = rds_proxy.endpoint 129 | // this.database_reader_endpoint = readOnlyEndpoint.attrEndpoint 130 | this.database_port = `${Token.asString(cluster.clusterEndpoint.port)}` 131 | 132 | new CfnOutput(this, 'database_proxy_endpoint', { 133 | value: this.database_endpoint 134 | 135 | }) 136 | new CfnOutput(this, 'database_port', { 137 | value: this.database_port 138 | }) 139 | 140 | 141 | 142 | } 143 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-api/apigateway/api-gateway-construct.ts: -------------------------------------------------------------------------------- 1 | 2 | import { CfnAuthorizer as CfnAuthorizerV2, CfnIntegration, CfnRoute, CfnStage, CfnVpcLink, CorsHttpMethod, HttpApi } from "aws-cdk-lib/aws-apigatewayv2" 3 | import { SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2" 4 | import { Construct } from "constructs" 5 | import { Function as LambdaFunction, Runtime, Code, Permission } from 'aws-cdk-lib/aws-lambda' 6 | import { ApplicationLoadBalancer } from "aws-cdk-lib/aws-elasticloadbalancingv2" 7 | import { HttpLambdaAuthorizer, HttpLambdaResponseType } from "aws-cdk-lib/aws-apigatewayv2-authorizers" 8 | import { ServicePrincipal } from "aws-cdk-lib/aws-iam" 9 | import { Aws, Duration } from "aws-cdk-lib" 10 | import { AuthorizationType, CfnAuthorizer } from "aws-cdk-lib/aws-apigateway" 11 | import { Parameters } from "../../../../configuration" 12 | 13 | export interface GarnetApiGatewayProps { 14 | readonly vpc: Vpc, 15 | readonly fargate_alb: ApplicationLoadBalancer 16 | readonly lambda_authorizer_arn: string 17 | } 18 | 19 | export class GarnetApiGateway extends Construct{ 20 | public readonly api_ref: string 21 | constructor(scope: Construct, id: string, props: GarnetApiGatewayProps) { 22 | super(scope, id) 23 | // Check props 24 | if (!props.vpc){ 25 | throw new Error('The property vpc is required') 26 | } 27 | if (!props.fargate_alb){ 28 | throw new Error('The property fargate_alb is required') 29 | } 30 | if (!props.lambda_authorizer_arn) { 31 | throw new Error('The property lambda_authorizer_arn is required') 32 | } 33 | 34 | const sg_vpc_link = new SecurityGroup(this, 'SgVpcLink', { 35 | securityGroupName: `garnet-vpclink-sg`, 36 | vpc: props.vpc 37 | }) 38 | 39 | 40 | 41 | const vpc_link = new CfnVpcLink(this, 'VpcLink', { 42 | name: `garnet-vpc-link`, 43 | subnetIds: props.vpc.privateSubnets.map( (m) => m.subnetId), 44 | securityGroupIds: [sg_vpc_link.securityGroupId] 45 | }) 46 | 47 | // Create HTTP API with CORS and default authorizer 48 | const api = new HttpApi(this, 'HttpApi', { 49 | apiName: 'garnet-api', 50 | corsPreflight: { 51 | maxAge: Duration.seconds(5), 52 | exposeHeaders: ['*'], 53 | allowHeaders: ['*', 'Authorization', 'Content-Type'], 54 | allowMethods: [CorsHttpMethod.GET, CorsHttpMethod.OPTIONS], 55 | allowOrigins: ['*'] 56 | }, 57 | createDefaultStage: true 58 | }) 59 | 60 | const lambda_authorizer = LambdaFunction.fromFunctionArn(this, 'LambdaAuthorizer', props.lambda_authorizer_arn) 61 | 62 | const integration = new CfnIntegration(this, 'HttpApiIntegration', { 63 | apiId: api.apiId, 64 | integrationMethod: "ANY", 65 | integrationType: "HTTP_PROXY", 66 | connectionType: "VPC_LINK", 67 | description: "API Integration", 68 | connectionId: vpc_link.ref, 69 | integrationUri: props.fargate_alb.listeners[0].listenerArn, 70 | payloadFormatVersion: "1.0", 71 | }) 72 | 73 | 74 | 75 | 76 | // Create CORS preflight Lambda function first 77 | const corsLambda = new LambdaFunction(this, 'CorsPreflightHandler', { 78 | functionName: 'garnet-api-cors-preflight', 79 | runtime: Runtime.NODEJS_LATEST, 80 | handler: 'index.handler', 81 | code: Code.fromInline(` 82 | exports.handler = async (event) => { 83 | return { 84 | statusCode: 200, 85 | headers: { 86 | "Access-Control-Allow-Origin": "*", 87 | "Access-Control-Allow-Headers": "Content-Type,Authorization,X-Amz-Date,X-Api-Key,X-Amz-Security-Token", 88 | "Access-Control-Allow-Methods": "GET,POST,OPTIONS" 89 | }, 90 | body: '' 91 | } 92 | } 93 | `) 94 | }) 95 | 96 | // Grant API Gateway permission to invoke the CORS Lambda function 97 | corsLambda.addPermission('ApiGatewayInvokePermission', { 98 | principal: new ServicePrincipal('apigateway.amazonaws.com'), 99 | sourceArn: `arn:aws:execute-api:${Aws.REGION}:${Aws.ACCOUNT_ID}:${api.apiId}/*/*` 100 | }) 101 | 102 | // Create Lambda integration for CORS preflight 103 | const corsIntegration = new CfnIntegration(this, 'CorsLambdaIntegration', { 104 | apiId: api.apiId, 105 | integrationMethod: "POST", 106 | integrationType: "AWS_PROXY", 107 | integrationUri: `arn:aws:apigateway:${Aws.REGION}:lambda:path/2015-03-31/functions/${corsLambda.functionArn}/invocations`, 108 | payloadFormatVersion: "2.0", 109 | }) 110 | 111 | const authorizer = new CfnAuthorizerV2(this, 'JwtAuthorizer', { 112 | apiId: api.apiId, 113 | authorizerType: 'REQUEST', 114 | authorizerPayloadFormatVersion: '2.0', 115 | authorizerResultTtlInSeconds: 600, 116 | authorizerUri: `arn:aws:apigateway:${Aws.REGION}:lambda:path/2015-03-31/functions/${props.lambda_authorizer_arn}/invocations`, 117 | enableSimpleResponses: true, 118 | identitySource: ['$request.header.Authorization'], 119 | name: 'jwt-authorizer' 120 | }) 121 | 122 | const route = new CfnRoute(this, 'AuthRoute', { 123 | apiId: api.apiId, 124 | routeKey: "ANY /{proxy+}", 125 | target: `integrations/${integration.ref}`, 126 | authorizationType: Parameters.authorization ? 'CUSTOM' : 'NONE', 127 | ...(Parameters.authorization ? { 128 | authorizerId: authorizer.ref, 129 | } : {}) 130 | }) 131 | 132 | if (Parameters.authorization) { 133 | route.node.addDependency(authorizer) 134 | } 135 | 136 | // Add OPTIONS route for CORS preflight AFTER the ANY route 137 | const optionsRoute = new CfnRoute(this, 'ApiOptionsRoute', { 138 | apiId: api.apiId, 139 | routeKey: "OPTIONS /{proxy+}", 140 | target: `integrations/${corsIntegration.ref}`, 141 | authorizationType: 'NONE' 142 | }) 143 | 144 | // Add explicit dependencies to ensure proper creation order 145 | optionsRoute.node.addDependency(corsIntegration) 146 | optionsRoute.node.addDependency(corsLambda) 147 | 148 | 149 | this.api_ref = api.apiId 150 | 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /architecture.ts: -------------------------------------------------------------------------------- 1 | // SIZING OF THE GARNET DEPLOYMENT 2 | 3 | /*** 4 | * achitecture: if "concentrated", the AllinOne container is used. If "distributed", the microservice architecture is deployed. 5 | * fargate_cpu : https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size 6 | * fargate_memory_limit: https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definition_parameters.html#task_size 7 | * *_autoscale_min_capacity; minimum number of task for the container. 8 | * aurora_min_capacity: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2.setting-capacity.html#aurora-serverless-v2.min_capacity_considerations 9 | * aurora_max_capacity: https://docs.aws.amazon.com/AmazonRDS/latest/AuroraUserGuide/aurora-serverless-v2.setting-capacity.html#aurora-serverless-v2.max_capacity_considerations 10 | */ 11 | 12 | import { DBClusterStorageType } from "aws-cdk-lib/aws-rds"; 13 | import {Parameters} from "./configuration" 14 | 15 | // const sizing = Parameters.sizing 16 | 17 | export const enum ARCHITECTURE { 18 | Concentrated = "concentrated", 19 | Distributed = "distributed" 20 | } 21 | 22 | type DeploymentParams = { 23 | architecture: string, 24 | autoscale_requests_number: number, 25 | lambda_broker_batch_window: number, 26 | lambda_broker_batch_size: number, 27 | lambda_broker_concurent_sqs: number, 28 | aurora_storage_type?: DBClusterStorageType, 29 | aurora_min_capacity: number, 30 | aurora_max_capacity: number, 31 | all_fargate_cpu?: number, 32 | all_fargate_memory_limit?: number, 33 | all_autoscale_min_capacity?: number, 34 | all_autoscale_max_capacity?: number, 35 | entitymanager_fargate_cpu?: number, 36 | entitymanager_fargate_memory_limit?: number, 37 | entitymanager_autoscale_min_capacity?: number, 38 | entitymanager_autoscale_max_capacity?: number, 39 | subscriptionmanager_fargate_cpu?: number, 40 | subscriptionmanager_fargate_memory_limit?: number, 41 | subscriptionmanager_autoscale_min_capacity?: number, 42 | subscriptionmanager_autoscale_max_capacity?: number, 43 | registrymanager_fargate_cpu?: number, 44 | registrymanager_fargate_memory_limit?: number, 45 | registrymanager_autoscale_min_capacity?: number, 46 | registrymanager_autoscale_max_capacity?: number, 47 | querymanager_fargate_cpu?: number, 48 | querymanager_fargate_memory_limit?: number, 49 | querymanager_autoscale_min_capacity?: number, 50 | querymanager_autoscale_max_capacity?: number, 51 | registrysubscriptionmanager_fargate_cpu?: number, 52 | registrysubscriptionmanager_fargate_memory_limit?: number, 53 | registrysubscriptionmanager_autoscale_min_capacity?: number, 54 | registrysubscriptionmanager_autoscale_max_capacity?: number, 55 | historyentitymanager_fargate_cpu?: number, 56 | historyentitymanager_fargate_memory_limit?: number, 57 | historyentitymanager_autoscale_min_capacity?: number, 58 | historyentitymanager_autoscale_max_capacity?: number, 59 | historyquerymanager_fargate_cpu?: number, 60 | historyquerymanager_fargate_memory_limit?: number, 61 | historyquerymanager_autoscale_min_capacity?: number, 62 | historyquerymanager_autoscale_max_capacity?: number, 63 | atcontextserver_fargate_cpu?: number, 64 | atcontextserver_fargate_memory_limit?: number, 65 | atcontextserver_autoscale_min_capacity?: number, 66 | atcontextserver_autoscale_max_capacity?: number 67 | } 68 | 69 | 70 | 71 | export let deployment_params: DeploymentParams 72 | 73 | if (Parameters.architecture == ARCHITECTURE.Concentrated) { 74 | deployment_params = { 75 | architecture: ARCHITECTURE.Concentrated, 76 | autoscale_requests_number: 50, 77 | 78 | lambda_broker_batch_window: 1, 79 | lambda_broker_batch_size: 10, 80 | lambda_broker_concurent_sqs: 10, 81 | 82 | aurora_min_capacity: 1, 83 | aurora_max_capacity: 200, 84 | 85 | all_fargate_cpu: 1024, 86 | all_fargate_memory_limit: 4096, 87 | 88 | all_autoscale_min_capacity: 2, 89 | all_autoscale_max_capacity: 10, 90 | 91 | } 92 | } else { 93 | deployment_params = { 94 | architecture: ARCHITECTURE.Distributed, 95 | aurora_min_capacity: 2, 96 | aurora_max_capacity: 200, 97 | autoscale_requests_number: 50, 98 | 99 | lambda_broker_batch_window: 1, 100 | lambda_broker_batch_size: 20, 101 | lambda_broker_concurent_sqs: 30, 102 | 103 | entitymanager_fargate_cpu: 1024, 104 | entitymanager_fargate_memory_limit: 4096, 105 | entitymanager_autoscale_min_capacity: 2, 106 | entitymanager_autoscale_max_capacity: 100, 107 | 108 | subscriptionmanager_fargate_cpu: 1024, 109 | subscriptionmanager_fargate_memory_limit: 4096, 110 | subscriptionmanager_autoscale_min_capacity: 2, 111 | subscriptionmanager_autoscale_max_capacity: 70, 112 | 113 | registrymanager_fargate_cpu: 1024, 114 | registrymanager_fargate_memory_limit: 4096, 115 | registrymanager_autoscale_min_capacity: 2, 116 | registrymanager_autoscale_max_capacity: 30, 117 | 118 | querymanager_fargate_cpu: 1024, 119 | querymanager_fargate_memory_limit: 4096, 120 | querymanager_autoscale_min_capacity: 2, 121 | querymanager_autoscale_max_capacity: 50, 122 | 123 | registrysubscriptionmanager_fargate_cpu: 1024, 124 | registrysubscriptionmanager_fargate_memory_limit: 4096, 125 | registrysubscriptionmanager_autoscale_min_capacity: 2, 126 | registrysubscriptionmanager_autoscale_max_capacity: 30, 127 | 128 | historyentitymanager_fargate_cpu: 1024, 129 | historyentitymanager_fargate_memory_limit: 4096, 130 | historyentitymanager_autoscale_min_capacity: 2, 131 | historyentitymanager_autoscale_max_capacity: 60, 132 | 133 | historyquerymanager_fargate_cpu: 1024, 134 | historyquerymanager_fargate_memory_limit: 4096, 135 | historyquerymanager_autoscale_min_capacity: 2, 136 | historyquerymanager_autoscale_max_capacity: 50, 137 | 138 | atcontextserver_fargate_cpu: 1024, 139 | atcontextserver_fargate_memory_limit: 4096, 140 | atcontextserver_autoscale_min_capacity: 2, 141 | atcontextserver_autoscale_max_capacity: 100 142 | } 143 | 144 | } 145 | 146 | deployment_params.aurora_storage_type = DBClusterStorageType.AURORA_IOPT1 147 | 148 | 149 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | All notable changes to the Garnet Framework will be documented in this file. 4 | 5 | ## [1.5.1] - 2025-09-17 6 | 7 | ### Bug Fixes 8 | 9 | - Fixed subscription notification duplication issue introduced by the new SQS-based Garnet Private Notification Endpoint feature in version 1.5.0 10 | - Updated Scorpio Broker to version [5.0.94](https://gallery.ecr.aws/garnet/) with subscription processing improvements 11 | 12 | ## [1.5.0] - 2025-08-18 13 | 14 | This version fixes bugs, introduces new features with potential breaking changes, and improves performance. Users can raise issues in the GitHub issues section if any problems occur. 15 | 16 | ### Performance Optimizations 17 | 18 | - Implemented IRI compaction caching to improve temporal query performance 19 | - Enhanced subscription service with type-based entity filtering 20 | 21 | ### Enhancements 22 | 23 | - Updated Scorpio Broker to version [5.0.93](https://gallery.ecr.aws/garnet/) with performance and functionality improvements 24 | - Upgraded Quarkus framework to version 3.24.5 with improved stability and performance 25 | 26 | ### New Features 27 | 28 | - **Garnet Private Notification Endpoint**: Added direct SQS integration alongside existing REST API Gateway for improved scalability: 29 | - Users can now configure `garnet:notification` endpoints for subscription notifications 30 | - Notifications are sent directly to a dedicated SQS queue using the same AWS IoT Core MQTT topics mechanism for consumption 31 | 32 | ### Deprecated 33 | 34 | - **Smart Data Models Context**: The `context.jsonld` file is deprecated and should no longer be used as it can conflict with NGSI-LD core context. 35 | 36 | 37 | ### Upgrade Notes 38 | 39 | - Existing subscriptions continue to work unchanged 40 | 41 | ## [1.4.3] - 2025-07-18 42 | 43 | ### Enhancements 44 | 45 | - Added CORS preflight support for API Gateway with dedicated OPTIONS method handler 46 | 47 | ## [1.4.2] - 2025-05-28 48 | 49 | ### Bug Fixes 50 | 51 | - Updated Scorpio Broker to version [5.0.92](https://gallery.ecr.aws/garnet/) 52 | - Fixed context resolution issue affecting transitions between concentrated and distributed architectures 53 | - Improved handling of external context URLs with robust fallback mechanism 54 | - Enhanced error handling and logging for context resolution 55 | 56 | 57 | ## [1.4.1] - 2025-05-20 58 | 59 | This new version includes enhancements to key components, bug fixes, and new integration features. 60 | 61 | ### Enhancements 62 | 63 | - Enhanced the datalake component to transmit normalized entity versions including system attributes, providing more comprehensive data for analysis 64 | - Updated Scorpio Broker to version [5.0.91](https://gallery.ecr.aws/garnet/) 65 | - Improved stability of the subscriptions component 66 | - Added multi-architecture support for seamless transitions between concentrated and distributed deployments: 67 | - Implemented intelligent context resolution with configurable fallback mechanism 68 | - Ensured backward compatibility for existing subscriptions when changing architectures 69 | - Added context caching for improved performance 70 | 71 | 72 | ### New Features 73 | 74 | - Added synchronization from AWS IoT to the context broker: 75 | - The system now listens to AWS IoT Core event messages and automatically updates the context broker 76 | - Creation, deletion, or updates of AWS IoT Things now trigger corresponding entity changes in the context broker using the AwsIotThing type 77 | - Similar lifecycle management for AWS IoT Thing Groups creates or updates entities using the AwsIotThingGroup type 78 | 79 | 80 | 81 | ## [1.4.0] - 2025-02-19 82 | 83 | We've implemented significant architectural changes in this release to improve cost efficiency and scalability. 84 | The documentation has been updated to reflect these changes and provides detailed guidance on using the new architecture and features. 85 | 86 | ### Major Changes 87 | 88 | - Redesigned Architecture and Stack 89 | - Complete architectural overhaul 90 | - New stack implementation 91 | 92 | - Simplified Architecture Configuration 93 | - Replaced sizing options with direct choice between Concentrated and Distributed architectures 94 | - Moved detailed configuration parameters to architecture.ts for better clarity and control 95 | 96 | - Context Broker Update 97 | - Upgraded Scorpio Broker to version ([5.0.90](https://gallery.ecr.aws/garnet/)) 98 | 99 | - Improved Ingestion Process 100 | - Eliminated AWS IoT Device Shadow dependency 101 | - Implemented direct ingestion via deployment-provided queue 102 | - Added automatic context broker updates using batch operation upsert 103 | - Delivered more cost-effective and scalable solution 104 | 105 | - Database Engine Update 106 | - Aurora Serverless v2 upgraded to PostgreSQL v16.6 107 | 108 | - Streamlined Data Lake Integration 109 | - Direct Kinesis Firehose integration for data lake delivery 110 | - Created more efficient data pipeline by removing IoT rule dependency 111 | 112 | - API and Data Model Changes 113 | - Removing the Garnet JSON-LD context 114 | - Deprecation of IoT API 115 | - Authorization now enforced with a token (provided as output of CloudFormation stack) 116 | - JSON-LD context change: AWS IoT thing is now referenced as AwsIotThing 117 | 118 | 119 | - NGSI-LD Type for Things changed. Now AWS IoT thing is AwsIotThing. 120 | 121 | ### Required Actions 122 | 123 | Users will need to: 124 | - Update their ingestion workflows to use the new queue for ingesting 125 | - Migrate any AWS IoT Device Shadow dependencies 126 | 127 | ## [1.3.0] - 2024-05-07 128 | 129 | This new version fixes a [bug](https://github.com/ScorpioBroker/ScorpioBroker/issues/556) we had due to the use of SQS in Scorpio Broker for fanning out messages. This led to missing messages in the datalake, the temporal storage and the subscriptions. 130 | 131 | ### [1.3.0] - Added 132 | 133 | - SNS for fanning out messages to dedicated SQS queues per service. 134 | - VPC endpoints for SNS and SQS 135 | 136 | ### [1.3.0] - Changed 137 | 138 | - Updated Aurora serverless v2 engine version to Postgresql v15.5 139 | 140 | 141 | ## [1.2.0] - 2024-02-23 142 | 143 | This new version fixes some bugs, introduces new features and potential breaking changes. 144 | 145 | ### [1.2.0] - Added 146 | 147 | - Distributed architecture. You can now deploy Garnet using the microservice version of Scorpio Broker. 148 | - T-shirt Sizing for deployment. You can now choose the size of the deployment between Small and Xlarge depending on your workload. 149 | 150 | ### [1.2.0] - Changed 151 | 152 | - Renames resources (logs, functions) 153 | 154 | ## [1.1.0] - 2024-02-06 155 | 156 | This new version fixes some bugs, introduces new features and potential breaking changes. 157 | 158 | ### Added 159 | 160 | - RDS Proxy for the database. 161 | - Multi-typing support. See [Multi-Typing](https://garnet-framework.tech/docs/how/context-broker#multi-typing) section for information. 162 | - Connectivity status of Things connected using AWS IoT Core. See [Connectivity Status](https://garnet-framework.tech/docs/how/garnet-iot#connectivity-status) for more information. 163 | - Sync of AWS Iot Things Group Membership with Shadows and the Context Broker. See [Garnet Thing](https://garnet-framework.tech/docs/how/garnet-iot#a-garnet-thing) section for more information. 164 | 165 | 166 | ### Changed 167 | 168 | - Aurora Serverless v2 is now used for the PostgreSQL instead of Amazon RDS. 169 | - Renamed resources (logs, functions) 170 | - Updated Scorpio Broker to version [4.1.14](https://gallery.ecr.aws/garnet/scorpio) 171 | 172 | 173 | 174 | ## [1.0.0] - 2023-11-02 175 | 176 | Initial commit of the Garnet Framework. 177 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-group/iot-group-construct.ts: -------------------------------------------------------------------------------- 1 | import { Aws, Duration, Names, RemovalPolicy } from "aws-cdk-lib"; 2 | import { Runtime, Function, Code, Architecture, LayerVersion, CfnPermission } from "aws-cdk-lib/aws-lambda"; 3 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 4 | import { Queue } from "aws-cdk-lib/aws-sqs"; 5 | import { Construct } from "constructs" 6 | import { PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam"; 7 | import { SqsEventSource } from "aws-cdk-lib/aws-lambda-event-sources"; 8 | import { CfnTopicRule } from "aws-cdk-lib/aws-iot"; 9 | import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources"; 10 | import { garnet_constant, garnet_nomenclature } from "../../../../constants"; 11 | import { SubnetType, Vpc } from "aws-cdk-lib/aws-ec2"; 12 | import { Parameters } from "../../../../configuration"; 13 | 14 | /*** 15 | * https://docs.aws.amazon.com/iot/latest/developerguide/registry-events.html#registry-events-thinggroup 16 | */ 17 | 18 | export interface GarnetIotGroupProps { 19 | vpc: Vpc, 20 | dns_context_broker: string 21 | } 22 | 23 | export class GarnetIotGroup extends Construct { 24 | 25 | public readonly private_sub_endpoint: string 26 | 27 | constructor(scope: Construct, id: string, props: GarnetIotGroupProps) { 28 | super(scope, id) 29 | 30 | //CHECK PROPS 31 | if (!props.vpc) { 32 | throw new Error( 33 | "The property vpc is required to create an instance of the construct" 34 | ); 35 | } 36 | if (!props.dns_context_broker) { 37 | throw new Error( 38 | "The property dns_context_broker is required to create an instance of the construct" 39 | ); 40 | } 41 | 42 | 43 | /* 44 | * SHARED LIBRARIES 45 | */ 46 | const layer_lambda_path = `./lib/layers`; 47 | const layer_lambda = new LayerVersion(this, "GarnetIoTSharedLayer", { 48 | code: Code.fromAsset(layer_lambda_path), 49 | compatibleRuntimes: [Runtime.NODEJS_22_X], 50 | }) 51 | 52 | ///////////////////////////////////////////// 53 | 54 | /* 55 | * THING GROUP MEMBERSHIP 56 | */ 57 | 58 | // LAMBDA TO UPDATE BROKER WITH GROUP MEMBERSHIP 59 | const lambda_update_group_membership_log = new LogGroup(this, 'GarnetIoTGroupMembershipLambdaLogs', { 60 | retention: RetentionDays.ONE_MONTH, 61 | removalPolicy: RemovalPolicy.DESTROY 62 | }) 63 | const lambda_update_group_membership_path = `${__dirname}/lambda/groupMembership`; 64 | const lambda_update_group_membership = new Function(this, "GarnetIoTGroupMembershipLambda", { 65 | functionName: `${garnet_nomenclature.garnet_iot_group_membership_lambda}`, 66 | description: 'Garnet Sync AWS IoT Things Group- Function that updates Things Group membership for Things', 67 | vpc: props.vpc, 68 | vpcSubnets: { 69 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 70 | }, 71 | runtime: Runtime.NODEJS_22_X, 72 | layers: [layer_lambda], 73 | code: Code.fromAsset(lambda_update_group_membership_path), 74 | handler: "index.handler", 75 | timeout: Duration.seconds(50), 76 | logGroup: lambda_update_group_membership_log, 77 | architecture: Architecture.ARM_64, 78 | environment: { 79 | DNS_CONTEXT_BROKER: props.dns_context_broker, 80 | AWSIOTTHINGTYPE: garnet_nomenclature.aws_iot_thing, 81 | AWSIOTTHINGGROUPTYPE: garnet_nomenclature.aws_iot_thing_group 82 | } 83 | }) 84 | 85 | lambda_update_group_membership.node.addDependency(lambda_update_group_membership_log) 86 | 87 | lambda_update_group_membership.addToRolePolicy( 88 | new PolicyStatement({ 89 | actions: ["iot:ListThingGroupsForThing"], 90 | resources: [ 91 | `arn:aws:iot:${Aws.REGION}:${Aws.ACCOUNT_ID}:thing/*`, 92 | ] 93 | }) 94 | ) 95 | 96 | // IOT RULE FOR GROUP MEMBERSHIP EVENTS 97 | const iot_rule_group_membership = new CfnTopicRule(this, "GarnetIoTGroupMembershipRule", { 98 | ruleName: `garnet_iot_thing_group_membership_rule`, 99 | topicRulePayload: { 100 | awsIotSqlVersion: "2016-03-23", 101 | ruleDisabled: false, 102 | sql: `SELECT *, topic(7) as thingName, topic(5) as thingGroup from '$aws/events/thingGroupMembership/thingGroup/+/thing/+/+'`, 103 | actions: [ 104 | { 105 | lambda: { 106 | functionArn: lambda_update_group_membership.functionArn 107 | } 108 | } 109 | ] 110 | } 111 | }) 112 | 113 | // GRANT IOT RULE PERMISSION TO INVOKE LAMBDA 114 | new CfnPermission(this, 'GarnetIoTGroupMembershipLambdaPermission', { 115 | principal: `iot.amazonaws.com`, 116 | action: 'lambda:InvokeFunction', 117 | functionName: lambda_update_group_membership.functionName, 118 | sourceArn: `${iot_rule_group_membership.attrArn}` 119 | }) 120 | 121 | /* 122 | * END THING GROUP MEMBERSHIP 123 | */ 124 | 125 | 126 | /* 127 | * THING GROUP LIFECYCLE 128 | */ 129 | 130 | // LAMBDA TO HANDLE THING GROUP CREATION/DELETION 131 | const lambda_group_lifecyle_log = new LogGroup(this, 'GarnetIoTGroupLifecycleLambdaLogs', { 132 | retention: RetentionDays.ONE_MONTH, 133 | removalPolicy: RemovalPolicy.DESTROY 134 | }) 135 | const lambda_group_lifecycle_path = `${__dirname}/lambda/groupLifecycle`; 136 | const lambda_group_lifecyle = new Function(this, "GarnetIoTGroupLifecycleLambda", { 137 | functionName: `${garnet_nomenclature.garnet_iot_group_lifecycle_lambda}`, 138 | description: 'Garnet AWS IoT Things Group Sync - Function that handles Thing Group lifecycle', 139 | vpc: props.vpc, 140 | vpcSubnets: { 141 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 142 | }, 143 | runtime: Runtime.NODEJS_22_X, 144 | layers: [layer_lambda], 145 | code: Code.fromAsset(lambda_group_lifecycle_path), 146 | handler: "index.handler", 147 | timeout: Duration.seconds(50), 148 | logGroup: lambda_group_lifecyle_log, 149 | architecture: Architecture.ARM_64, 150 | environment: { 151 | DNS_CONTEXT_BROKER: props.dns_context_broker, 152 | AWSIOTTHINGGROUPTYPE: garnet_nomenclature.aws_iot_thing_group 153 | } 154 | }) 155 | lambda_group_lifecyle.node.addDependency(lambda_group_lifecyle_log) 156 | 157 | // IOT RULE FOR GROUP LIFECYCLE EVENTS 158 | const iot_rule_group_lifecycle = new CfnTopicRule(this, "GarnetIoTGroupLifecycleRule", { 159 | ruleName: `garnet_iot_thing_group_lifecycle_rule`, 160 | topicRulePayload: { 161 | awsIotSqlVersion: "2016-03-23", 162 | ruleDisabled: false, 163 | sql: `SELECT * from '$aws/events/thingGroup/#'`, 164 | actions: [ 165 | { 166 | lambda: { 167 | functionArn: lambda_group_lifecyle.functionArn 168 | } 169 | } 170 | ] 171 | } 172 | }) 173 | 174 | // GRANT IOT RULE PERMISSION TO INVOKE LAMBDA 175 | new CfnPermission(this, 'GarnetIoTGroupLifecycleLambdaPermission', { 176 | principal: `iot.amazonaws.com`, 177 | action: 'lambda:InvokeFunction', 178 | functionName: lambda_group_lifecyle.functionName, 179 | sourceArn: `${iot_rule_group_lifecycle.attrArn}` 180 | }) 181 | 182 | /* 183 | * END THING GROUP LIFECYCLE 184 | */ 185 | 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /lib/stacks/garnet-iot/iot-thing/iot-thing-construct.ts: -------------------------------------------------------------------------------- 1 | import { Aws, Duration, Names, RemovalPolicy } from "aws-cdk-lib"; 2 | import { Runtime, Function, Code, Architecture, LayerVersion, CfnPermission } from "aws-cdk-lib/aws-lambda"; 3 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 4 | import { Queue } from "aws-cdk-lib/aws-sqs"; 5 | import { Construct } from "constructs" 6 | import { PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam"; 7 | import { SqsEventSource } from "aws-cdk-lib/aws-lambda-event-sources"; 8 | import { CfnTopicRule } from "aws-cdk-lib/aws-iot"; 9 | import { AwsCustomResource, AwsCustomResourcePolicy, PhysicalResourceId } from "aws-cdk-lib/custom-resources"; 10 | import { garnet_constant, garnet_nomenclature } from "../../../../constants"; 11 | import { SubnetType, Vpc } from "aws-cdk-lib/aws-ec2"; 12 | import { Parameters } from "../../../../configuration"; 13 | 14 | /*** 15 | * https://docs.aws.amazon.com/iot/latest/developerguide/registry-events.html#registry-events-thing 16 | */ 17 | 18 | 19 | export interface GarnetIotThingProps { 20 | vpc: Vpc, 21 | dns_context_broker: string 22 | } 23 | 24 | export class GarnetIotThing extends Construct { 25 | 26 | public readonly private_sub_endpoint: string 27 | 28 | constructor(scope: Construct, id: string, props: GarnetIotThingProps) { 29 | super(scope, id) 30 | 31 | 32 | // LAMBDA LAYER (SHARED LIBRARIES) 33 | const layer_lambda_path = `./lib/layers`; 34 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 35 | code: Code.fromAsset(layer_lambda_path), 36 | compatibleRuntimes: [Runtime.NODEJS_22_X], 37 | }) 38 | 39 | 40 | // CONNECTIVITY STATUS 41 | 42 | 43 | // SQS ENTRY POINT CONNECTIVITY STATUS 44 | const sqs_garnet_iot_presence = new Queue(this, "SqsGarnetPresenceThing", { 45 | queueName: garnet_nomenclature.garnet_iot_presence_queue, 46 | visibilityTimeout: Duration.seconds(55) 47 | }) 48 | 49 | 50 | // LAMBDA TO PUSH IN QUEUE WITH CONNECTIVITY STATUS 51 | const lambda_update_presence_log = new LogGroup(this, 'LambdaUpdatePresenceThingLogs', { 52 | retention: RetentionDays.ONE_MONTH, 53 | // logGroupName: `${garnet_nomenclature.garnet_iot_presence_shadow_lambda}-logs`, 54 | removalPolicy: RemovalPolicy.DESTROY 55 | }) 56 | const lambda_update_presence_path = `${__dirname}/lambda/presence`; 57 | const lambda_update_presence = new Function(this, "LambdaUpdatePresenceThing", { 58 | functionName: garnet_nomenclature.garnet_iot_presence_lambda, 59 | description: 'Garnet IoT Things Presence- Function that updates presence for Iot MQTT connected things', 60 | vpc: props.vpc, 61 | vpcSubnets: { 62 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 63 | }, 64 | runtime: Runtime.NODEJS_22_X, 65 | layers: [layer_lambda], 66 | code: Code.fromAsset(lambda_update_presence_path), 67 | handler: "index.handler", 68 | timeout: Duration.seconds(50), 69 | logGroup: lambda_update_presence_log, 70 | architecture: Architecture.ARM_64, 71 | environment: { 72 | DNS_CONTEXT_BROKER: props.dns_context_broker, 73 | AWSIOTTHINGTYPE: garnet_nomenclature.aws_iot_thing 74 | } 75 | }) 76 | lambda_update_presence.node.addDependency(lambda_update_presence_log) 77 | // ADD PERMISSION FOR LAMBDA THAT UPDATES SHADOW TO ACCESS SQS ENTRY POINT 78 | lambda_update_presence.addToRolePolicy( 79 | new PolicyStatement({ 80 | actions: [ 81 | "sqs:ReceiveMessage", 82 | "sqs:DeleteMessage", 83 | "sqs:GetQueueAttributes" 84 | ], 85 | resources: [`${sqs_garnet_iot_presence.queueArn}`], 86 | }) 87 | ) 88 | 89 | // ADD THE SQS AS EVENT SOURCE FOR LAMBDA 90 | lambda_update_presence.addEventSource( 91 | new SqsEventSource(sqs_garnet_iot_presence, { batchSize: 10 }) 92 | ) 93 | 94 | // ROLE THAT GRANT ACCESS TO IOT RULE TO ACTIONS 95 | const iot_rule_actions_role = new Role(this, "RoleGarnetIotRulePresence", { 96 | assumedBy: new ServicePrincipal("iot.amazonaws.com"), 97 | }) 98 | 99 | iot_rule_actions_role.addToPolicy( 100 | new PolicyStatement({ 101 | resources: [ 102 | `${sqs_garnet_iot_presence.queueArn}` 103 | ], 104 | actions: [ 105 | "sqs:SendMessage" 106 | ], 107 | }) 108 | ) 109 | 110 | 111 | // IOT RULE THAT LISTENS TO CHANGES IN IoT PRESENCE AND PUSH TO SQS 112 | const iot_rule = new CfnTopicRule(this, "IoTRulePresence", { 113 | ruleName: garnet_nomenclature.garnet_iot_presence_rule, 114 | topicRulePayload: { 115 | awsIotSqlVersion: "2016-03-23", 116 | ruleDisabled: false, 117 | sql: `SELECT * from '$aws/events/presence/#'`, 118 | actions: [ 119 | { 120 | sqs: { 121 | queueUrl: sqs_garnet_iot_presence.queueUrl, 122 | roleArn: iot_rule_actions_role.roleArn, 123 | }, 124 | } 125 | ], 126 | }, 127 | }) 128 | 129 | // END CONNECTIVITY STATUS 130 | 131 | 132 | 133 | 134 | /* 135 | * THING LIFECYCLE 136 | */ 137 | 138 | // LAMBDA TO HANDLE THING CREATION/DELETION 139 | const lambda_thing_lifecyle_log = new LogGroup(this, 'GarnetIotThingLifecycleLambdaLogs', { 140 | retention: RetentionDays.ONE_MONTH, 141 | removalPolicy: RemovalPolicy.DESTROY 142 | }) 143 | const lambda_thing_lifecycle_path = `${__dirname}/lambda/thingLifecycle`; 144 | const lambda_thing_lifecyle = new Function(this, "GarnetIotThingLifecycleLambda", { 145 | functionName: `${garnet_nomenclature.garnet_iot_lifecycle_lambda}`, 146 | description: 'Garnet AWS IoT Things Sync - Function that handles Thing lifecycle', 147 | vpc: props.vpc, 148 | vpcSubnets: { 149 | subnetType: SubnetType.PRIVATE_WITH_EGRESS, 150 | }, 151 | runtime: Runtime.NODEJS_22_X, 152 | layers: [layer_lambda], 153 | code: Code.fromAsset(lambda_thing_lifecycle_path), 154 | handler: "index.handler", 155 | timeout: Duration.seconds(50), 156 | logGroup: lambda_thing_lifecyle_log, 157 | architecture: Architecture.ARM_64, 158 | environment: { 159 | DNS_CONTEXT_BROKER: props.dns_context_broker, 160 | AWSIOTTHINGTYPE: garnet_nomenclature.aws_iot_thing 161 | } 162 | }) 163 | lambda_thing_lifecyle.node.addDependency(lambda_thing_lifecyle_log) 164 | 165 | // IOT RULE FOR THING LIFECYCLE EVENTS 166 | const iot_rule_thing_lifecycle = new CfnTopicRule(this, "GarnetIotThingLifecycleRule", { 167 | ruleName: `garnet_iot_thing_lifecycle_rule`, 168 | topicRulePayload: { 169 | awsIotSqlVersion: "2016-03-23", 170 | ruleDisabled: false, 171 | sql: `SELECT * from '$aws/events/thing/#'`, 172 | actions: [ 173 | { 174 | lambda: { 175 | functionArn: lambda_thing_lifecyle.functionArn 176 | } 177 | } 178 | ] 179 | } 180 | }) 181 | 182 | // GRANT IOT RULE PERMISSION TO INVOKE LAMBDA 183 | new CfnPermission(this, 'GarnetIotThingLifecycleLambdaPermission', { 184 | principal: `iot.amazonaws.com`, 185 | action: 'lambda:InvokeFunction', 186 | functionName: lambda_thing_lifecyle.functionName, 187 | sourceArn: `${iot_rule_thing_lifecycle.attrArn}` 188 | }) 189 | 190 | /* 191 | * END THING LIFECYCLE 192 | */ 193 | 194 | } 195 | } -------------------------------------------------------------------------------- /lib/stacks/garnet-common/utils/utils-construct.ts: -------------------------------------------------------------------------------- 1 | import { Construct } from "constructs"; 2 | import { azlist, garnet_constant, garnet_nomenclature, scorpiobroker_sqs_object } from "../../../../constants" 3 | import { Aws, CustomResource, Duration, RemovalPolicy, Stack } from "aws-cdk-lib"; 4 | import { Code, Runtime, Function, Architecture } from "aws-cdk-lib/aws-lambda"; 5 | import { PolicyStatement } from "aws-cdk-lib/aws-iam"; 6 | import { Provider } from "aws-cdk-lib/custom-resources"; 7 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs"; 8 | 9 | export interface GarnetUtilProps {} 10 | 11 | export class Utils extends Construct { 12 | 13 | public readonly az1: string 14 | public readonly az2: string 15 | 16 | constructor(scope: Construct, id: string, props?: GarnetUtilProps) { 17 | super(scope, id) 18 | 19 | 20 | // CHECK THE AZs TO DEPLOY GARNET 21 | 22 | if(Stack.of(this).region.startsWith('$')){ 23 | throw new Error('Please type a valid region in the parameter.ts file') 24 | } 25 | 26 | if(!azlist[`${Stack.of(this).region}`]){ 27 | throw new Error('The stack is not yet available in the region selected') 28 | } 29 | 30 | const compatible_azs = azlist[`${Stack.of(this).region}`] 31 | 32 | const get_az_lambda_log = new LogGroup(this, 'LambdaAzFunctionLogs', { 33 | retention: RetentionDays.ONE_MONTH, 34 | removalPolicy: RemovalPolicy.DESTROY 35 | }) 36 | const get_az_func_path = `${__dirname}/lambda/getAzs` 37 | const get_az_func = new Function(this, 'AzFunction', { 38 | functionName: garnet_nomenclature.garnet_utils_az_lambda, 39 | description: 'Garnet Utils - Function that checks if which AZs the stack can be deployed for HTTP VPC Link and IoT VPC Endpoint service availability', 40 | runtime: Runtime.NODEJS_22_X, 41 | logGroup: get_az_lambda_log, 42 | code: Code.fromAsset(get_az_func_path), 43 | handler: 'index.handler', 44 | timeout: Duration.seconds(50), 45 | architecture: Architecture.ARM_64, 46 | environment: { 47 | COMPATIBLE_AZS: JSON.stringify(compatible_azs) 48 | } 49 | }) 50 | 51 | get_az_func.node.addDependency(get_az_lambda_log) 52 | get_az_func.addToRolePolicy(new PolicyStatement({ 53 | actions: [ 54 | "ec2:DescribeAvailabilityZones", 55 | "ec2:DescribeVpcEndpointServices" 56 | ], 57 | resources: ['*'] 58 | })) 59 | 60 | const get_az_log = new LogGroup(this, 'getAzCleanUpProviderLogs', { 61 | retention: RetentionDays.ONE_MONTH, 62 | // logGroupName: `garnet-provider-utils-az-lambda-cw-logs`, 63 | removalPolicy: RemovalPolicy.DESTROY 64 | }) 65 | 66 | const get_az_provider = new Provider(this, 'getAzCleanUpprovider', { 67 | onEventHandler: get_az_func, 68 | providerFunctionName: `${garnet_nomenclature.garnet_utils_az_lambda}-provider`, 69 | logGroup: get_az_log 70 | }) 71 | get_az_provider.node.addDependency(get_az_log) 72 | 73 | const get_az = new CustomResource(this, 'getAzCustomResource', { 74 | serviceToken: get_az_provider.serviceToken 75 | }) 76 | 77 | this.az1 = get_az.getAtt('az1').toString() 78 | this.az2 = get_az.getAtt('az2').toString() 79 | 80 | // CLEAN SQS QUEUES CREATED BY SCORPIO BROKER 81 | 82 | let sqs_urls = Object.values(scorpiobroker_sqs_object).map(q => `https://sqs.${Aws.REGION}.amazonaws.com/${Aws.ACCOUNT_ID}/${q}`) 83 | 84 | const scorpio_sqs_lambda_log = new LogGroup(this, 'LambdaScorpioSqsFunctionLogs', { 85 | retention: RetentionDays.ONE_MONTH, 86 | // logGroupName: `garnet-utils-scorpio-sqs-lambda-cw-logs`, 87 | removalPolicy: RemovalPolicy.DESTROY 88 | }) 89 | const scorpio_sqs_lambda_path = `${__dirname}/lambda/scorpioSqs` 90 | const scorpio_sqs_lambda = new Function(this, 'ScorpioCleanSqsFunction', { 91 | functionName: garnet_nomenclature.garnet_utils_scorpio_sqs_lambda, 92 | description: 'Garnet Utils - Function that deletes the SQS Queue created by the Scorpio Context Broker', 93 | runtime: Runtime.NODEJS_22_X, 94 | logGroup: scorpio_sqs_lambda_log, 95 | code: Code.fromAsset(scorpio_sqs_lambda_path), 96 | handler: 'index.handler', 97 | timeout: Duration.seconds(50), 98 | architecture: Architecture.ARM_64, 99 | environment: { 100 | SQS_QUEUES: JSON.stringify(sqs_urls) 101 | } 102 | }) 103 | scorpio_sqs_lambda.node.addDependency(scorpio_sqs_lambda_log) 104 | scorpio_sqs_lambda.addToRolePolicy(new PolicyStatement({ 105 | actions: ["sqs:DeleteQueue"], 106 | resources: [`arn:aws:sqs:${Aws.REGION}:${Aws.ACCOUNT_ID}:garnet-scorpiobroker-*`] 107 | })) 108 | 109 | const scorpio_sqs_provider_log = new LogGroup(this, 'LambdaScorpioSqsProviderLogs', { 110 | retention: RetentionDays.ONE_MONTH, 111 | // logGroupName: `garnet-provider-utils-scorpio-sqs-lambda-cw-logs`, 112 | removalPolicy: RemovalPolicy.DESTROY 113 | }) 114 | 115 | const scorpio_sqs_provider = new Provider(this, 'scorpioSqsProvider', { 116 | onEventHandler: scorpio_sqs_lambda, 117 | providerFunctionName: `${garnet_nomenclature.garnet_utils_scorpio_sqs_lambda}-provider`, 118 | logGroup: scorpio_sqs_provider_log 119 | }) 120 | scorpio_sqs_provider.node.addDependency(scorpio_sqs_provider_log) 121 | new CustomResource(this, 'scorpioSqsCustomResource', { 122 | serviceToken: scorpio_sqs_provider.serviceToken 123 | }) 124 | 125 | 126 | // CLEAN INACTIVE GARNET TASK DEFINITION IN ECS 127 | const clean_ecs_lambda_log = new LogGroup(this, 'LambdaCleanEcsFunctionLogs', { 128 | retention: RetentionDays.ONE_MONTH, 129 | // logGroupName: `garnet-utils-clean-ecs-lambda-cw-logs`, 130 | removalPolicy: RemovalPolicy.DESTROY 131 | }) 132 | const clean_ecs_lambda_path = `${__dirname}/lambda/cleanTasks` 133 | const clean_ecs_lambda = new Function(this, 'CleanEcsFunction', { 134 | functionName: garnet_nomenclature.garnet_utils_clean_ecs_taks_lambda, 135 | description: 'Garnet Utils - Function that removes unactive ECS task definitions', 136 | runtime: Runtime.NODEJS_22_X, 137 | logGroup: clean_ecs_lambda_log, 138 | code: Code.fromAsset(clean_ecs_lambda_path), 139 | handler: 'index.handler', 140 | timeout: Duration.seconds(50), 141 | architecture: Architecture.ARM_64, 142 | environment: { 143 | } 144 | }) 145 | clean_ecs_lambda.node.addDependency(clean_ecs_lambda_log) 146 | clean_ecs_lambda.addToRolePolicy(new PolicyStatement({ 147 | actions: [ 148 | "ecs:RegisterTaskDefinition", 149 | "ecs:ListTaskDefinitions", 150 | "ecs:DescribeTaskDefinition" 151 | ], 152 | resources: [`*`] 153 | })) 154 | 155 | clean_ecs_lambda.addToRolePolicy(new PolicyStatement({ 156 | actions: [ 157 | "ecs:DeleteTaskDefinition" 158 | ], 159 | resources: [`arn:aws:ecs:${Aws.REGION}:${Aws.ACCOUNT_ID}:task-definition/Garnet*`] 160 | })) 161 | 162 | 163 | const clean_ecs_logs = new LogGroup(this, 'LambdacleanEcsProviderLogs', { 164 | retention: RetentionDays.ONE_MONTH, 165 | // logGroupName: `garnet-provider-utils-clean-ecs-lambda-cw-logs`, 166 | removalPolicy: RemovalPolicy.DESTROY 167 | }) 168 | 169 | const clean_ecs_provider = new Provider(this, 'cleanEcsProvider', { 170 | onEventHandler: clean_ecs_lambda, 171 | providerFunctionName: `${garnet_nomenclature.garnet_utils_clean_ecs_taks_lambda}-provider`, 172 | logGroup: clean_ecs_logs 173 | }) 174 | clean_ecs_provider.node.addDependency(clean_ecs_logs) 175 | const scorpio_sqs_resource = new CustomResource(this, 'cleanEcsCustomResource', { 176 | serviceToken: clean_ecs_provider.serviceToken 177 | }) 178 | 179 | 180 | 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /constants.ts: -------------------------------------------------------------------------------- 1 | 2 | 3 | // List of AZs that support VPC links for HTTP APIs as https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-vpc-links.html#http-api-vpc-link-availability 4 | 5 | import { Aws } from "aws-cdk-lib" 6 | import { Parameters } from "./configuration" 7 | const {version} = require('./package.json') 8 | 9 | const garnet_scorpio_version = "5.0.94" 10 | 11 | export const garnet_bucket = `garnet-datalake-${Aws.REGION}-${Aws.ACCOUNT_ID}` // DO NOT CHANGE 12 | export const garnet_bucket_athena = `${garnet_bucket}-athena-results` 13 | export const garnet_broker = "Scorpio" 14 | 15 | export const garnet_constant = { 16 | garnet_version: version, 17 | shadow_prefix: "Garnet", 18 | dbname: 'scorpio', 19 | iotDomainName: 'garnet-iot-domain', 20 | gluedbName: 'garnetdb' 21 | } 22 | 23 | export const garnet_scorpio_images = { 24 | allInOne: `public.ecr.aws/garnet/scorpio:${garnet_scorpio_version}`, 25 | at_context_server: `public.ecr.aws/garnet/scorpio/at-context-server:${garnet_scorpio_version}`, 26 | entity_manager: `public.ecr.aws/garnet/scorpio/entity-manager:${garnet_scorpio_version}`, 27 | history_entity_manager: `public.ecr.aws/garnet/scorpio/history-entity-manager:${garnet_scorpio_version}`, 28 | history_query_manager: `public.ecr.aws/garnet/scorpio/history-query-manager:${garnet_scorpio_version}`, 29 | query_manager: `public.ecr.aws/garnet/scorpio/query-manager:${garnet_scorpio_version}`, 30 | registry_manager: `public.ecr.aws/garnet/scorpio/registry-manager:${garnet_scorpio_version}`, 31 | registry_subscription_manager: `public.ecr.aws/garnet/scorpio/registry-subscription-manager:${garnet_scorpio_version}`, 32 | subscription_manager: `public.ecr.aws/garnet/scorpio/subscription-manager:${garnet_scorpio_version}` 33 | } 34 | 35 | export const scorpiobroker_sqs_object = { 36 | "SCORPIO_TOPICS_ENTITY": `garnet-scorpiobroker-${Parameters.architecture}-entity`, 37 | "SCORPIO_TOPICS_REGISTRY": `garnet-scorpiobroker-${Parameters.architecture}-registry`, 38 | "SCORPIO_TOPICS_TEMPORAL": `garnet-scorpiobroker-${Parameters.architecture}-temporal`, 39 | "SCORPIO_TOPICS_INTERNALNOTIFICATION": `garnet-scorpiobroker-internalnotification`, 40 | "SCORPIO_TOPICS_INTERNALREGSUB": `garnet-scorpiobroker-internalregsub`, 41 | "SCORPIO_TOPICS_PRIVATE_NOTIFICATION": `garnet-scorpiobroker-private-notification` 42 | 43 | } 44 | 45 | export const garnet_nomenclature = { 46 | // DEPRECATED 47 | garnet_iot_rule: `garnet_iot_rule`, 48 | garnet_iot_update_shadow_lambda: `garnet-iot-update-shadow-lambda`, 49 | garnet_iot_update_broker_lambda: `garnet-iot-update-broker-lambda`, 50 | 51 | 52 | // GARNET MODEL 53 | aws_iot_thing: "AwsIotThing", 54 | aws_iot_thing_group: "AwsIotThingGroup", 55 | aws_iot_lorawan_thing: "AwsIotLorawanThing", 56 | aws_iot_lorawan_gateway: "AwsIotLorawanGateway", 57 | 58 | //GARNET INGESTION LAMBDA 59 | garnet_ingestion_update_broker_lambda: `garnet-ingestion-update-broker-lambda`, 60 | garnet_lake_transform_lambda: `garnet-lake-transform-lambda`, 61 | garnet_iot_lifecycle_lambda: `garnet-iot-thing-lifecycle-lambda`, 62 | garnet_iot_presence_lambda: `garnet-iot-presence-lambda`, 63 | garnet_iot_group_membership_lambda: `garnet-iot-group-membership-lambda`, 64 | garnet_iot_group_hierarchy_lambda: `garnet-iot-group-hierarchy-lambda`, 65 | garnet_iot_group_lifecycle_lambda: `garnet-iot-group-lifecycle-lambda`, 66 | garnet_iot_authorizer_lambda: `garnet-iot-authorizer-lambda`, 67 | garnet_private_sub_lambda: `garnet-private-sub-lambda`, 68 | garnet_private_sub_sqs_lambda: `garnet-private-sub-sqs-lambda`, 69 | garnet_lake_rule:`garnet_lake_rule`, 70 | garnet_subscriptions_rule: `garnet_subscriptions_rule`, 71 | garnet_iot_presence_rule: `garnet_iot_presence_rule`, 72 | 73 | // GARNET API AUTH 74 | garnet_api_auth_jwt_lambda: `garnet-api-auth-jwt-lambda`, 75 | garnet_api_authorizer_lambda: `garnet-api-authorizer-lambda`, 76 | 77 | garnet_api_auth_audience: `garnet-api`, 78 | garnet_api_auth_issuer: `garnet-framework`, 79 | garnet_api_auth_sub: `garnet:default-user`, 80 | 81 | // GARNET IOT SQS 82 | garnet_iot_queue: `garnet-iot-sqs-${Aws.REGION}`, // DEPRECATED 83 | garnet_ingestion_queue: `garnet-ingestion-queue-${Aws.REGION}`, // DEPRECATED 84 | garnet_iot_contextbroker_queue: `garnet-iot-sqs-contextbroker-${Aws.REGION}`, 85 | garnet_iot_presence_queue: `garnet-iot-presence-${Aws.REGION}`, 86 | garnet_iot_group_queue: `garnet-iot-presence-${Aws.REGION}`, 87 | 88 | // GARNET FIREHOSE 89 | garnet_lake_firehose_stream: `garnet-datalake-firehose-stream`, 90 | garnet_sub_firehose_stream: `garnet-subs-firehose-stream`, 91 | garnet_lake_firehose_interval: 60, // seconds 92 | garnet_lake_buffer_size: 64, // MB 93 | 94 | // GARNET BROKER CLUSTER 95 | garnet_broker_cluster: `garnet-broker-cluster`, 96 | 97 | // GARNET BROKER SERVICES 98 | garnet_broker_entitymanager: `garnet-broker-entity-manager`, 99 | garnet_broker_querymanager: `garnet-broker-query-manager`, 100 | garnet_broker_subscriptionmanager: `garnet-broker-subscription-manager`, 101 | garnet_broker_historyentitymanager: `garnet-broker-history-entity-manager`, 102 | garnet_broker_historyquerymanager: `garnet-broker-history-querymanager`, 103 | garnet_broker_atcontextserver: `garnet-broker-at-context-server`, 104 | garnet_broker_registrymanager: `garnet-broker-registry-manager`, 105 | garnet_broker_registrysubscriptionmanager: `garnet-broker-registry-subscription-manager`, 106 | garnet_broker_allinone: `garnet-broker-all-in-one`, 107 | 108 | // GARNET LOAD BALANCER 109 | garnet_load_balancer: `garnet-broker-alb`, 110 | 111 | // SECRET 112 | garnet_secret: `garnet/secret/brokerdb`, 113 | garnet_api_jwt_secret: `garnet/secret/api`, 114 | 115 | // SECURITY GROUPS 116 | garnet_broker_sg_database: `garnet-broker-database-sg`, 117 | garnet_broker_sg_rds: `garnet-broker-rds-proxy-sg`, 118 | garnet_broker_sg_alb: `garnet-broker-alb-sg`, 119 | garnet_broker_sg_fargate: `garnet-broker-fargate-sg`, 120 | 121 | // GARNET DB 122 | garnet_proxy_rds: `garnet-proxy-rds`, 123 | garnet_db_cluster_id: `garnet-aurora-cluster`, 124 | 125 | // GARNET PRIVATE SUB 126 | 127 | garnet_scorpiobroker_private_notification_queue: scorpiobroker_sqs_object.SCORPIO_TOPICS_PRIVATE_NOTIFICATION, 128 | garnet_scorpiobroker_private_notification_lambda: `${scorpiobroker_sqs_object.SCORPIO_TOPICS_PRIVATE_NOTIFICATION}-lambda`, 129 | 130 | 131 | // GARNET UTILS 132 | 133 | garnet_utils_clean_ecs_taks_lambda :`garnet-utils-clean-ecstasks-lambda`, 134 | garnet_utils_scorpio_sqs_lambda :`garnet-utils-scorpio-cleansqs-lambda`, 135 | garnet_utils_az_lambda :`garnet-utils-getaz-lambda`, 136 | garnet_utils_bucket_create_lambda: `garnet-utils-bucket-create-lambda`, 137 | garnet_utils_bucket_check_lambda: `garnet-utils-bucket-check-lambda`, 138 | garnet_utils_bucket_provider: `garnet-utils-bucket-provider-lambda`, 139 | garnet_utils_sqs_notification_provider: `garnet-utils-sqs-notification-provider-lambda`, 140 | } 141 | 142 | 143 | 144 | export const azlist: any = { 145 | "us-east-2": ["use2-az1", "use2-az2", "use2-az3"], 146 | "us-east-1": ["use1-az1", "use1-az2", "use1-az4", "use1-az5", "use1-az6"], 147 | "us-west-1": ["usw1-az1", "usw1-az3"], 148 | "us-west-2": ["usw2-az1", "usw2-az2", "usw2-az3", "usw2-az4"], 149 | "ap-east-1": ["ape1-az2", "ape1-az3"], 150 | "ap-south-1": ["aps1-az1", "aps1-az2", "aps1-az3"], 151 | "ap-northeast-2": ["apne2-az1", "apne2-az2", "apne2-az3"], 152 | "ap-southeast-1": ["apse1-az1", "apse1-az2", "apse1-az3"], 153 | "ap-southeast-2": ["apse2-az1", "apse2-az2", "apse2-az3"], 154 | "ap-northeast-1": ["apne1-az1", "apne1-az2", "apne1-az4"], 155 | "ca-central-1": ["cac1-az1", "cac1-az2"], 156 | "eu-central-1": ["euc1-az1", "euc1-az2", "euc1-az3"], 157 | "eu-west-1": ["euw1-az1", "euw1-az2", "euw1-az3"], 158 | "eu-west-2": ["euw2-az1", "euw2-az2", "euw2-az3"], 159 | "eu-west-3": ["euw3-az1", "euw3-az3"], 160 | "eu-north-1": ["eun1-az1", "eun1-az2", "eun1-az3"], 161 | "me-south-1": ["mes1-az1", "mes1-az2", "mes1-az3"], 162 | "sa-east-1": ["sae1-az1", "sae1-az2", "sae1-az3"], 163 | "us-gov-west-1": ["usgw1-az1", "usgw1-az2", "usgw1-az3"] 164 | } 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /lib/stacks/garnet-privatesub/garnet-privatesub-stack.ts: -------------------------------------------------------------------------------- 1 | import { Aws, CfnOutput, CustomResource, Duration, Names, NestedStack, NestedStackProps, RemovalPolicy } from "aws-cdk-lib" 2 | import { EndpointType, LambdaRestApi } from "aws-cdk-lib/aws-apigateway" 3 | import { InterfaceVpcEndpoint, Peer, Port, SecurityGroup, Vpc } from "aws-cdk-lib/aws-ec2" 4 | import { AnyPrincipal, Effect, Policy, PolicyDocument, PolicyStatement, Role, ServicePrincipal } from "aws-cdk-lib/aws-iam" 5 | import { Architecture, Code, Function, LayerVersion, Runtime } from "aws-cdk-lib/aws-lambda" 6 | import { Construct } from "constructs" 7 | import { garnet_constant, garnet_nomenclature } from "../../../constants" 8 | import { CfnTopicRule } from "aws-cdk-lib/aws-iot" 9 | import { CfnDeliveryStream } from "aws-cdk-lib/aws-kinesisfirehose" 10 | import { Bucket } from "aws-cdk-lib/aws-s3" 11 | import { LogGroup, RetentionDays } from "aws-cdk-lib/aws-logs" 12 | import { Provider } from "aws-cdk-lib/custom-resources" 13 | import { SqsEventSource } from "aws-cdk-lib/aws-lambda-event-sources" 14 | import { Queue } from "aws-cdk-lib/aws-sqs" 15 | import { deployment_params } from "../../../architecture" 16 | 17 | export interface GarnetPrivateSubProps extends NestedStackProps{ 18 | vpc: Vpc, 19 | bucket_name: string 20 | } 21 | 22 | export class GarnetPrivateSub extends NestedStack { 23 | 24 | public readonly private_sub_endpoint: string 25 | 26 | constructor(scope: Construct, id: string, props: GarnetPrivateSubProps) { 27 | super(scope, id) 28 | 29 | // LAMBDA LAYER (SHARED LIBRARIES) 30 | const layer_lambda_path = `./lib/layers`; 31 | const layer_lambda = new LayerVersion(this, "LayerLambda", { 32 | code: Code.fromAsset(layer_lambda_path), 33 | compatibleRuntimes: [Runtime.NODEJS_22_X], 34 | }) 35 | 36 | // SECURITY GROUP 37 | const sg_garnet_vpc_endpoint = new SecurityGroup(this, 'PrivateSubSecurityGroup', { 38 | securityGroupName: `garnet-private-sub-endpoint-sg`, 39 | vpc: props.vpc, 40 | allowAllOutbound: true 41 | }) 42 | sg_garnet_vpc_endpoint.addIngressRule(Peer.anyIpv4(), Port.tcp(443)) 43 | 44 | // VPC ENDPOINT 45 | const vpc_endpoint = new InterfaceVpcEndpoint(this, 'GarnetPrivateSubEndpoint', { 46 | vpc: props.vpc, 47 | service: { 48 | name: `com.amazonaws.${Aws.REGION}.execute-api`, 49 | port: 443 50 | }, 51 | privateDnsEnabled: true, 52 | securityGroups: [sg_garnet_vpc_endpoint] 53 | }) 54 | 55 | // LAMBDA 56 | const lambda_garnet_private_sub_log = new LogGroup(this, 'LambdaGarnetSubFunctionLogs', { 57 | retention: RetentionDays.ONE_MONTH, 58 | // logGroupName: `garnet-private-sub-lambda-cw-logs`, 59 | removalPolicy: RemovalPolicy.DESTROY 60 | }) 61 | const lambda_garnet_private_sub_path = `${__dirname}/lambda/garnetSub` 62 | const lambda_garnet_private_sub = new Function(this, 'LambdaGarnetSubFunction', { 63 | functionName: garnet_nomenclature.garnet_private_sub_lambda, 64 | logGroup: lambda_garnet_private_sub_log, 65 | description: 'Garnet Private Sub - Function for the private subscription', 66 | runtime: Runtime.NODEJS_22_X, 67 | layers: [layer_lambda], 68 | code: Code.fromAsset(lambda_garnet_private_sub_path), 69 | handler: 'index.handler', 70 | timeout: Duration.seconds(50), 71 | architecture: Architecture.ARM_64, 72 | environment: { 73 | AWSIOTREGION: Aws.REGION 74 | } 75 | }) 76 | lambda_garnet_private_sub.node.addDependency(lambda_garnet_private_sub_log) 77 | lambda_garnet_private_sub.addToRolePolicy(new PolicyStatement({ 78 | actions: ["iot:Publish"], 79 | resources: [ 80 | `arn:aws:iot:${Aws.REGION}:${Aws.ACCOUNT_ID}:topic/garnet/subscriptions/*`, 81 | ] 82 | })) 83 | 84 | // POLICY FOR API 85 | const api_policy = new PolicyDocument({ 86 | statements: [ 87 | new PolicyStatement({ 88 | principals: [new AnyPrincipal], 89 | actions: ['execute-api:Invoke'], 90 | resources: ['execute-api:/*'], 91 | effect: Effect.DENY, 92 | conditions: { 93 | StringNotEquals: { 94 | "aws:SourceVpce": vpc_endpoint.vpcEndpointId 95 | } 96 | } 97 | }), 98 | new PolicyStatement({ 99 | principals: [new AnyPrincipal], 100 | actions: ['execute-api:Invoke'], 101 | resources: ['execute-api:/*'], 102 | effect: Effect.ALLOW 103 | }) 104 | ] 105 | }) 106 | 107 | const api_private_sub = new LambdaRestApi(this, 'ApiPrivateSub', { 108 | restApiName:'garnet-private-sub-endpoint-api', 109 | endpointTypes: [EndpointType.PRIVATE], 110 | handler: lambda_garnet_private_sub, 111 | policy: api_policy, 112 | description: "Garnet Private Endpoint for Subscriptions", 113 | deployOptions: { 114 | stageName: "privatesub" 115 | } 116 | }) 117 | 118 | this.private_sub_endpoint = api_private_sub.url 119 | 120 | new CfnOutput(this, 'ApiEndpoint', { 121 | value: api_private_sub.url, 122 | description: 'Private API Endpoint for Subscriptions' 123 | }) 124 | 125 | 126 | // KINESIS FIREHOSE TO DATALAKE BUCKET 127 | 128 | // DATALAKE BUCKET 129 | const bucket = Bucket.fromBucketName(this, "GarnetBucket", props.bucket_name) 130 | 131 | 132 | // ROLE THAT GRANTS ACCESS TO FIREHOSE TO READ/WRITE BUCKET 133 | const role_firehose = new Role(this, "FirehoseRole", { 134 | assumedBy: new ServicePrincipal("firehose.amazonaws.com"), 135 | }); 136 | bucket.grantReadWrite(role_firehose) 137 | 138 | // KINESIS FIREHOSE DELIVERY STREAM 139 | const kinesis_firehose = new CfnDeliveryStream( this, "GarnetFirehose", { 140 | deliveryStreamName: garnet_nomenclature.garnet_sub_firehose_stream, 141 | deliveryStreamType: "DirectPut", 142 | extendedS3DestinationConfiguration: { 143 | bucketArn: bucket.bucketArn, 144 | roleArn: role_firehose.roleArn, 145 | bufferingHints: { 146 | intervalInSeconds: 60, 147 | sizeInMBs: 64, 148 | }, 149 | processingConfiguration: { 150 | enabled: true, 151 | processors: [ 152 | { 153 | type: "MetadataExtraction", 154 | parameters: [ 155 | { 156 | parameterName: "MetadataExtractionQuery", 157 | parameterValue: "{type:.type}", 158 | }, 159 | { 160 | parameterName: "JsonParsingEngine", 161 | parameterValue: "JQ-1.6", 162 | }, 163 | ], 164 | }, 165 | ], 166 | }, 167 | dynamicPartitioningConfiguration: { 168 | enabled: true, 169 | }, 170 | prefix: `type=!{partitionKeyFromQuery:type}/dt=!{timestamp:yyyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/`, 171 | errorOutputPrefix: `type=!{firehose:error-output-type}/dt=!{timestamp:yyy}-!{timestamp:MM}-!{timestamp:dd}-!{timestamp:HH}/`, 172 | }, 173 | } 174 | ) 175 | 176 | // IOT RULE THAT LISTENS TO SUBSCRIPTIONS AND PUSH TO FIREHOSE 177 | const iot_rule_sub_name = garnet_nomenclature.garnet_subscriptions_rule 178 | 179 | const iot_rule_sub_role = new Role(this, "RoleGarnetIotRuleIngestion", { 180 | assumedBy: new ServicePrincipal("iot.amazonaws.com"), 181 | }) 182 | 183 | const iot_rule_sub_policy = new Policy(this, 'PolicyGarnetIotRuleIngestion', { 184 | statements: [ 185 | new PolicyStatement({ 186 | resources: [ `${kinesis_firehose.attrArn}` ], 187 | actions: [ 188 | "firehose:DescribeDeliveryStream", 189 | "firehose:ListDeliveryStreams", 190 | "firehose:ListTagsForDeliveryStream", 191 | "firehose:PutRecord", 192 | "firehose:PutRecordBatch", 193 | ] 194 | }) 195 | ] 196 | }) 197 | 198 | iot_rule_sub_role.attachInlinePolicy(iot_rule_sub_policy) 199 | 200 | const iot_rule_sub = new CfnTopicRule(this, "IotRuleSub", { 201 | ruleName: iot_rule_sub_name, 202 | topicRulePayload: { 203 | awsIotSqlVersion: "2016-03-23", 204 | ruleDisabled: false, 205 | sql: `SELECT * FROM 'garnet/subscriptions/+'`, 206 | actions: [ 207 | { 208 | firehose: { 209 | deliveryStreamName: kinesis_firehose.ref, 210 | roleArn: iot_rule_sub_role.roleArn, 211 | separator: "\n", 212 | }, 213 | }, 214 | ], 215 | }, 216 | }) 217 | 218 | 219 | 220 | // NEW PRIVATE SQS BASED PRIVATE SUB 221 | 222 | // garnet_scorpiobroker_private_notification_queue 223 | 224 | // HERE A CUSTOM RESOURCE THAT CHECK IF SQS EXISTS IF NOT THEN CREATE IT AND GIVE BACK THE QUEUE ARN. 225 | // CUSTOM RESOURCE WITH A LAMBDA THAT WILL CREATE SQS PRIVATE NOTIFICATION QUEUE IF IT DOES NOT EXIST 226 | 227 | const lambda_sqs_create_logs = new LogGroup(this, 'LambdaSqsCreateFunctionLogs', { 228 | retention: RetentionDays.ONE_MONTH, 229 | removalPolicy: RemovalPolicy.DESTROY 230 | }) 231 | 232 | const lambda_sqs_private_notification_path = `${__dirname}/lambda/sqsCreate` 233 | const lambda_sqs_private_notification_lambda = new Function(this, 'SqsPrivateNotificationCreateFunction', { 234 | functionName: garnet_nomenclature.garnet_scorpiobroker_private_notification_lambda, 235 | description: 'Garnet Utils - Function that creates SQS private notification Queue if it does not already exist', 236 | runtime: Runtime.NODEJS_LATEST, 237 | code: Code.fromAsset(lambda_sqs_private_notification_path), 238 | logGroup: lambda_sqs_create_logs, 239 | handler: 'index.handler', 240 | timeout: Duration.seconds(30), 241 | architecture: Architecture.ARM_64, 242 | environment: { 243 | QUEUE_NAME: garnet_nomenclature.garnet_scorpiobroker_private_notification_queue 244 | } 245 | }) 246 | 247 | 248 | lambda_sqs_private_notification_lambda.addToRolePolicy(new PolicyStatement({ 249 | actions: [ 250 | "sqs:CreateQueue", 251 | "sqs:GetQueueUrl", 252 | "sqs:GetQueueAttributes" 253 | ], 254 | resources: [`arn:aws:sqs:${Aws.REGION}:${Aws.ACCOUNT_ID}:garnet-*`] 255 | })) 256 | 257 | lambda_sqs_private_notification_lambda.node.addDependency(lambda_sqs_create_logs) 258 | 259 | 260 | // CHECK QUEUE 261 | 262 | 263 | const lambda_sqs_private_check_logs = new LogGroup(this, 'LambdaSqsCheckFunctionLogs', { 264 | retention: RetentionDays.ONE_MONTH, 265 | removalPolicy: RemovalPolicy.DESTROY 266 | }) 267 | 268 | const lambda_sqs_private_check_path = `${__dirname}/lambda/sqsCheck` 269 | const lambda_sqs_private_check = new Function(this, 'SqsPrivateCheckFunction', { 270 | functionName: `garnet-utils-sqs-check-lambda`, 271 | description: 'Garnet Utils - Function that check if SQS Private Queue exists', 272 | runtime: Runtime.NODEJS_LATEST, 273 | code: Code.fromAsset(lambda_sqs_private_check_path), 274 | handler: 'index.handler', 275 | timeout: Duration.seconds(50), 276 | logGroup: lambda_sqs_private_check_logs, 277 | architecture: Architecture.ARM_64, 278 | environment: { 279 | QUEUE_NAME: garnet_nomenclature.garnet_scorpiobroker_private_notification_queue 280 | } 281 | }) 282 | 283 | lambda_sqs_private_check.node.addDependency(lambda_sqs_private_check_logs) 284 | 285 | lambda_sqs_private_check.addToRolePolicy(new PolicyStatement({ 286 | actions: [ 287 | "sqs:GetQueueUrl", 288 | "sqs:GetQueueAttributes" 289 | ], 290 | resources: [`arn:aws:sqs:${Aws.REGION}:${Aws.ACCOUNT_ID}:garnet-*`] 291 | })) 292 | 293 | 294 | 295 | const sqs_private_provider_log = new LogGroup(this, 'LambdaCustomSqsNotificationProviderLogs', { 296 | retention: RetentionDays.ONE_MONTH, 297 | // logGroupName: `garnet-provider-custom-bucket-lambda-cw-logs`, 298 | removalPolicy: RemovalPolicy.DESTROY 299 | }) 300 | 301 | const sqs_private_provider = new Provider(this, 'CustomSqsProvider', { 302 | onEventHandler: lambda_sqs_private_notification_lambda, 303 | isCompleteHandler: lambda_sqs_private_check, 304 | providerFunctionName: garnet_nomenclature.garnet_utils_sqs_notification_provider, 305 | logGroup: sqs_private_provider_log, 306 | }) 307 | 308 | sqs_private_provider.node.addDependency(sqs_private_provider_log) 309 | 310 | const sqs_private_resource = new CustomResource(this, 'CustomSqsNotificationResource', { 311 | serviceToken: sqs_private_provider.serviceToken, 312 | }) 313 | 314 | const sqs_name = sqs_private_resource.getAtt('queue_name').toString() 315 | 316 | // LAMBDA SUB PRIVATE 317 | const lambda_garnet_sqs_private_sub_log = new LogGroup(this, 'LambdaGarnetSqsSubFunctionLogs', { 318 | retention: RetentionDays.ONE_MONTH, 319 | // logGroupName: `garnet-private-sub-lambda-cw-logs`, 320 | removalPolicy: RemovalPolicy.DESTROY 321 | }) 322 | const lambda_garnet_sqs_private_sub_path = `${__dirname}/lambda/garnetSubSqs` 323 | const lambda_garnet_sqs_private_sub = new Function(this, 'LambdaGarnetSubSqsFunction', { 324 | functionName: garnet_nomenclature.garnet_private_sub_sqs_lambda, 325 | logGroup: lambda_garnet_sqs_private_sub_log, 326 | description: 'Garnet Private Sub - Function for the private subscription from SQS', 327 | runtime: Runtime.NODEJS_LATEST, 328 | layers: [layer_lambda], 329 | code: Code.fromAsset(lambda_garnet_sqs_private_sub_path), 330 | handler: 'index.handler', 331 | timeout: Duration.seconds(25), 332 | architecture: Architecture.ARM_64, 333 | environment: { 334 | AWSIOTREGION: Aws.REGION 335 | } 336 | }) 337 | lambda_garnet_sqs_private_sub.node.addDependency(lambda_garnet_sqs_private_sub_log) 338 | lambda_garnet_sqs_private_sub.addToRolePolicy(new PolicyStatement({ 339 | actions: ["iot:Publish"], 340 | resources: [ 341 | `arn:aws:iot:${Aws.REGION}:${Aws.ACCOUNT_ID}:topic/garnet/subscriptions/*`, 342 | ] 343 | })) 344 | 345 | 346 | 347 | const sqs_private_queue = Queue.fromQueueArn(this, `SqsPrivateQueue`, `arn:aws:sqs:${Aws.REGION}:${Aws.ACCOUNT_ID}:${sqs_name}`) 348 | 349 | sqs_private_queue.node.addDependency(sqs_private_resource) 350 | 351 | lambda_garnet_sqs_private_sub.addToRolePolicy( 352 | new PolicyStatement({ 353 | actions: [ 354 | "sqs:ReceiveMessage", 355 | "sqs:DeleteMessage", 356 | "sqs:GetQueueAttributes", 357 | ], 358 | resources: [`${sqs_private_queue.queueArn}`], 359 | }) 360 | ) 361 | 362 | lambda_garnet_sqs_private_sub.addEventSource( 363 | new SqsEventSource(sqs_private_queue, { 364 | batchSize: deployment_params.lambda_broker_batch_size, 365 | maxBatchingWindow: Duration.seconds(deployment_params.lambda_broker_batch_window), 366 | maxConcurrency: deployment_params.lambda_broker_concurent_sqs 367 | }) 368 | ) 369 | 370 | 371 | } 372 | } 373 | -------------------------------------------------------------------------------- /lib/layers/nodejs/package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs", 3 | "version": "1.0.0", 4 | "lockfileVersion": 3, 5 | "requires": true, 6 | "packages": { 7 | "": { 8 | "name": "nodejs", 9 | "version": "1.0.0", 10 | "license": "ISC", 11 | "dependencies": { 12 | "axios": "^1.12.2", 13 | "jsonwebtoken": "^9.0.2" 14 | } 15 | }, 16 | "node_modules/asynckit": { 17 | "version": "0.4.0", 18 | "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", 19 | "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==", 20 | "license": "MIT" 21 | }, 22 | "node_modules/axios": { 23 | "version": "1.12.2", 24 | "resolved": "https://registry.npmjs.org/axios/-/axios-1.12.2.tgz", 25 | "integrity": "sha512-vMJzPewAlRyOgxV2dU0Cuz2O8zzzx9VYtbJOaBgXFeLc4IV/Eg50n4LowmehOOR61S8ZMpc2K5Sa7g6A4jfkUw==", 26 | "license": "MIT", 27 | "dependencies": { 28 | "follow-redirects": "^1.15.6", 29 | "form-data": "^4.0.4", 30 | "proxy-from-env": "^1.1.0" 31 | } 32 | }, 33 | "node_modules/buffer-equal-constant-time": { 34 | "version": "1.0.1", 35 | "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", 36 | "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", 37 | "license": "BSD-3-Clause" 38 | }, 39 | "node_modules/call-bind-apply-helpers": { 40 | "version": "1.0.2", 41 | "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", 42 | "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", 43 | "license": "MIT", 44 | "dependencies": { 45 | "es-errors": "^1.3.0", 46 | "function-bind": "^1.1.2" 47 | }, 48 | "engines": { 49 | "node": ">= 0.4" 50 | } 51 | }, 52 | "node_modules/combined-stream": { 53 | "version": "1.0.8", 54 | "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", 55 | "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==", 56 | "license": "MIT", 57 | "dependencies": { 58 | "delayed-stream": "~1.0.0" 59 | }, 60 | "engines": { 61 | "node": ">= 0.8" 62 | } 63 | }, 64 | "node_modules/delayed-stream": { 65 | "version": "1.0.0", 66 | "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", 67 | "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==", 68 | "license": "MIT", 69 | "engines": { 70 | "node": ">=0.4.0" 71 | } 72 | }, 73 | "node_modules/dunder-proto": { 74 | "version": "1.0.1", 75 | "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", 76 | "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", 77 | "license": "MIT", 78 | "dependencies": { 79 | "call-bind-apply-helpers": "^1.0.1", 80 | "es-errors": "^1.3.0", 81 | "gopd": "^1.2.0" 82 | }, 83 | "engines": { 84 | "node": ">= 0.4" 85 | } 86 | }, 87 | "node_modules/ecdsa-sig-formatter": { 88 | "version": "1.0.11", 89 | "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", 90 | "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", 91 | "license": "Apache-2.0", 92 | "dependencies": { 93 | "safe-buffer": "^5.0.1" 94 | } 95 | }, 96 | "node_modules/es-define-property": { 97 | "version": "1.0.1", 98 | "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", 99 | "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==", 100 | "license": "MIT", 101 | "engines": { 102 | "node": ">= 0.4" 103 | } 104 | }, 105 | "node_modules/es-errors": { 106 | "version": "1.3.0", 107 | "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", 108 | "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", 109 | "license": "MIT", 110 | "engines": { 111 | "node": ">= 0.4" 112 | } 113 | }, 114 | "node_modules/es-object-atoms": { 115 | "version": "1.1.1", 116 | "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.1.tgz", 117 | "integrity": "sha512-FGgH2h8zKNim9ljj7dankFPcICIK9Cp5bm+c2gQSYePhpaG5+esrLODihIorn+Pe6FGJzWhXQotPv73jTaldXA==", 118 | "license": "MIT", 119 | "dependencies": { 120 | "es-errors": "^1.3.0" 121 | }, 122 | "engines": { 123 | "node": ">= 0.4" 124 | } 125 | }, 126 | "node_modules/es-set-tostringtag": { 127 | "version": "2.1.0", 128 | "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz", 129 | "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==", 130 | "license": "MIT", 131 | "dependencies": { 132 | "es-errors": "^1.3.0", 133 | "get-intrinsic": "^1.2.6", 134 | "has-tostringtag": "^1.0.2", 135 | "hasown": "^2.0.2" 136 | }, 137 | "engines": { 138 | "node": ">= 0.4" 139 | } 140 | }, 141 | "node_modules/follow-redirects": { 142 | "version": "1.15.6", 143 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", 144 | "integrity": "sha512-wWN62YITEaOpSK584EZXJafH1AGpO8RVgElfkuXbTOrPX4fIfOyEpW/CsiNd8JdYrAoOvafRTOEnvsO++qCqFA==", 145 | "funding": [ 146 | { 147 | "type": "individual", 148 | "url": "https://github.com/sponsors/RubenVerborgh" 149 | } 150 | ], 151 | "engines": { 152 | "node": ">=4.0" 153 | }, 154 | "peerDependenciesMeta": { 155 | "debug": { 156 | "optional": true 157 | } 158 | } 159 | }, 160 | "node_modules/form-data": { 161 | "version": "4.0.4", 162 | "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.4.tgz", 163 | "integrity": "sha512-KrGhL9Q4zjj0kiUt5OO4Mr/A/jlI2jDYs5eHBpYHPcBEVSiipAvn2Ko2HnPe20rmcuuvMHNdZFp+4IlGTMF0Ow==", 164 | "license": "MIT", 165 | "dependencies": { 166 | "asynckit": "^0.4.0", 167 | "combined-stream": "^1.0.8", 168 | "es-set-tostringtag": "^2.1.0", 169 | "hasown": "^2.0.2", 170 | "mime-types": "^2.1.12" 171 | }, 172 | "engines": { 173 | "node": ">= 6" 174 | } 175 | }, 176 | "node_modules/function-bind": { 177 | "version": "1.1.2", 178 | "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", 179 | "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", 180 | "license": "MIT", 181 | "funding": { 182 | "url": "https://github.com/sponsors/ljharb" 183 | } 184 | }, 185 | "node_modules/get-intrinsic": { 186 | "version": "1.3.0", 187 | "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", 188 | "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", 189 | "license": "MIT", 190 | "dependencies": { 191 | "call-bind-apply-helpers": "^1.0.2", 192 | "es-define-property": "^1.0.1", 193 | "es-errors": "^1.3.0", 194 | "es-object-atoms": "^1.1.1", 195 | "function-bind": "^1.1.2", 196 | "get-proto": "^1.0.1", 197 | "gopd": "^1.2.0", 198 | "has-symbols": "^1.1.0", 199 | "hasown": "^2.0.2", 200 | "math-intrinsics": "^1.1.0" 201 | }, 202 | "engines": { 203 | "node": ">= 0.4" 204 | }, 205 | "funding": { 206 | "url": "https://github.com/sponsors/ljharb" 207 | } 208 | }, 209 | "node_modules/get-proto": { 210 | "version": "1.0.1", 211 | "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", 212 | "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", 213 | "license": "MIT", 214 | "dependencies": { 215 | "dunder-proto": "^1.0.1", 216 | "es-object-atoms": "^1.0.0" 217 | }, 218 | "engines": { 219 | "node": ">= 0.4" 220 | } 221 | }, 222 | "node_modules/gopd": { 223 | "version": "1.2.0", 224 | "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", 225 | "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==", 226 | "license": "MIT", 227 | "engines": { 228 | "node": ">= 0.4" 229 | }, 230 | "funding": { 231 | "url": "https://github.com/sponsors/ljharb" 232 | } 233 | }, 234 | "node_modules/has-symbols": { 235 | "version": "1.1.0", 236 | "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", 237 | "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==", 238 | "license": "MIT", 239 | "engines": { 240 | "node": ">= 0.4" 241 | }, 242 | "funding": { 243 | "url": "https://github.com/sponsors/ljharb" 244 | } 245 | }, 246 | "node_modules/has-tostringtag": { 247 | "version": "1.0.2", 248 | "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz", 249 | "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==", 250 | "license": "MIT", 251 | "dependencies": { 252 | "has-symbols": "^1.0.3" 253 | }, 254 | "engines": { 255 | "node": ">= 0.4" 256 | }, 257 | "funding": { 258 | "url": "https://github.com/sponsors/ljharb" 259 | } 260 | }, 261 | "node_modules/hasown": { 262 | "version": "2.0.2", 263 | "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", 264 | "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", 265 | "license": "MIT", 266 | "dependencies": { 267 | "function-bind": "^1.1.2" 268 | }, 269 | "engines": { 270 | "node": ">= 0.4" 271 | } 272 | }, 273 | "node_modules/jsonwebtoken": { 274 | "version": "9.0.2", 275 | "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.2.tgz", 276 | "integrity": "sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==", 277 | "license": "MIT", 278 | "dependencies": { 279 | "jws": "^3.2.2", 280 | "lodash.includes": "^4.3.0", 281 | "lodash.isboolean": "^3.0.3", 282 | "lodash.isinteger": "^4.0.4", 283 | "lodash.isnumber": "^3.0.3", 284 | "lodash.isplainobject": "^4.0.6", 285 | "lodash.isstring": "^4.0.1", 286 | "lodash.once": "^4.0.0", 287 | "ms": "^2.1.1", 288 | "semver": "^7.5.4" 289 | }, 290 | "engines": { 291 | "node": ">=12", 292 | "npm": ">=6" 293 | } 294 | }, 295 | "node_modules/jwa": { 296 | "version": "1.4.1", 297 | "resolved": "https://registry.npmjs.org/jwa/-/jwa-1.4.1.tgz", 298 | "integrity": "sha512-qiLX/xhEEFKUAJ6FiBMbes3w9ATzyk5W7Hvzpa/SLYdxNtng+gcurvrI7TbACjIXlsJyr05/S1oUhZrc63evQA==", 299 | "license": "MIT", 300 | "dependencies": { 301 | "buffer-equal-constant-time": "1.0.1", 302 | "ecdsa-sig-formatter": "1.0.11", 303 | "safe-buffer": "^5.0.1" 304 | } 305 | }, 306 | "node_modules/jws": { 307 | "version": "3.2.2", 308 | "resolved": "https://registry.npmjs.org/jws/-/jws-3.2.2.tgz", 309 | "integrity": "sha512-YHlZCB6lMTllWDtSPHz/ZXTsi8S00usEV6v1tjq8tOUZzw7DpSDWVXjXDre6ed1w/pd495ODpHZYSdkRTsa0HA==", 310 | "license": "MIT", 311 | "dependencies": { 312 | "jwa": "^1.4.1", 313 | "safe-buffer": "^5.0.1" 314 | } 315 | }, 316 | "node_modules/lodash.includes": { 317 | "version": "4.3.0", 318 | "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", 319 | "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", 320 | "license": "MIT" 321 | }, 322 | "node_modules/lodash.isboolean": { 323 | "version": "3.0.3", 324 | "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", 325 | "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", 326 | "license": "MIT" 327 | }, 328 | "node_modules/lodash.isinteger": { 329 | "version": "4.0.4", 330 | "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", 331 | "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", 332 | "license": "MIT" 333 | }, 334 | "node_modules/lodash.isnumber": { 335 | "version": "3.0.3", 336 | "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", 337 | "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", 338 | "license": "MIT" 339 | }, 340 | "node_modules/lodash.isplainobject": { 341 | "version": "4.0.6", 342 | "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", 343 | "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", 344 | "license": "MIT" 345 | }, 346 | "node_modules/lodash.isstring": { 347 | "version": "4.0.1", 348 | "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", 349 | "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", 350 | "license": "MIT" 351 | }, 352 | "node_modules/lodash.once": { 353 | "version": "4.1.1", 354 | "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", 355 | "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", 356 | "license": "MIT" 357 | }, 358 | "node_modules/math-intrinsics": { 359 | "version": "1.1.0", 360 | "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", 361 | "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==", 362 | "license": "MIT", 363 | "engines": { 364 | "node": ">= 0.4" 365 | } 366 | }, 367 | "node_modules/mime-db": { 368 | "version": "1.52.0", 369 | "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", 370 | "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==", 371 | "license": "MIT", 372 | "engines": { 373 | "node": ">= 0.6" 374 | } 375 | }, 376 | "node_modules/mime-types": { 377 | "version": "2.1.35", 378 | "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz", 379 | "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==", 380 | "license": "MIT", 381 | "dependencies": { 382 | "mime-db": "1.52.0" 383 | }, 384 | "engines": { 385 | "node": ">= 0.6" 386 | } 387 | }, 388 | "node_modules/ms": { 389 | "version": "2.1.3", 390 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", 391 | "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", 392 | "license": "MIT" 393 | }, 394 | "node_modules/proxy-from-env": { 395 | "version": "1.1.0", 396 | "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", 397 | "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" 398 | }, 399 | "node_modules/safe-buffer": { 400 | "version": "5.2.1", 401 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", 402 | "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", 403 | "funding": [ 404 | { 405 | "type": "github", 406 | "url": "https://github.com/sponsors/feross" 407 | }, 408 | { 409 | "type": "patreon", 410 | "url": "https://www.patreon.com/feross" 411 | }, 412 | { 413 | "type": "consulting", 414 | "url": "https://feross.org/support" 415 | } 416 | ], 417 | "license": "MIT" 418 | }, 419 | "node_modules/semver": { 420 | "version": "7.6.3", 421 | "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", 422 | "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", 423 | "license": "ISC", 424 | "bin": { 425 | "semver": "bin/semver.js" 426 | }, 427 | "engines": { 428 | "node": ">=10" 429 | } 430 | } 431 | } 432 | } 433 | -------------------------------------------------------------------------------- /lib/stacks/garnet-ops/garnet-ops-stack.ts: -------------------------------------------------------------------------------- 1 | import { Aws, Duration, NestedStack, NestedStackProps } from "aws-cdk-lib"; 2 | import { Dashboard, Metric, Row, SingleValueWidget } from "aws-cdk-lib/aws-cloudwatch"; 3 | import { Construct } from "constructs"; 4 | import { garnet_bucket, garnet_nomenclature, scorpiobroker_sqs_object } from "../../../constants"; 5 | import { deployment_params } from "../../../architecture"; 6 | 7 | 8 | export interface GarnetOpsProps extends NestedStackProps{ 9 | 10 | } 11 | 12 | export class GarnetOps extends NestedStack { 13 | 14 | constructor(scope: Construct, id: string, props: GarnetOpsProps) { 15 | super(scope, id, props) 16 | 17 | // CLOUDWATCH DASHBOARD 18 | const garnet_dashboard = new Dashboard(this, `GarnetCwDashboard`, { 19 | dashboardName: `Garnet-Ops-Dashboard-${Aws.REGION}` 20 | }) 21 | 22 | 23 | const set_service_widgets = (GarnetDashboard: Dashboard, Name:string, ClusterName:string, ServiceName: string, DiscoveryName: string) => { 24 | 25 | let garnet_broker_service_metrics: Array = [] 26 | garnet_broker_service_metrics = [ 27 | new Metric({ 28 | label: `Scorpio - ${Name} - Memory %`, 29 | namespace: 'AWS/ECS', 30 | metricName: 'MemoryUtilization', 31 | dimensionsMap: { 32 | ClusterName: ClusterName, 33 | ServiceName: ServiceName 34 | }, 35 | statistic: 'Average', 36 | period: Duration.minutes(30) 37 | 38 | }), 39 | new Metric({ 40 | label: `Scorpio - ${Name} - CPU %`, 41 | namespace: 'AWS/ECS', 42 | metricName: 'CPUUtilization', 43 | dimensionsMap: { 44 | ClusterName: ClusterName, 45 | ServiceName: ServiceName 46 | }, 47 | statistic: 'Average', 48 | period: Duration.minutes(30) 49 | 50 | }), 51 | new Metric({ 52 | label: `Scorpio - ${Name} - Processed Bytes`, 53 | namespace: 'AWS/ECS', 54 | metricName: 'ProcessedBytes', 55 | dimensionsMap: { 56 | ClusterName: ClusterName, 57 | ServiceName: ServiceName, 58 | DiscoveryName: DiscoveryName 59 | }, 60 | statistic: 'Average', 61 | period: Duration.minutes(30) 62 | 63 | }), 64 | new Metric({ 65 | label: `Scorpio - ${Name} - Active Connection Count`, 66 | namespace: 'AWS/ECS', 67 | metricName: 'ActiveConnectionCount', 68 | dimensionsMap: { 69 | ClusterName: ClusterName, 70 | ServiceName: ServiceName, 71 | DiscoveryName: DiscoveryName 72 | }, 73 | statistic: 'IQM', 74 | period: Duration.minutes(30) 75 | }), 76 | ] 77 | 78 | 79 | let garnet_broker_service_widget = new SingleValueWidget({ 80 | title: `Garnet Scorpio - ${Name}`, 81 | width: 24, 82 | period: Duration.seconds(60), 83 | metrics: garnet_broker_service_metrics, 84 | setPeriodToTimeRange: true 85 | }) 86 | GarnetDashboard.addWidgets(garnet_broker_service_widget) 87 | } 88 | 89 | const set_lambda_widgets = (label:string, functionName: string ) => { 90 | let garnet_lambda_metrics = [ 91 | new Metric({ 92 | label: `${label}- Invocations`, 93 | namespace: 'AWS/Lambda', 94 | metricName: 'Invocations', 95 | dimensionsMap: { 96 | FunctionName: functionName 97 | }, 98 | statistic: 'Sum', 99 | period: Duration.minutes(30) 100 | 101 | }), 102 | new Metric({ 103 | label: `${label} - Errors`, 104 | namespace: 'AWS/Lambda', 105 | metricName: 'Errors', 106 | dimensionsMap: { 107 | FunctionName: functionName 108 | }, 109 | statistic: 'Sum', 110 | period: Duration.minutes(30) 111 | 112 | }), 113 | new Metric({ 114 | label: `${label} - Duration`, 115 | namespace: 'AWS/Lambda', 116 | metricName: 'Duration', 117 | dimensionsMap: { 118 | FunctionName: functionName 119 | }, 120 | statistic: 'IQM', 121 | period: Duration.minutes(30) 122 | 123 | }), 124 | new Metric({ 125 | label: `${label} - ConcurrentExecutions`, 126 | namespace: 'AWS/Lambda', 127 | metricName: 'ConcurrentExecutions', 128 | dimensionsMap: { 129 | FunctionName: functionName 130 | }, 131 | statistic: 'IQM', 132 | period: Duration.minutes(30) 133 | }), 134 | new Metric({ 135 | label: `${label} - Throttles`, 136 | namespace: 'AWS/Lambda', 137 | metricName: 'Throttles', 138 | dimensionsMap: { 139 | FunctionName: functionName 140 | }, 141 | statistic: 'Sum', 142 | period: Duration.minutes(30) 143 | 144 | }) 145 | ] 146 | return new SingleValueWidget({ 147 | title: `Garnet Ingestion- Lambda ${label}`, 148 | width: 24, 149 | period: Duration.seconds(60), 150 | metrics: garnet_lambda_metrics, 151 | setPeriodToTimeRange: true 152 | }) 153 | } 154 | 155 | const set_sqs_widgets = (label:string, queueName:string) => { 156 | let garnet_sqs_metrics = [ 157 | new Metric({ 158 | label: `${label} - Nb Message Sent`, 159 | namespace: 'AWS/SQS', 160 | metricName: 'NumberOfMessagesSent', 161 | dimensionsMap: { 162 | QueueName: queueName 163 | }, 164 | statistic: 'Sum', 165 | period: Duration.minutes(30) 166 | }), 167 | new Metric({ 168 | label: `${label}- Sent Message Size`, 169 | namespace: 'AWS/SQS', 170 | metricName: 'SentMessageSize', 171 | dimensionsMap: { 172 | QueueName: queueName 173 | }, 174 | statistic: 'IQM', 175 | period: Duration.minutes(30) 176 | }), 177 | new Metric({ 178 | label: `${label} - Nb Message Received`, 179 | namespace: 'AWS/SQS', 180 | metricName: 'NumberOfMessagesReceived', 181 | dimensionsMap: { 182 | QueueName: queueName 183 | }, 184 | statistic: 'Sum', 185 | period: Duration.minutes(30) 186 | }), 187 | new Metric({ 188 | label: `${label} - Approx Age Oldest Message`, 189 | namespace: 'AWS/SQS', 190 | metricName: 'ApproximateAgeOfOldestMessage', 191 | dimensionsMap: { 192 | QueueName: queueName 193 | }, 194 | statistic: 'IQM', 195 | period: Duration.minutes(30) 196 | }) 197 | ] 198 | return new SingleValueWidget({ 199 | title: `${label}`, 200 | width: 24, 201 | period: Duration.seconds(60), 202 | metrics: garnet_sqs_metrics, 203 | setPeriodToTimeRange: true 204 | }) 205 | } 206 | 207 | 208 | 209 | 210 | 211 | // GARNET INGESTION LAMBDA UPDATE BROKER 212 | 213 | let garnet_ingestion_lambda_update_broker_widget = set_lambda_widgets('Ingestion Lambda', garnet_nomenclature.garnet_ingestion_update_broker_lambda) 214 | 215 | 216 | // GARNET SQS INGESTION 217 | let garnet_ingestion_sqs_broker_widget = set_sqs_widgets('Garnet SQS Ingestion', garnet_nomenclature.garnet_ingestion_queue) 218 | 219 | // GARNET DATALAKE 220 | let garnet_datalake_metrics = [ 221 | new Metric({ 222 | label: 'Garnet Lake - Number of Objects Stored', 223 | namespace: 'AWS/S3', 224 | metricName: 'NumberOfObjects', 225 | dimensionsMap: { 226 | BucketName: garnet_bucket, 227 | StorageType: "AllStorageTypes" 228 | }, 229 | statistic: 'Sum', 230 | period: Duration.minutes(30) 231 | }), 232 | new Metric({ 233 | label: 'Garnet Lake - Bytes Stored', 234 | namespace: 'AWS/S3', 235 | metricName: 'BucketSizeBytes', 236 | dimensionsMap: { 237 | BucketName: garnet_bucket, 238 | StorageType: "StandardStorage" 239 | }, 240 | statistic: 'Sum', 241 | period: Duration.minutes(30) 242 | }), 243 | new Metric({ 244 | label: 'Garnet Lake - Firehose', 245 | namespace: 'AWS/Firehose', 246 | metricName: 'DeliveryToS3.Records', 247 | dimensionsMap: { 248 | DeliveryStreamName: garnet_nomenclature.garnet_lake_firehose_stream, 249 | }, 250 | statistic: 'Sum', 251 | period: Duration.minutes(30) 252 | }) 253 | ] 254 | 255 | let garnet_datalake_widget = new SingleValueWidget({ 256 | title: 'Garnet Data Lake', 257 | width: 24, 258 | period: Duration.seconds(60), 259 | metrics: garnet_datalake_metrics, 260 | setPeriodToTimeRange: true 261 | }) 262 | 263 | 264 | let garnet_ingestion = new Row( 265 | garnet_ingestion_lambda_update_broker_widget, 266 | garnet_ingestion_sqs_broker_widget, 267 | garnet_datalake_widget) 268 | 269 | 270 | garnet_dashboard.addWidgets(garnet_ingestion) 271 | 272 | 273 | 274 | let garnet_broker_db_metrics = [ 275 | new Metric({ 276 | label: 'Garnet Broker Aurora - DataBase Connections', 277 | namespace: 'AWS/RDS', 278 | metricName: 'DatabaseConnections', 279 | dimensionsMap: { 280 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 281 | }, 282 | statistic: 'IQM', 283 | period: Duration.minutes(30) 284 | }), 285 | new Metric({ 286 | label: 'Garnet Broker Aurora - ACU Utilization', 287 | namespace: 'AWS/RDS', 288 | metricName: 'ACUUtilization', 289 | dimensionsMap: { 290 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 291 | }, 292 | statistic: 'IQM' 293 | }), 294 | new Metric({ 295 | label: 'Garnet Broker Aurora - ACU Capacity', 296 | namespace: 'AWS/RDS', 297 | metricName: 'ServerlessDatabaseCapacity', 298 | dimensionsMap: { 299 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 300 | }, 301 | statistic: 'IQM', 302 | period: Duration.minutes(30) 303 | }), 304 | new Metric({ 305 | label: 'Garnet Broker Aurora - CPU Utilization', 306 | namespace: 'AWS/RDS', 307 | metricName: 'CPUUtilization', 308 | dimensionsMap: { 309 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 310 | }, 311 | statistic: 'IQM' 312 | }), 313 | new Metric({ 314 | label: 'Garnet Broker Aurora - Write Ops', 315 | namespace: 'AWS/RDS', 316 | metricName: 'WriteIOPS', 317 | dimensionsMap: { 318 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 319 | }, 320 | statistic: 'IQM' 321 | }), 322 | new Metric({ 323 | label: 'Garnet Broker Aurora - Write Latency', 324 | namespace: 'AWS/RDS', 325 | metricName: 'WriteLatency', 326 | dimensionsMap: { 327 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 328 | }, 329 | statistic: 'IQM', 330 | period: Duration.minutes(30) 331 | }), 332 | new Metric({ 333 | label: 'Garnet Broker Aurora - Read Ops', 334 | namespace: 'AWS/RDS', 335 | metricName: 'ReadIOPS', 336 | dimensionsMap: { 337 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 338 | }, 339 | statistic: 'IQM', 340 | period: Duration.minutes(30) 341 | }), 342 | new Metric({ 343 | label: 'Garnet Broker Aurora - Read Latency', 344 | namespace: 'AWS/RDS', 345 | metricName: 'ReadLatency', 346 | dimensionsMap: { 347 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 348 | }, 349 | statistic: 'IQM' 350 | }), 351 | new Metric({ 352 | label: 'Garnet Broker Aurora - Freeable Memory', 353 | namespace: 'AWS/RDS', 354 | metricName: 'FreeableMemory', 355 | dimensionsMap: { 356 | DBClusterIdentifier: garnet_nomenclature.garnet_db_cluster_id 357 | }, 358 | statistic: 'IQM', 359 | period: Duration.minutes(30) 360 | }), 361 | ] 362 | 363 | let garnet_broker_db_widget = new SingleValueWidget({ 364 | title: 'Garnet Broker - Database', 365 | width: 24, 366 | period: Duration.seconds(60), 367 | metrics: garnet_broker_db_metrics, 368 | setPeriodToTimeRange: true 369 | }) 370 | 371 | garnet_dashboard.addWidgets(garnet_broker_db_widget) 372 | 373 | 374 | let service_widget: any = [] 375 | 376 | if (deployment_params.architecture == 'distributed'){ 377 | 378 | service_widget = [{ 379 | Name: `Entity Manager`, 380 | DiscoveryName: garnet_nomenclature.garnet_broker_entitymanager 381 | }, 382 | { 383 | Name: `Subscription Manager`, 384 | DiscoveryName: garnet_nomenclature.garnet_broker_subscriptionmanager 385 | }, 386 | { 387 | Name: `Query Manager`, 388 | DiscoveryName: garnet_nomenclature.garnet_broker_querymanager 389 | }, 390 | { 391 | Name: `At Context Server`, 392 | DiscoveryName: garnet_nomenclature.garnet_broker_atcontextserver 393 | }, 394 | 395 | { 396 | Name: `History Entity Manager`, 397 | DiscoveryName: garnet_nomenclature.garnet_broker_historyentitymanager 398 | }, 399 | { 400 | Name: `History Query Manager`, 401 | DiscoveryName: garnet_nomenclature.garnet_broker_historyquerymanager 402 | }, 403 | 404 | { 405 | Name: `Registry Manager`, 406 | DiscoveryName: garnet_nomenclature.garnet_broker_registrymanager 407 | }, 408 | { 409 | Name: `Registry Subscription Manager`, 410 | DiscoveryName: garnet_nomenclature.garnet_broker_registrysubscriptionmanager 411 | }] 412 | 413 | } else { 414 | service_widget = [{ 415 | Name: `All In One`, 416 | DiscoveryName: garnet_nomenclature.garnet_broker_allinone 417 | }] 418 | } 419 | 420 | 421 | 422 | service_widget.forEach( 423 | (service:any) => { 424 | set_service_widgets(garnet_dashboard, 425 | service.Name, 426 | garnet_nomenclature.garnet_broker_cluster, 427 | `${service.DiscoveryName}-service`, 428 | service.DiscoveryName) 429 | 430 | 431 | } 432 | ) 433 | 434 | 435 | let service_sqs_widget: any = [] 436 | Object.entries(scorpiobroker_sqs_object).forEach(([key, value]) => { 437 | let sqs_service_widget = set_sqs_widgets(`${key} SQS`, value) 438 | garnet_dashboard.addWidgets(sqs_service_widget) 439 | }) 440 | 441 | 442 | 443 | 444 | 445 | 446 | 447 | 448 | 449 | 450 | 451 | 452 | } 453 | } 454 | 455 | --------------------------------------------------------------------------------