├── .gitignore ├── .vscode └── settings.json ├── README.md ├── images └── demo.gif ├── package-lock.json ├── src ├── .eslintignore ├── .eslintrc.js ├── .prettierrc.js ├── DeleteOldSnippets.ts ├── GetSnippet.ts ├── ListSnippets.ts ├── jest.config.ts ├── package-lock.json ├── package.json └── tsconfig.json ├── state-machine └── statemachine.yaml └── template.yaml /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Created by https://www.toptal.com/developers/gitignore/api/osx,node,linux,windows,sam 3 | # Edit at https://www.toptal.com/developers/gitignore?templates=osx,node,linux,windows,sam 4 | 5 | ### Linux ### 6 | *~ 7 | 8 | # temporary files which can be created if a process still has a handle open of a deleted file 9 | .fuse_hidden* 10 | 11 | # KDE directory preferences 12 | .directory 13 | 14 | # Linux trash folder which might appear on any partition or disk 15 | .Trash-* 16 | 17 | # .nfs files are created when an open file is removed but is still being accessed 18 | .nfs* 19 | 20 | ### Node ### 21 | # Logs 22 | logs 23 | *.log 24 | npm-debug.log* 25 | yarn-debug.log* 26 | yarn-error.log* 27 | lerna-debug.log* 28 | 29 | # Diagnostic reports (https://nodejs.org/api/report.html) 30 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 31 | 32 | # Runtime data 33 | pids 34 | *.pid 35 | *.seed 36 | *.pid.lock 37 | 38 | # Directory for instrumented libs generated by jscoverage/JSCover 39 | lib-cov 40 | 41 | # Coverage directory used by tools like istanbul 42 | coverage 43 | *.lcov 44 | 45 | # nyc test coverage 46 | .nyc_output 47 | 48 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 49 | .grunt 50 | 51 | # Bower dependency directory (https://bower.io/) 52 | bower_components 53 | 54 | # node-waf configuration 55 | .lock-wscript 56 | 57 | # Compiled binary addons (https://nodejs.org/api/addons.html) 58 | build/Release 59 | 60 | # Dependency directories 61 | node_modules/ 62 | jspm_packages/ 63 | 64 | # TypeScript v1 declaration files 65 | typings/ 66 | 67 | # TypeScript cache 68 | *.tsbuildinfo 69 | 70 | # Optional npm cache directory 71 | .npm 72 | 73 | # Optional eslint cache 74 | .eslintcache 75 | 76 | # Optional stylelint cache 77 | .stylelintcache 78 | 79 | # Microbundle cache 80 | .rpt2_cache/ 81 | .rts2_cache_cjs/ 82 | .rts2_cache_es/ 83 | .rts2_cache_umd/ 84 | 85 | # Optional REPL history 86 | .node_repl_history 87 | 88 | # Output of 'npm pack' 89 | *.tgz 90 | 91 | # Yarn Integrity file 92 | .yarn-integrity 93 | 94 | # dotenv environment variables file 95 | .env 96 | .env.test 97 | .env*.local 98 | 99 | # parcel-bundler cache (https://parceljs.org/) 100 | .cache 101 | .parcel-cache 102 | 103 | # Next.js build output 104 | .next 105 | 106 | # Nuxt.js build / generate output 107 | .nuxt 108 | dist 109 | 110 | # Storybook build outputs 111 | .out 112 | .storybook-out 113 | storybook-static 114 | 115 | # rollup.js default build output 116 | dist/ 117 | 118 | # Gatsby files 119 | .cache/ 120 | # Comment in the public line in if your project uses Gatsby and not Next.js 121 | # https://nextjs.org/blog/next-9-1#public-directory-support 122 | # public 123 | 124 | # vuepress build output 125 | .vuepress/dist 126 | 127 | # Serverless directories 128 | .serverless/ 129 | 130 | # FuseBox cache 131 | .fusebox/ 132 | 133 | # DynamoDB Local files 134 | .dynamodb/ 135 | 136 | # TernJS port file 137 | .tern-port 138 | 139 | # Stores VSCode versions used for testing VSCode extensions 140 | .vscode-test 141 | 142 | # Temporary folders 143 | tmp/ 144 | temp/ 145 | 146 | ### OSX ### 147 | # General 148 | .DS_Store 149 | .AppleDouble 150 | .LSOverride 151 | 152 | # Icon must end with two \r 153 | Icon 154 | 155 | 156 | # Thumbnails 157 | ._* 158 | 159 | # Files that might appear in the root of a volume 160 | .DocumentRevisions-V100 161 | .fseventsd 162 | .Spotlight-V100 163 | .TemporaryItems 164 | .Trashes 165 | .VolumeIcon.icns 166 | .com.apple.timemachine.donotpresent 167 | 168 | # Directories potentially created on remote AFP share 169 | .AppleDB 170 | .AppleDesktop 171 | Network Trash Folder 172 | Temporary Items 173 | .apdisk 174 | 175 | ### SAM ### 176 | # Ignore build directories for the AWS Serverless Application Model (SAM) 177 | # Info: https://aws.amazon.com/serverless/sam/ 178 | # Docs: https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-reference.html 179 | 180 | **/.aws-sam 181 | 182 | ### Windows ### 183 | # Windows thumbnail cache files 184 | Thumbs.db 185 | Thumbs.db:encryptable 186 | ehthumbs.db 187 | ehthumbs_vista.db 188 | 189 | # Dump file 190 | *.stackdump 191 | 192 | # Folder config file 193 | [Dd]esktop.ini 194 | 195 | # Recycle Bin used on file shares 196 | $RECYCLE.BIN/ 197 | 198 | # Windows Installer files 199 | *.cab 200 | *.msi 201 | *.msix 202 | *.msm 203 | *.msp 204 | 205 | # Windows shortcuts 206 | *.lnk 207 | 208 | # End of https://www.toptal.com/developers/gitignore/api/osx,node,linux,windows,sam 209 | 210 | samconfig.toml -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "cfn-resource-actions.stackName": "cw-logs-insights-snippets" 3 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cw-logs-insights-snippets 2 | 3 | [Serverlessland.com/snippets](https://serverlessland.com/snippets) hosts a growing number of community provided snippets. Many of these are useful [CloudWatch Logs Insights](https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/AnalyzingLogData.html) snippets. 4 | 5 | Until AWS integrates these in the console natively you can use this stack to get them there. 6 | 7 | 8 | ![Demo](images/demo.gif) 9 | 10 | 11 | It's designed to poll the [GitHub repo](https://github.com/aws-samples/serverless-snippets) every 12 hours. This can be changed [here](https://github.com/ljacobsson/cw-logs-insights-snippets/blob/main/template.yaml#:~:text=Schedule%3A%20rate(12%20hours)). 12 | 13 | Note that if you have access to the repository settings you are better off using [webhooks directly to EventBridge](https://aws.amazon.com/about-aws/whats-new/2022/08/amazon-eventbridge-supports-receiving-events-github-stripe-twilio-using-webhooks/) instead of polling. 14 | 15 | If a snippet is removed from the repo it will be removed from the AWS console after 48 hours via DynamoDB TTL expiry. 16 | 17 | ## Installation 18 | 19 | * Clone this repo and `cd` to project root 20 | * Get a recent version of [SAM-cli](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/serverless-sam-cli-install.html) 21 | * `sam build --parallel --beta-features && sam deploy --guided` 22 | 23 | A StepFunctions state machine will now run every 12 hours. You may want to kick it off manually the first time. 24 | -------------------------------------------------------------------------------- /images/demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ljacobsson/cw-logs-insights-snippets/7fac98fc9bc22343103454f44d37959781336e99/images/demo.gif -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cw-logs-insights-snippets", 3 | "lockfileVersion": 2, 4 | "requires": true, 5 | "packages": {} 6 | } 7 | -------------------------------------------------------------------------------- /src/.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .aws-sam -------------------------------------------------------------------------------- /src/.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | parser: "@typescript-eslint/parser", 3 | parserOptions: { 4 | ecmaVersion: 2020, // Allows for the parsing of modern ECMAScript features 5 | sourceType: "module" 6 | }, 7 | extends: [ 8 | "plugin:@typescript-eslint/recommended", // recommended rules from the @typescript-eslint/eslint-plugin 9 | "plugin:prettier/recommended" // Enables eslint-plugin-prettier and eslint-config-prettier. This will display prettier errors as ESLint errors. Make sure this is always the last configuration in the extends array. 10 | ], 11 | rules: { 12 | // Place to specify ESLint rules. Can be used to overwrite rules specified from the extended configs 13 | // e.g. "@typescript-eslint/explicit-function-return-type": "off", 14 | } 15 | }; -------------------------------------------------------------------------------- /src/.prettierrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | semi: true, 3 | trailingComma: "all", 4 | singleQuote: true, 5 | printWidth: 120, 6 | tabWidth: 4 7 | }; -------------------------------------------------------------------------------- /src/DeleteOldSnippets.ts: -------------------------------------------------------------------------------- 1 | import CloudWatchLogs from 'aws-sdk/clients/cloudwatchlogs'; 2 | import { DynamoDBStreamEvent } from 'aws-lambda'; 3 | const cloudwatchlogs = new CloudWatchLogs(); 4 | export const handler = async (event: DynamoDBStreamEvent): Promise => { 5 | for (const record of event.Records) { 6 | if (record.eventName === 'REMOVE') { 7 | { 8 | await cloudwatchlogs 9 | .deleteQueryDefinition({ 10 | queryDefinitionId: record.dynamodb?.OldImage?.QueryId.S || '', 11 | }) 12 | .promise(); 13 | } 14 | } 15 | } 16 | }; 17 | -------------------------------------------------------------------------------- /src/GetSnippet.ts: -------------------------------------------------------------------------------- 1 | /* eslint-disable @typescript-eslint/no-explicit-any */ 2 | import { Octokit } from '@octokit/rest'; 3 | const octokit = new Octokit(); 4 | 5 | export const handler = async (event: any): Promise => { 6 | console.log(event); 7 | const snippetResponse = await octokit.repos.getContent({ 8 | owner: 'aws-samples', 9 | repo: 'serverless-snippets', 10 | path: `${event.name}/snippet.txt`, 11 | }); 12 | const metadataResponse = await octokit.repos.getContent({ 13 | owner: 'aws-samples', 14 | repo: 'serverless-snippets', 15 | path: `${event.name}/snippet-data.json`, 16 | }); 17 | const snippet = Buffer.from((snippetResponse.data as any).content, 'base64').toString('utf8'); 18 | let metadata; 19 | try { 20 | const str = Buffer.from((metadataResponse.data as any).content, 'base64') 21 | .toString('utf8') 22 | .replace(/[^:]\/\/.*/g, ''); 23 | console.log(str); 24 | metadata = JSON.parse(str); 25 | } catch (e) { 26 | console.log(e); 27 | metadata = { 28 | title: event.name.replace(/-/g, ' '), 29 | }; 30 | } 31 | 32 | return { snippet, metadata }; 33 | }; 34 | -------------------------------------------------------------------------------- /src/ListSnippets.ts: -------------------------------------------------------------------------------- 1 | import { Octokit } from '@octokit/rest'; 2 | const octokit = new Octokit(); 3 | 4 | export const handler = async (): Promise => { 5 | const response = await octokit.repos.getContent({ 6 | owner: 'aws-samples', 7 | repo: 'serverless-snippets', 8 | path: '', 9 | }); 10 | 11 | const TWO_DAYS = 60 * 60 * 48; 12 | const secondsSinceEpoch = Math.round(Date.now() / 1000); 13 | const ttl = secondsSinceEpoch + 24 * TWO_DAYS; 14 | return { 15 | Items: (response.data as any[]) 16 | .filter((p) => p.name.startsWith('cloudwatch-')) 17 | .map((p) => { 18 | return { name: p.name, ttl }; 19 | }), 20 | }; 21 | }; 22 | -------------------------------------------------------------------------------- /src/jest.config.ts: -------------------------------------------------------------------------------- 1 | /* 2 | * For a detailed explanation regarding each configuration property and type check, visit: 3 | * https://jestjs.io/docs/configuration 4 | */ 5 | 6 | export default { 7 | transform: { 8 | '^.+\\.ts?$': 'esbuild-jest', 9 | }, 10 | clearMocks: true, 11 | collectCoverage: true, 12 | coverageDirectory: 'coverage', 13 | coverageProvider: 'v8', 14 | testMatch: ['**/tests/unit/*.test.ts'], 15 | }; 16 | -------------------------------------------------------------------------------- /src/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "hello_world", 3 | "version": "1.0.0", 4 | "description": "hello world sample for NodeJS", 5 | "main": "app.js", 6 | "repository": "https://github.com/awslabs/aws-sam-cli/tree/develop/samcli/local/init/templates/cookiecutter-aws-sam-hello-nodejs", 7 | "author": "SAM CLI", 8 | "license": "MIT", 9 | "dependencies": { 10 | "@octokit/rest": "^19.0.3", 11 | "aws-sdk": "^2.1194.0" 12 | }, 13 | "scripts": { 14 | "unit": "jest", 15 | "lint": "eslint '*.ts' --quiet --fix", 16 | "compile": "tsc", 17 | "test": "npm run compile && npm run unit" 18 | }, 19 | "devDependencies": { 20 | "@types/aws-lambda": "^8.10.92", 21 | "@types/jest": "^27.4.0", 22 | "@types/node": "^17.0.13", 23 | "@typescript-eslint/eslint-plugin": "^5.10.2", 24 | "@typescript-eslint/parser": "^5.10.2", 25 | "esbuild": "^0.14.14", 26 | "esbuild-jest": "^0.5.0", 27 | "eslint": "^8.8.0", 28 | "eslint-config-prettier": "^8.3.0", 29 | "eslint-plugin-prettier": "^4.0.0", 30 | "jest": "^27.5.0", 31 | "prettier": "^2.5.1", 32 | "ts-node": "^10.4.0", 33 | "typescript": "^4.5.5" 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es2020", 4 | "strict": true, 5 | "preserveConstEnums": true, 6 | "noEmit": true, 7 | "sourceMap": false, 8 | "module":"es2015", 9 | "moduleResolution":"node", 10 | "esModuleInterop": true, 11 | "skipLibCheck": true, 12 | "forceConsistentCasingInFileNames": true, 13 | }, 14 | "exclude": ["node_modules", "**/*.test.ts"], 15 | 16 | } -------------------------------------------------------------------------------- /state-machine/statemachine.yaml: -------------------------------------------------------------------------------- 1 | StartAt: List snippets 2 | States: 3 | List snippets: 4 | Type: Task 5 | Resource: arn:aws:states:::aws-sdk:lambda:invoke 6 | ResultSelector: 7 | Payload.$: States.StringToJson($.Payload) 8 | Parameters: 9 | FunctionName: ${ListSnippets} 10 | Next: For each snippet 11 | For each snippet: 12 | Type: Map 13 | ItemsPath: $.Payload.Items 14 | MaxConcurrency: 5 15 | Iterator: 16 | StartAt: Parallel 17 | States: 18 | Parallel: 19 | Type: Parallel 20 | Next: Query definition id exists? 21 | ResultPath: $.ParallelResult 22 | Branches: 23 | - StartAt: Get snippet 24 | States: 25 | Get snippet: 26 | Type: Task 27 | Resource: arn:aws:states:::aws-sdk:lambda:invoke 28 | ResultSelector: 29 | Payload.$: States.StringToJson($.Payload) 30 | Parameters: 31 | FunctionName: ${GetSnippet} 32 | Payload.$: $ 33 | End: true 34 | - StartAt: Get query definition id 35 | States: 36 | Get query definition id: 37 | Type: Task 38 | Resource: arn:aws:states:::aws-sdk:dynamodb:getItem 39 | Parameters: 40 | Key: 41 | Id: 42 | S.$: $.name 43 | TableName: ${Table} 44 | End: true 45 | Save snippet: 46 | Type: Task 47 | Resource: arn:aws:states:::aws-sdk:cloudwatchlogs:putQueryDefinition 48 | Parameters: 49 | Name.$: States.Format('Community snippets/{}', $.ParallelResult[0].Payload.metadata.title) 50 | QueryString.$: $.ParallelResult[0].Payload.snippet 51 | Next: Save query definition id 52 | ResultPath: $.PutQueryResult 53 | Retry: 54 | - ErrorEquals: [States.ALL] 55 | BackoffRate: 2 56 | IntervalSeconds: 1 57 | MaxAttempts: 2 58 | Catch: 59 | - ErrorEquals: [States.ALL] 60 | Next: Catch failure 61 | Update snippet: 62 | Type: Task 63 | Resource: arn:aws:states:::aws-sdk:cloudwatchlogs:putQueryDefinition 64 | Parameters: 65 | Name.$: States.Format('Community snippets/{}', $.ParallelResult[0].Payload.metadata.title) 66 | QueryString.$: $.ParallelResult[0].Payload.snippet 67 | QueryDefinitionId.$: $.ParallelResult[1].Item.QueryId.S 68 | Next: Save query definition id 69 | ResultPath: $.PutQueryResult 70 | Retry: 71 | - ErrorEquals: [States.ALL] 72 | BackoffRate: 2 73 | IntervalSeconds: 1 74 | MaxAttempts: 2 75 | Catch: 76 | - ErrorEquals: [States.ALL] 77 | Next: Catch failure 78 | Catch failure: 79 | Type: Pass 80 | End: true 81 | Save query definition id: 82 | Type: Task 83 | Resource: arn:aws:states:::aws-sdk:dynamodb:putItem 84 | ResultPath: null 85 | Parameters: 86 | Item: 87 | Id: 88 | S.$: $.name 89 | QueryId: 90 | S.$: $.PutQueryResult.QueryDefinitionId 91 | TTL: 92 | N.$: States.Format('{}',$.ttl) 93 | TableName: ${Table} 94 | End: true 95 | Query definition id exists?: 96 | Type: Choice 97 | Choices: 98 | - Variable: $.ParallelResult[1].Item 99 | IsPresent: true 100 | Next: Update snippet 101 | Default: Save snippet 102 | End: true 103 | -------------------------------------------------------------------------------- /template.yaml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: 'AWS::Serverless-2016-10-31' 3 | Globals: 4 | Function: 5 | Timeout: 10 6 | MemorySize: 128 7 | Runtime: nodejs16.x 8 | 9 | Resources: 10 | ListSnippets: 11 | Type: 'AWS::Serverless::Function' 12 | Properties: 13 | CodeUri: src/ 14 | Handler: ListSnippets.handler 15 | Metadata: 16 | BuildMethod: esbuild 17 | GetSnippet: 18 | Type: 'AWS::Serverless::Function' 19 | Properties: 20 | CodeUri: src/ 21 | Handler: GetSnippet.handler 22 | Metadata: 23 | BuildMethod: esbuild 24 | DeleteOldSnippets: 25 | Type: 'AWS::Serverless::Function' 26 | Properties: 27 | CodeUri: src/ 28 | Handler: DeleteOldSnippets.handler 29 | Events: 30 | MyDynamoDBtable: 31 | Type: DynamoDB 32 | Properties: 33 | Stream: !GetAtt Table.StreamArn 34 | StartingPosition: TRIM_HORIZON 35 | BatchSize: 10 36 | Policies: 37 | - Version: 2012-10-17 38 | Statement: 39 | - Sid: Statement1 40 | Effect: Allow 41 | Action: 42 | - 'logs:DeleteQueryDefinition' 43 | Resource: 44 | - '*' 45 | Metadata: 46 | BuildMethod: esbuild 47 | Table: 48 | Type: 'AWS::DynamoDB::Table' 49 | Properties: 50 | AttributeDefinitions: 51 | - AttributeName: Id 52 | AttributeType: S 53 | KeySchema: 54 | - AttributeName: Id 55 | KeyType: HASH 56 | BillingMode: PAY_PER_REQUEST 57 | StreamSpecification: 58 | StreamViewType: OLD_IMAGE 59 | TimeToLiveSpecification: 60 | AttributeName: TTL 61 | Enabled: true 62 | StateMachine: 63 | Type: 'AWS::Serverless::StateMachine' 64 | Properties: 65 | Name: !Sub '${AWS::StackName}-StateMachine' 66 | DefinitionUri: state-machine/statemachine.yaml 67 | DefinitionSubstitutions: 68 | Table: !Ref Table 69 | ListSnippets: !GetAtt ListSnippets.Arn 70 | GetSnippet: !GetAtt GetSnippet.Arn 71 | Policies: 72 | - LambdaInvokePolicy: 73 | FunctionName: !Ref ListSnippets 74 | - DynamoDBCrudPolicy: 75 | TableName: !Ref Table 76 | - LambdaInvokePolicy: 77 | FunctionName: !Ref GetSnippet 78 | - Version: 2012-10-17 79 | Statement: 80 | - Sid: Statement1 81 | Effect: Allow 82 | Action: 83 | - 'logs:PutQueryDefinition' 84 | Resource: 85 | - '*' 86 | Events: 87 | Schedule: 88 | Type: Schedule 89 | Properties: 90 | Schedule: rate(12 hours) 91 | Enabled: true 92 | Outputs: {} 93 | --------------------------------------------------------------------------------