├── .circleci └── config.yml ├── .editorconfig ├── .eslintignore ├── .eslintrc.json ├── .gitignore ├── LICENSE ├── README.md ├── commitlint.config.js ├── greenkeeper.json ├── jest.config.js ├── lerna.json ├── package-lock.json ├── package.json └── packages ├── cloudwatch-logs-auto-set-retention ├── .gitignore ├── LICENSE ├── README.md ├── functions │ ├── lib │ │ ├── aws.js │ │ └── cloudwatch-logs.js │ ├── set-retention.js │ └── set-retention.test.js ├── package-lock.json ├── package.json └── template.yml └── cloudwatch-logs-auto-subscribe ├── .gitignore ├── LICENSE ├── README.md ├── functions ├── lib │ ├── aws.js │ ├── cloudwatch-logs.js │ └── lambda.js ├── subscribe.js └── subscribe.test.js ├── package-lock.json ├── package.json └── template.yml /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | build: 5 | docker: 6 | - image: circleci/node:10.16.0 7 | working_directory: ~/repo 8 | steps: 9 | - checkout 10 | - run: npm ci 11 | - run: npm run install 12 | - run: npm run test:lint 13 | - run: npm run test 14 | - run: npm run codecov 15 | 16 | deploy: 17 | docker: 18 | - image: nikolaik/python-nodejs:python3.7-nodejs10 19 | working_directory: ~/repo 20 | steps: 21 | - checkout 22 | - run: pip install awscli 23 | - run: pip install aws-sam-cli 24 | - run: npm ci 25 | - run: npm run install 26 | - run: npm run package 27 | - run: npm run publish 28 | 29 | workflows: 30 | version: 2 31 | build: 32 | jobs: 33 | - build: 34 | filters: 35 | branches: 36 | ignore: deploy 37 | 38 | build_and_deploy: 39 | jobs: 40 | - build: 41 | filters: 42 | branches: 43 | only: deploy 44 | - deploy: 45 | requires: 46 | - build 47 | filters: 48 | branches: 49 | only: deploy 50 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | 6 | # Unix-style newlines with a newline ending every file 7 | [*] 8 | end_of_line = lf 9 | insert_final_newline = true 10 | charset = utf-8 11 | indent_style = space 12 | indent_size = 2 -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | coverage 3 | docs -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": false, 4 | "es6": true, 5 | "mocha": true, 6 | "node": true, 7 | "jest": true 8 | }, 9 | "parserOptions": { 10 | "ecmaVersion": 2017 11 | }, 12 | "rules": { 13 | "indent": ["error", "tab"], 14 | "linebreak-style": ["error", "unix"], 15 | "quotes": ["error", "double", { "avoidEscape": true }], 16 | "semi": ["error", "always"], 17 | "no-console": 0 18 | }, 19 | "extends": ["eslint:recommended", "prettier"], 20 | "globals": { 21 | "console": true, 22 | "require": true, 23 | "module": true, 24 | "process": true, 25 | "setTimeout": true 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # package directories 61 | node_modules 62 | jspm_packages 63 | 64 | # Serverless directories 65 | .serverless -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2019] [Lumigo] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # SAR-Logging 2 | 3 | [![Greenkeeper badge](https://badges.greenkeeper.io/lumigo/SAR-Logging.svg)](https://greenkeeper.io/) 4 | [![CircleCI](https://circleci.com/gh/lumigo-io/SAR-Logging.svg?style=svg)](https://circleci.com/gh/lumigo-io/SAR-Logging) 5 | 6 | Serverless applications for managing CloudWatch Logs log groups for your Lambda functions: 7 | 8 | * [cloudwatch-logs-auto-subscribe](packages/cloudwatch-logs-auto-subscribe): SAR app to manage the subscription filter for both new and existing log groups. 9 | 10 | * [cloudwatch-logs-auto-set-retention](packages/cloudwatch-logs-auto-set-retention): SAR app to manage the retention policy for both new and existing log groups. 11 | 12 | Both can be deployed through the AWS console, as well as with the Serverless or SAM frameworks, or with plain CloudFormation. 13 | -------------------------------------------------------------------------------- /commitlint.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { extends: ["@commitlint/config-conventional"] }; 2 | -------------------------------------------------------------------------------- /greenkeeper.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": { 3 | "default": { 4 | "packages": [ 5 | "package.json", 6 | "packages/cloudwatch-logs-auto-set-retention/package.json", 7 | "packages/cloudwatch-logs-auto-subscribe/package.json" 8 | ] 9 | } 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | collectCoverage: true, 3 | coverageReporters: [ 4 | "text", 5 | "html", 6 | "lcov" 7 | ], 8 | testEnvironment: "node" 9 | }; 10 | -------------------------------------------------------------------------------- /lerna.json: -------------------------------------------------------------------------------- 1 | { 2 | "lerna": "3.10.8", 3 | "packages": [ 4 | "packages/*" 5 | ], 6 | "version": "1.0.0", 7 | "command": { 8 | "version": { 9 | "allowBranch": "master" 10 | } 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sar-logging", 3 | "version": "1.0.0", 4 | "private": true, 5 | "description": "", 6 | "main": "index.js", 7 | "scripts": { 8 | "codecov": "codecov", 9 | "install": "lerna bootstrap", 10 | "test": "lerna run --concurrency 2 test -- -- --config=../../jest.config.js", 11 | "test:lint": "eslint .", 12 | "package": "lerna run package", 13 | "publish": "lerna run publish" 14 | }, 15 | "author": "Yan Cui", 16 | "license": "Apache 2", 17 | "devDependencies": { 18 | "@commitlint/cli": "^8.1.0", 19 | "@commitlint/config-conventional": "^8.0.0", 20 | "codecov": "^3.5.0", 21 | "coveralls": "^3.0.2", 22 | "eslint": "5.16.0", 23 | "eslint-config-prettier": "^6.0.0", 24 | "eslint-config-standard": "^14.0.0", 25 | "eslint-plugin-import": "^2.16.0", 26 | "eslint-plugin-node": "^9.1.0", 27 | "eslint-plugin-promise": "^4.0.1", 28 | "eslint-plugin-standard": "^4.0.0", 29 | "husky": "^3.0.0", 30 | "jest": "^24.8.0", 31 | "lerna": "^3.15.0", 32 | "lint-staged": "^9.1.0" 33 | }, 34 | "greenkeeper": { 35 | "ignore": [ 36 | "eslint" 37 | ] 38 | }, 39 | "husky": { 40 | "hooks": { 41 | "pre-commit": "lint-staged", 42 | "commit-msg": "commitlint -E HUSKY_GIT_PARAMS" 43 | } 44 | }, 45 | "lint-staged": { 46 | "*.js": [ 47 | "eslint" 48 | ] 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # packaged.yml 9 | packaged.yml 10 | 11 | # Coverage directory used by tools like istanbul 12 | coverage 13 | 14 | # Optional npm cache directory 15 | .npm 16 | 17 | # Optional eslint cache 18 | .eslintcache 19 | 20 | # Optional REPL history 21 | .node_repl_history 22 | 23 | # vscode folders 24 | .vscode/ 25 | 26 | # dstore 27 | .DS_Store 28 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2019] [Lumigo] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/README.md: -------------------------------------------------------------------------------- 1 | # cloudwatch-logs-auto-set-retention 2 | 3 | [![Version](https://img.shields.io/badge/semver-1.7.0-blue)](template.yml) 4 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) 5 | 6 | Updates the retention policy for **new and existing** CloudWatch log groups to the specified number of days. 7 | 8 | ## Deploying to your account (via the console) 9 | 10 | Go to this [page](https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:374852340823:applications~auto-set-log-group-retention) and click the Deploy button. 11 | 12 | ## Deploying via SAM/Serverless framework/CloudFormation 13 | 14 | To deploy this via SAM, you need something like this in the CloudFormation template: 15 | 16 | ```yml 17 | AutoSetLogRetention: 18 | Type: AWS::Serverless::Application 19 | Properties: 20 | Location: 21 | ApplicationId: arn:aws:serverlessrepo:us-east-1:374852340823:applications/auto-set-log-group-retention 22 | SemanticVersion: 23 | Parameters: 24 | RetentionDays: 25 | ``` 26 | 27 | To do the same via `CloudFormation` or the `Serverless` framework, you need to first add the following `Transform`: 28 | 29 | ```yml 30 | Transform: AWS::Serverless-2016-10-31 31 | ``` 32 | 33 | For more details, read this [post](https://theburningmonk.com/2019/05/how-to-include-serverless-repository-apps-in-serverless-yml/). 34 | 35 | ## Parameters 36 | 37 | `RetentionDays`: The number of days to retain logs in CloudWatch Logs for. 38 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/functions/lib/aws.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | const https = require("https"); 3 | const sslAgent = new https.Agent({ 4 | keepAlive: true, 5 | maxSockets: 50, 6 | rejectUnauthorized: true 7 | }); 8 | sslAgent.setMaxListeners(0); 9 | AWS.config.update({ 10 | httpOptions: { 11 | agent: sslAgent 12 | } 13 | }); 14 | 15 | module.exports = AWS; 16 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/functions/lib/cloudwatch-logs.js: -------------------------------------------------------------------------------- 1 | const AWS = require("./aws"); 2 | const cloudWatchLogs = new AWS.CloudWatchLogs(); 3 | const log = require("@dazn/lambda-powertools-logger"); 4 | const retry = require("async-retry"); 5 | 6 | const bailIfErrorNotRetryable = (bail) => (error) => { 7 | if (!error.retryable) { 8 | bail(error); 9 | } else { 10 | throw error; 11 | } 12 | }; 13 | 14 | const getRetryConfig = (onRetry) => ( 15 | { 16 | retries: parseInt(process.env.RETRIES || "5"), 17 | minTimeout: parseFloat(process.env.RETRY_MIN_TIMEOUT || "5000"), 18 | maxTimeout: parseFloat(process.env.RETRY_MAX_TIMEOUT || "60000"), 19 | factor: 2, 20 | onRetry 21 | } 22 | ); 23 | 24 | const getLogGroups = async () => { 25 | const loop = async (nextToken, acc = []) => { 26 | const req = { 27 | nextToken: nextToken 28 | }; 29 | 30 | try { 31 | const resp = await retry( 32 | (bail) => cloudWatchLogs 33 | .describeLogGroups(req) 34 | .promise() 35 | .catch(bailIfErrorNotRetryable(bail)), 36 | getRetryConfig((err) => log.warn("retrying describeLogGroups after error...", { req }, err)) 37 | ); 38 | 39 | const logGroups = resp.logGroups.map(x => ({ 40 | logGroupName: x.logGroupName, 41 | retentionInDays: x.retentionInDays 42 | })); 43 | const newAcc = acc.concat(logGroups); 44 | 45 | if (resp.nextToken) { 46 | return await loop(resp.nextToken, newAcc); 47 | } else { 48 | return newAcc; 49 | } 50 | } catch (error) { 51 | log.error(`failed to fetch log groups, processing the fetched groups [${acc.length}] so far`, error); 52 | return acc; 53 | } 54 | }; 55 | 56 | return await loop(); 57 | }; 58 | 59 | const setExpiry = async (logGroupName) => { 60 | const retentionDays = parseInt(process.env.RETENTION_DAYS || "7"); 61 | const req = { 62 | logGroupName: logGroupName, 63 | retentionInDays: retentionDays 64 | }; 65 | 66 | await retry( 67 | (bail) => cloudWatchLogs 68 | .putRetentionPolicy(req) 69 | .promise() 70 | .catch(bailIfErrorNotRetryable(bail)), 71 | getRetryConfig((err) => log.warn("retrying putRetentionPolicy after error...", { logGroupName }, err)) 72 | ) 73 | .then(() => log.debug(`${logGroupName}: retention policy updated`, { logGroupName })) 74 | .catch(err => log.error(`${logGroupName}: failed to update retention policy, skipped...`, { logGroupName }, err)); 75 | }; 76 | 77 | module.exports = { 78 | getLogGroups, 79 | setExpiry 80 | }; 81 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/functions/set-retention.js: -------------------------------------------------------------------------------- 1 | const log = require("@dazn/lambda-powertools-logger"); 2 | const cloudWatchLogs = require("./lib/cloudwatch-logs"); 3 | 4 | module.exports.existingLogGroups = async () => { 5 | const retentionDays = parseInt(process.env.RETENTION_DAYS || "7"); 6 | const logGroups = await cloudWatchLogs.getLogGroups(); 7 | for (const { logGroupName, retentionInDays } of logGroups) { 8 | if (retentionInDays !== retentionDays) { 9 | log.info( 10 | `${logGroupName}: has different retention days [${retentionInDays}], updating...`, 11 | { logGroupName, retentionDays }); 12 | await cloudWatchLogs.setExpiry(logGroupName); 13 | } 14 | } 15 | }; 16 | 17 | module.exports.newLogGroups = async (event) => { 18 | log.debug("processing new log group...", { event }); 19 | 20 | const logGroupName = event.detail.requestParameters.logGroupName; 21 | await cloudWatchLogs.setExpiry(logGroupName); 22 | }; 23 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/functions/set-retention.test.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | 3 | const mockPutRetentionPolicy = jest.fn(); 4 | AWS.CloudWatchLogs.prototype.putRetentionPolicy = mockPutRetentionPolicy; 5 | const mockDescribeLogGroups = jest.fn(); 6 | AWS.CloudWatchLogs.prototype.describeLogGroups = mockDescribeLogGroups; 7 | 8 | process.env.RETENTION_DAYS = 7; 9 | 10 | console.log = jest.fn(); 11 | 12 | beforeEach(() => { 13 | process.env.RETRY_MIN_TIMEOUT = "100"; 14 | process.env.RETRY_MAX_TIMEOUT = "100"; 15 | 16 | mockPutRetentionPolicy.mockReturnValue({ 17 | promise: () => Promise.resolve() 18 | }); 19 | }); 20 | 21 | afterEach(() => { 22 | mockPutRetentionPolicy.mockReset(); 23 | mockDescribeLogGroups.mockReset(); 24 | }); 25 | 26 | describe("new log group", () => { 27 | const handler = require("./set-retention").newLogGroups; 28 | 29 | test("retention policy is updated to 7 days", async () => { 30 | const event = { 31 | detail: { 32 | requestParameters: { 33 | logGroupName: "/aws/lambda/my-function" 34 | } 35 | } 36 | }; 37 | await handler(event); 38 | 39 | expect(mockPutRetentionPolicy).toBeCalledWith({ 40 | logGroupName: "/aws/lambda/my-function", 41 | retentionInDays: 7 42 | }); 43 | }); 44 | }); 45 | 46 | describe("existing log groups", () => { 47 | const handler = require("./set-retention").existingLogGroups; 48 | 49 | test("retention policy for all log groups are updated to 7 days", async () => { 50 | givenDescribeLogGroupsReturns([ 51 | { 52 | logGroupName: "group-1", 53 | retentionInDays: 5 54 | }, { 55 | logGroupName: "group-2", 56 | retentionInDays: undefined 57 | }], 58 | true); 59 | givenDescribeLogGroupsReturns([ 60 | { 61 | logGroupName: "group-3", 62 | retentionInDays: null 63 | }, { 64 | logGroupName: "group-4", 65 | retentionInDays: 7 // this one is ignored 66 | } 67 | ]); 68 | 69 | await handler(); 70 | 71 | expect(mockPutRetentionPolicy).toHaveBeenCalledTimes(3); 72 | expect(mockPutRetentionPolicy).toBeCalledWith({ 73 | logGroupName: "group-1", 74 | retentionInDays: 7 75 | }); 76 | expect(mockPutRetentionPolicy).toBeCalledWith({ 77 | logGroupName: "group-2", 78 | retentionInDays: 7 79 | }); 80 | expect(mockPutRetentionPolicy).toBeCalledWith({ 81 | logGroupName: "group-3", 82 | retentionInDays: 7 83 | }); 84 | }); 85 | 86 | describe("error handling", () => { 87 | beforeEach(() => { 88 | mockPutRetentionPolicy.mockReset(); 89 | }); 90 | 91 | test("it should retry retryable errors when listing log groups", async () => { 92 | givenDescribeLogGroupsFailsWith("ThrottlingException", "Rate exceeded"); 93 | givenDescribeLogGroupsReturns([]); 94 | 95 | await handler(); 96 | 97 | expect(mockDescribeLogGroups).toBeCalledTimes(2); 98 | }); 99 | 100 | test("it should not retry non-retryable errors when listing log groups", async () => { 101 | givenDescribeLogGroupsFailsWith("Foo", "Bar", false); 102 | 103 | await expect(handler()).rejects; 104 | 105 | expect(mockDescribeLogGroups).toBeCalledTimes(1); 106 | }); 107 | 108 | test("it should retry retryable errors when putting retention policy", async () => { 109 | givenDescribeLogGroupsReturns([{ 110 | logGroupName: "group-1", 111 | retentionInDays: null 112 | }]); 113 | 114 | givenPutRetentionPolicyFailsWith("ThrottlingException", "Rate exceeded"); 115 | givenPutRetentionPolicySucceeds(); 116 | 117 | await expect(handler()).resolves.toEqual(undefined); 118 | 119 | expect(mockPutRetentionPolicy).toBeCalledTimes(2); 120 | }); 121 | 122 | test("it should not retry non-retryable errors when putting retention policy", async () => { 123 | givenDescribeLogGroupsReturns([{ 124 | logGroupName: "group-1", 125 | retentionInDays: null 126 | }]); 127 | 128 | givenPutRetentionPolicyFailsWith("Foo", "Bar", false); 129 | 130 | await expect(handler()).resolves.toEqual(undefined); 131 | 132 | expect(mockPutRetentionPolicy).toBeCalledTimes(1); 133 | }); 134 | }); 135 | }); 136 | 137 | const givenDescribeLogGroupsReturns = (logGroups, hasMore = false) => { 138 | mockDescribeLogGroups.mockReturnValueOnce({ 139 | promise: () => Promise.resolve({ 140 | logGroups: logGroups, 141 | nextToken: hasMore ? "more" : undefined 142 | }) 143 | }); 144 | }; 145 | 146 | const givenDescribeLogGroupsFailsWith = (code, message, retryable = true) => { 147 | mockDescribeLogGroups.mockReturnValueOnce({ 148 | promise: () => Promise.reject(new AwsError(code, message, retryable)) 149 | }); 150 | }; 151 | 152 | const givenPutRetentionPolicySucceeds = () => { 153 | mockPutRetentionPolicy.mockReturnValueOnce({ 154 | promise: () => Promise.resolve() 155 | }); 156 | }; 157 | 158 | const givenPutRetentionPolicyFailsWith = (code, message, retryable = true) => { 159 | mockPutRetentionPolicy.mockReturnValueOnce({ 160 | promise: () => Promise.reject(new AwsError(code, message, retryable)) 161 | }); 162 | }; 163 | 164 | class AwsError extends Error { 165 | constructor (code, message, retryable) { 166 | super(message); 167 | 168 | this.code = code; 169 | this.retryable = retryable; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatch-logs-auto-set-retention", 3 | "private": true, 4 | "version": "1.0.0", 5 | "description": "", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "jest", 9 | "package": "sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket sar-logging-set-retention", 10 | "publish": "sam publish --template packaged.yml --region us-east-1" 11 | }, 12 | "author": "Yan Cui", 13 | "license": "MIT", 14 | "devDependencies": { 15 | "jest": "^24.8.0" 16 | }, 17 | "dependencies": { 18 | "@dazn/lambda-powertools-logger": "^1.9.0", 19 | "async-retry": "^1.2.3", 20 | "aws-sdk": "^2.814.0" 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-set-retention/template.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | 4 | Metadata: 5 | AWS::ServerlessRepo::Application: 6 | Name: auto-set-log-group-retention 7 | Description: Updates the retention policy for new and existing CloudWatch log groups to the specified number of days. 8 | Author: Lumigo 9 | SpdxLicenseId: Apache-2.0 10 | LicenseUrl: LICENSE 11 | ReadmeUrl: README.md 12 | Labels: ['cloudwatch', 'logs', 'ops', 'devops'] 13 | HomePageUrl: https://github.com/lumigo/SAR-Logging 14 | SemanticVersion: 1.7.0 15 | SourceCodeUrl: https://github.com/lumigo/SAR-Logging 16 | 17 | Globals: 18 | Function: 19 | Runtime: nodejs18.x 20 | MemorySize: 128 21 | Environment: 22 | Variables: 23 | RETENTION_DAYS: !Ref RetentionDays 24 | 25 | Resources: 26 | SetRetentionForNewLogGroups: 27 | Type: AWS::Serverless::Function 28 | Properties: 29 | Handler: functions/set-retention.newLogGroups 30 | Description: Updates the retention policy for a newly create CloudWatch log group to the specified number of days. 31 | Timeout: 6 32 | Policies: 33 | - Statement: 34 | Effect: Allow 35 | Action: logs:PutRetentionPolicy 36 | Resource: '*' 37 | Events: 38 | SubscribeEvent: 39 | Type: CloudWatchEvent 40 | Properties: 41 | Pattern: 42 | source: 43 | - aws.logs 44 | detail-type: 45 | - AWS API Call via CloudTrail 46 | detail: 47 | eventSource: 48 | - logs.amazonaws.com 49 | eventName: 50 | - CreateLogGroup 51 | 52 | SetRetentionForNewLogGroupsLogGroup: 53 | Type: AWS::Logs::LogGroup 54 | Properties: 55 | LogGroupName: !Sub /aws/lambda/${SetRetentionForNewLogGroups} 56 | 57 | SetRetentionForExistingLogGroups: 58 | Type: AWS::Serverless::Function 59 | Properties: 60 | Handler: functions/set-retention.existingLogGroups 61 | Description: Updates the retention policy for existing log groups to match the configured number of days. 62 | Timeout: 900 63 | Policies: 64 | - Statement: 65 | Effect: Allow 66 | Action: 67 | - logs:PutRetentionPolicy 68 | - logs:DescribeLogGroups 69 | Resource: '*' 70 | 71 | SetRetentionForExistingLogGroupsLogGroup: 72 | Type: AWS::Logs::LogGroup 73 | Properties: 74 | LogGroupName: !Sub /aws/lambda/${SetRetentionForExistingLogGroups} 75 | 76 | LambdaInvocationCustomResource: 77 | Type: AWS::Serverless::Application 78 | Properties: 79 | Location: 80 | ApplicationId: arn:aws:serverlessrepo:us-east-1:374852340823:applications/lambda-invocation-cfn-custom-resource 81 | SemanticVersion: 1.6.0 82 | 83 | # custom resource to invoke the SubscribeExistingLogGroups function during deployment 84 | InvokeSetRetentionForExistingLogGroups: 85 | Type: Custom::LambdaInvocation 86 | DependsOn: 87 | - SetRetentionForExistingLogGroups 88 | - LambdaInvocationCustomResource 89 | Properties: 90 | ServiceToken: !GetAtt LambdaInvocationCustomResource.Outputs.FunctionArn 91 | FunctionName: !Ref SetRetentionForExistingLogGroups 92 | InvocationType: Event # don't wait for the subscribe existing function to finish 93 | # this ties this custom resource to changes to the function's configuration 94 | # so that when CloudFormation would detect this custom resource as "changed" 95 | # and triggers "Update" event during deployment 96 | Configurations: 97 | RETENTION_DAYS: !Ref RetentionDays 98 | 99 | Parameters: 100 | RetentionDays: 101 | Type: Number 102 | Default: 7 103 | Description: The number of days to retain logs in CloudWatch Logs for. 104 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless 7 | 8 | # packaged.yml 9 | packaged.yml 10 | 11 | # Coverage directory used by tools like istanbul 12 | coverage 13 | 14 | # Optional npm cache directory 15 | .npm 16 | 17 | # Optional eslint cache 18 | .eslintcache 19 | 20 | # Optional REPL history 21 | .node_repl_history 22 | 23 | # vscode folders 24 | .vscode/ 25 | 26 | # dstore 27 | .DS_Store 28 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2019] [Lumigo] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/README.md: -------------------------------------------------------------------------------- 1 | # cloudwatch-logs-auto-subscribe 2 | 3 | [![Version](https://img.shields.io/badge/semver-1.15.0-blue)](template.yml) 4 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](LICENSE) 5 | 6 | Subscribes **new and existing** CloudWatch log groups to Lambda/Kinesis/Firehose by ARN. 7 | 8 | ## Deploying to your account (via the console) 9 | 10 | Go to this [page](https://serverlessrepo.aws.amazon.com/applications/arn:aws:serverlessrepo:us-east-1:374852340823:applications~auto-subscribe-log-group-to-arn) and click the Deploy button. 11 | 12 | ## Deploying via SAM/Serverless framework/CloudFormation 13 | 14 | To deploy this via SAM, you need something like this in the CloudFormation template: 15 | 16 | ```yml 17 | AutoSubscribeLogGroups: 18 | Type: AWS::Serverless::Application 19 | Properties: 20 | Location: 21 | ApplicationId: arn:aws:serverlessrepo:us-east-1:374852340823:applications/auto-subscribe-log-group-to-arn 22 | SemanticVersion: 23 | Parameters: 24 | DestinationArn: 25 | Prefix: 26 | Tags: 27 | TagsMode: 28 | ExcludePrefix: 29 | FilterName: 30 | FilterPattern: 31 | UnsubscribeOnDelete: 32 | ``` 33 | 34 | To do the same via `CloudFormation` or the `Serverless` framework, you need to first add the following `Transform`: 35 | 36 | ```yml 37 | Transform: AWS::Serverless-2016-10-31 38 | ``` 39 | 40 | For more details, read this [post](https://theburningmonk.com/2019/05/how-to-include-serverless-repository-apps-in-serverless-yml/). 41 | 42 | ## Parameters 43 | 44 | `DestinationArn`: The ARN of the Lambda function or Kinesis stream to subscribe a newly created CloudWatch log group to. 45 | 46 | `Prefix`: (Optional) if specified then only log groups with the prefix will be subscribed. E.g. `/aws/lambda/` will subscribe only Lambda function logs. 47 | 48 | `Tags`: (Optional) specify a common separated list of tags, e.g. `tag1=value1,tag2,tag3` Which can test for the existence of a tag (e.g. `tag2`), and can also test the value of the tag too (e.g. `tag1=value1`). Whether the tests are treated with `AND` or `OR` semantics depends on the `TagMode` parameter. 49 | 50 | `TagsMode`: (Optional) controls how to combine the different tag tests, whether to join them using AND or OR semantic. Allowed values are `AND` or `OR`. 51 | 52 | `ExcludePrefix`: (Optional) if specified then log groups that match the prefix will not be subscribed. E.g. `/aws/lambda/my-function-` will exclude Lambda function logs for functions that start with `my-function-`. 53 | 54 | `FilterName`: (Optional) if specified, will override the filter name for the subscription. 55 | 56 | `FilterPattern`: (Optional) if specified, will override the filter pattern used to create the subscription. 57 | 58 | `UnsubscribeOnDelete`: (Optional) whether to remove the subscription filters that were added by this app. Defaults to "false", allowed values are "true" or "false". 59 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/functions/lib/aws.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | const https = require("https"); 3 | const sslAgent = new https.Agent({ 4 | keepAlive: true, 5 | maxSockets: 50, 6 | rejectUnauthorized: true 7 | }); 8 | sslAgent.setMaxListeners(0); 9 | AWS.config.update({ 10 | httpOptions: { 11 | agent: sslAgent 12 | } 13 | }); 14 | 15 | module.exports = AWS; 16 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/functions/lib/cloudwatch-logs.js: -------------------------------------------------------------------------------- 1 | const AWS = require("./aws"); 2 | const cloudWatchLogs = new AWS.CloudWatchLogs(); 3 | const log = require("@dazn/lambda-powertools-logger"); 4 | const retry = require("async-retry"); 5 | 6 | const { DESTINATION_ARN, FILTER_NAME, FILTER_PATTERN, ROLE_ARN } = process.env; 7 | const isLambda = DESTINATION_ARN.startsWith("arn:aws:lambda"); 8 | const filterName = FILTER_NAME || "ship-logs"; 9 | const filterPattern = FILTER_PATTERN || ""; 10 | 11 | const getRetryConfig = (onRetry) => ( 12 | { 13 | retries: parseInt(process.env.RETRIES || "5"), 14 | minTimeout: parseFloat(process.env.RETRY_MIN_TIMEOUT || "5000"), 15 | maxTimeout: parseFloat(process.env.RETRY_MAX_TIMEOUT || "60000"), 16 | factor: 2, 17 | onRetry 18 | } 19 | ); 20 | 21 | const bailIfErrorNotRetryable = (bail) => (error) => { 22 | if (!error.retryable) { 23 | bail(error); 24 | } else { 25 | throw error; 26 | } 27 | }; 28 | 29 | const getTags = async (logGroupName) => { 30 | const resp = await retry( 31 | (bail) => cloudWatchLogs 32 | .listTagsLogGroup({ logGroupName }) 33 | .promise() 34 | .catch(bailIfErrorNotRetryable(bail)), 35 | getRetryConfig((err) => log.warn("retrying listTagsLogGroup after error...", { logGroupName }, err)) 36 | ); 37 | 38 | return resp.tags; 39 | }; 40 | 41 | const getSubscriptionFilter = async (logGroupName) => { 42 | const resp = await retry( 43 | (bail) => cloudWatchLogs 44 | .describeSubscriptionFilters({ logGroupName }) 45 | .promise() 46 | .catch(bailIfErrorNotRetryable(bail)), 47 | getRetryConfig((err) => log.warn("retrying describeSubscriptionFilter after error...", { logGroupName }, err)) 48 | ); 49 | 50 | if (resp.subscriptionFilters.length === 0) { 51 | return null; 52 | } else { 53 | return resp.subscriptionFilters[0]; 54 | } 55 | }; 56 | 57 | const putSubscriptionFilter = async (logGroupName) => { 58 | // when subscribing a stream to Kinesis/Firehose, you need to specify the roleArn 59 | const roleArn = !isLambda ? ROLE_ARN : undefined; 60 | 61 | const req = { 62 | destinationArn: DESTINATION_ARN, 63 | logGroupName: logGroupName, 64 | filterName: filterName, 65 | filterPattern: filterPattern, 66 | roleArn: roleArn 67 | }; 68 | 69 | log.debug("adding subscription filter...", { 70 | logGroupName, 71 | arn: DESTINATION_ARN, 72 | filterName, 73 | filterPattern 74 | }); 75 | 76 | await retry( 77 | (bail) => cloudWatchLogs 78 | .putSubscriptionFilter(req) 79 | .promise() 80 | .catch(bailIfErrorNotRetryable(bail)), 81 | getRetryConfig((err) => { 82 | log.warn("retrying putSubscriptionFilter after error...", { logGroupName }, err); 83 | }) 84 | ); 85 | 86 | log.info(`subscribed log group to [${DESTINATION_ARN}]`, { 87 | logGroupName, 88 | arn: DESTINATION_ARN 89 | }); 90 | }; 91 | 92 | const deleteSubscriptionFilter = async (logGroupName, filterName) => { 93 | const req = { 94 | logGroupName: logGroupName, 95 | filterName: filterName 96 | }; 97 | 98 | log.debug("deleting existing filter...", { logGroupName, filterName }); 99 | 100 | await retry( 101 | (bail) => cloudWatchLogs 102 | .deleteSubscriptionFilter(req) 103 | .promise() 104 | .catch(bailIfErrorNotRetryable(bail)), 105 | getRetryConfig((err) => log.warn("retrying deleteSubscriptionFilter after error...", { logGroupName }, err)) 106 | ); 107 | 108 | log.info(`deleted subscription filter [${filterName}]`, { 109 | logGroupName, 110 | filterName 111 | }); 112 | }; 113 | 114 | const getLogGroups = async () => { 115 | const loop = async (nextToken, acc = []) => { 116 | const req = { 117 | nextToken: nextToken 118 | }; 119 | 120 | if (process.env.PREFIX) { 121 | req.logGroupNamePrefix = process.env.PREFIX; 122 | } 123 | 124 | try { 125 | const resp = await retry( 126 | (bail) => cloudWatchLogs 127 | .describeLogGroups(req) 128 | .promise() 129 | .catch(bailIfErrorNotRetryable(bail)), 130 | getRetryConfig((err) => log.warn("retrying describeLogGroup after error...", { req }, err)) 131 | ); 132 | const logGroupNames = resp.logGroups.map(x => x.logGroupName); 133 | const newAcc = acc.concat(logGroupNames); 134 | 135 | if (resp.nextToken) { 136 | return await loop(resp.nextToken, newAcc); 137 | } else { 138 | return newAcc; 139 | } 140 | } catch (error) { 141 | log.error(`failed to fetch log groups, processing the fetched groups [${acc.length}] so far`, error); 142 | return acc; 143 | } 144 | }; 145 | 146 | return await loop(); 147 | }; 148 | 149 | module.exports = { 150 | getTags, 151 | getSubscriptionFilter, 152 | putSubscriptionFilter, 153 | deleteSubscriptionFilter, 154 | getLogGroups 155 | }; 156 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/functions/lib/lambda.js: -------------------------------------------------------------------------------- 1 | const AWS = require("./aws"); 2 | const lambda = new AWS.Lambda(); 3 | const uuid = require("uuid/v4"); 4 | 5 | const addLambdaPermission = async (functionArn) => { 6 | const req = { 7 | Action: "lambda:InvokeFunction", 8 | FunctionName: functionArn, 9 | Principal: "logs.amazonaws.com", 10 | StatementId: `invoke-${uuid().substring(0, 8)}` 11 | }; 12 | await lambda.addPermission(req).promise(); 13 | }; 14 | 15 | module.exports = { 16 | addLambdaPermission 17 | }; 18 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/functions/subscribe.js: -------------------------------------------------------------------------------- 1 | const lambda = require("./lib/lambda"); 2 | const cloudWatchLogs = require("./lib/cloudwatch-logs"); 3 | const log = require("@dazn/lambda-powertools-logger"); 4 | 5 | const { FILTER_NAME, DESTINATION_ARN } = process.env; 6 | 7 | module.exports.existingLogGroups = async () => { 8 | const logGroupNames = await cloudWatchLogs.getLogGroups(); 9 | for (const logGroupName of logGroupNames) { 10 | try { 11 | if (await filter(logGroupName)) { 12 | await subscribe(logGroupName); 13 | } 14 | } catch(error) { 15 | log.warn("cannot process existing log group, skipped...", { logGroupName }, error); 16 | } 17 | } 18 | }; 19 | 20 | module.exports.newLogGroups = async (event) => { 21 | log.debug("received event...", { event }); 22 | 23 | // eg. /aws/lambda/logging-demo-dev-api 24 | const logGroupName = event.detail.requestParameters.logGroupName; 25 | if (await filter(logGroupName)) { 26 | await subscribe(logGroupName); 27 | } 28 | }; 29 | 30 | module.exports.undo = async () => { 31 | const logGroupNames = await cloudWatchLogs.getLogGroups(); 32 | for (const logGroupName of logGroupNames) { 33 | try { 34 | if (await filter(logGroupName)) { 35 | await unsubscribe(logGroupName); 36 | } 37 | } catch(error) { 38 | log.warn("cannot unsubscribe existing log group, skipped...", { logGroupName }, error); 39 | } 40 | } 41 | }; 42 | 43 | const tagPredicates = tagsCsv => 44 | (tagsCsv || "") 45 | .split(",") 46 | .filter(x => x.length > 0) 47 | .map(tag => { 48 | const segments = tag.split("="); 49 | 50 | // e.g. tag1=value1 51 | if (segments.length === 2) { 52 | const [tagName, tagValue] = segments; 53 | return (tags) => tags[tagName] === tagValue; 54 | } else { // e.g tag2 55 | const [tagName] = segments; 56 | return (tags) => tags[tagName]; 57 | } 58 | }); 59 | 60 | const filter = async (logGroupName) => { 61 | log.debug("checking log group...", { logGroupName }); 62 | 63 | const { PREFIX, EXCLUDE_PREFIX } = process.env; 64 | 65 | if (EXCLUDE_PREFIX && logGroupName.startsWith(EXCLUDE_PREFIX)) { 66 | log.debug(`ignored [${logGroupName}] because it matches the exclude prefix`, { 67 | logGroupName, 68 | excludePrefix: EXCLUDE_PREFIX 69 | }); 70 | return false; 71 | } 72 | 73 | if (PREFIX && !logGroupName.startsWith(PREFIX)) { 74 | log.debug(`ignored [${logGroupName}] because it doesn't match the prefix`, { 75 | logGroupName, 76 | prefix: PREFIX 77 | }); 78 | return false; 79 | } 80 | 81 | const excludeTagPredicates = tagPredicates(process.env.EXCLUDE_TAGS); 82 | const includeTagPredicates = tagPredicates(process.env.TAGS); 83 | 84 | if (includeTagPredicates.length === 0 && excludeTagPredicates.length === 0) { 85 | return true; 86 | } 87 | 88 | const logGroupTags = await cloudWatchLogs.getTags(logGroupName); 89 | 90 | if (excludeTagPredicates.length > 0) { 91 | const isExcluded = 92 | process.env.EXCLUDE_TAGS_MODE === "AND" 93 | ? excludeTagPredicates.every(f => f(logGroupTags)) 94 | : excludeTagPredicates.some(f => f(logGroupTags)); 95 | 96 | if (isExcluded) { 97 | log.debug(`ignored [${logGroupName}] because of exclude tags`, { 98 | logGroupName, 99 | logGroupTags, 100 | excludeTags: process.env.EXCLUDE_TAGS, 101 | mode: process.env.EXCLUDE_TAGS_MODE 102 | }); 103 | 104 | return false; 105 | } 106 | } 107 | 108 | if (includeTagPredicates.length > 0) { 109 | const isIncluded = 110 | process.env.TAGS_MODE === "AND" 111 | ? includeTagPredicates.every(f => f(logGroupTags)) 112 | : includeTagPredicates.some(f => f(logGroupTags)); 113 | 114 | if (!isIncluded) { 115 | log.debug(`ignored [${logGroupName}] because of tags`, { 116 | logGroupName, 117 | logGroupTags, 118 | tags: process.env.TAGS, 119 | mode: process.env.TAGS_MODE 120 | }); 121 | 122 | return false; 123 | } 124 | } 125 | 126 | return true; 127 | }; 128 | 129 | const subscribe = async (logGroupName) => { 130 | try { 131 | await cloudWatchLogs.putSubscriptionFilter(logGroupName); 132 | } catch (err) { 133 | log.error("failed to subscribe log group", { logGroupName }, err); 134 | 135 | // when subscribing a log group to a Lambda function, CloudWatch Logs needs permission 136 | // to invoke the function 137 | if (err.code === "InvalidParameterException" && 138 | err.message === "Could not execute the lambda function. Make sure you have given CloudWatch Logs permission to execute your function.") { 139 | log.info(`adding lambda:InvokeFunction permission to CloudWatch Logs for [${DESTINATION_ARN}]`); 140 | await lambda.addLambdaPermission(DESTINATION_ARN); 141 | 142 | // retry! 143 | await cloudWatchLogs.putSubscriptionFilter(logGroupName); 144 | } else { 145 | throw err; 146 | } 147 | } 148 | }; 149 | 150 | const unsubscribe = async (logGroupName) => { 151 | try { 152 | await cloudWatchLogs.deleteSubscriptionFilter(logGroupName, FILTER_NAME); 153 | } catch (err) { 154 | log.error("failed to unsubscribe log group", { logGroupName }, err); 155 | } 156 | }; 157 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/functions/subscribe.test.js: -------------------------------------------------------------------------------- 1 | const AWS = require("aws-sdk"); 2 | 3 | const mockListTagsLogGroup = jest.fn(); 4 | AWS.CloudWatchLogs.prototype.listTagsLogGroup = mockListTagsLogGroup; 5 | const mockPutSubscriptionFilter = jest.fn(); 6 | AWS.CloudWatchLogs.prototype.putSubscriptionFilter = mockPutSubscriptionFilter; 7 | const mockDescribeLogGroups = jest.fn(); 8 | AWS.CloudWatchLogs.prototype.describeLogGroups = mockDescribeLogGroups; 9 | const mockDescribeSubscriptionFilters = jest.fn(); 10 | AWS.CloudWatchLogs.prototype.describeSubscriptionFilters = mockDescribeSubscriptionFilters; 11 | const mockDeleteSubscriptionFilter = jest.fn(); 12 | AWS.CloudWatchLogs.prototype.deleteSubscriptionFilter = mockDeleteSubscriptionFilter; 13 | const mockAddPermission = jest.fn(); 14 | AWS.Lambda.prototype.addPermission = mockAddPermission; 15 | 16 | const destinationArn = "arn:aws:lambda:us-east-1:123456789:function:boohoo"; 17 | 18 | console.log = jest.fn(); 19 | 20 | beforeEach(() => { 21 | process.env.RETRY_MIN_TIMEOUT = "100"; 22 | process.env.RETRY_MAX_TIMEOUT = "100"; 23 | process.env.FILTER_NAME = "ship-logs"; 24 | process.env.TAGS_MODE = "OR"; 25 | process.env.EXCLUDE_TAGS_MODE = "OR"; 26 | process.env.DESTINATION_ARN = destinationArn; 27 | 28 | mockPutSubscriptionFilter.mockReturnValue({ 29 | promise: () => Promise.resolve() 30 | }); 31 | 32 | mockDeleteSubscriptionFilter.mockReturnValue({ 33 | promise: () => Promise.resolve() 34 | }); 35 | 36 | mockAddPermission.mockReturnValue({ 37 | promise: () => Promise.resolve() 38 | }); 39 | }); 40 | 41 | afterEach(() => { 42 | mockPutSubscriptionFilter.mockReset(); 43 | mockDeleteSubscriptionFilter.mockReset(); 44 | mockDescribeLogGroups.mockReset(); 45 | mockDescribeSubscriptionFilters.mockReset(); 46 | mockListTagsLogGroup.mockReset(); 47 | 48 | delete process.env.PREFIX; 49 | delete process.env.EXCLUDE_PREFIX; 50 | delete process.env.TAGS; 51 | delete process.env.EXCLUDE_TAGS; 52 | }); 53 | 54 | const givenPrefixIsDefined = () => process.env.PREFIX = "/aws/lambda/"; 55 | const givenExcludePrefixIsDefined = () => process.env.EXCLUDE_PREFIX = "/aws/lambda/exclude"; 56 | const givenTagsIsDefined = (tags) => process.env.TAGS = tags; 57 | const givenExcludeTagsIsDefined = (tags) => process.env.EXCLUDE_TAGS = tags; 58 | 59 | describe("new log group", () => { 60 | const getEvent = (logGroupName = "/aws/lambda/test-me") => ({ 61 | detail: { 62 | requestParameters: { 63 | logGroupName 64 | } 65 | } 66 | }); 67 | 68 | describe("prefix", () => { 69 | test("log group is subscribed if it matches prefix", async () => { 70 | givenPrefixIsDefined(); 71 | const handler = require("./subscribe").newLogGroups; 72 | await handler(getEvent()); 73 | 74 | expect(mockPutSubscriptionFilter).toBeCalled(); 75 | expect(mockListTagsLogGroup).not.toBeCalled(); 76 | }); 77 | 78 | test("log group is not subscribed if it does not match prefix", async () => { 79 | givenPrefixIsDefined(); 80 | 81 | const handler = require("./subscribe").newLogGroups; 82 | await handler(getEvent("/api/gateway/test-me")); 83 | 84 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 85 | }); 86 | 87 | test("any log group would be subscribed when there are no prefix", async () => { 88 | const handler = require("./subscribe").newLogGroups; 89 | await handler(getEvent("/api/gateway/test-me")); 90 | 91 | expect(mockPutSubscriptionFilter).toBeCalled(); 92 | expect(mockListTagsLogGroup).not.toBeCalled(); 93 | }); 94 | }); 95 | 96 | describe("tags", () => { 97 | beforeEach(() => { 98 | mockListTagsLogGroup.mockReturnValueOnce({ 99 | promise: () => Promise.resolve({ 100 | tags: { 101 | tag1: "value1", 102 | tag2: "value2" 103 | } 104 | }) 105 | }); 106 | }); 107 | 108 | describe("when TAGS_MODE is OR", () => { 109 | beforeEach(() => { 110 | process.env.TAGS_MODE = "OR"; 111 | }); 112 | 113 | test("log group is subscribed if it contains at least one matching tag", async () => { 114 | givenTagsIsDefined("tag2,tag3"); 115 | const handler = require("./subscribe").newLogGroups; 116 | await handler(getEvent()); 117 | 118 | expect(mockPutSubscriptionFilter).toBeCalled(); 119 | expect(mockListTagsLogGroup).toBeCalled(); 120 | }); 121 | 122 | test("log group is subscribed if it contains at least one matching tag AND value", async () => { 123 | givenTagsIsDefined("tag2=value2,tag3"); 124 | const handler = require("./subscribe").newLogGroups; 125 | await handler(getEvent()); 126 | 127 | expect(mockPutSubscriptionFilter).toBeCalled(); 128 | expect(mockListTagsLogGroup).toBeCalled(); 129 | }); 130 | 131 | test("log group is subscribed if it matches all tags and values", async () => { 132 | givenTagsIsDefined("tag1=value1,tag2=value2"); 133 | const handler = require("./subscribe").newLogGroups; 134 | await handler(getEvent()); 135 | 136 | expect(mockPutSubscriptionFilter).toBeCalled(); 137 | expect(mockListTagsLogGroup).toBeCalled(); 138 | }); 139 | }); 140 | 141 | describe("when TAGS_MODE is AND", () => { 142 | beforeEach(() => { 143 | process.env.TAGS_MODE = "AND"; 144 | }); 145 | 146 | test("log group is not subscribed if it contains only one matching tag", async () => { 147 | givenTagsIsDefined("tag2,tag3"); 148 | const handler = require("./subscribe").newLogGroups; 149 | await handler(getEvent()); 150 | 151 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 152 | expect(mockListTagsLogGroup).toBeCalled(); 153 | }); 154 | 155 | test("log group is not subscribed if it contains only one matching tag AND value", async () => { 156 | givenTagsIsDefined("tag2=value2,tag3"); 157 | const handler = require("./subscribe").newLogGroups; 158 | await handler(getEvent()); 159 | 160 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 161 | expect(mockListTagsLogGroup).toBeCalled(); 162 | }); 163 | 164 | test("log group is subscribed if it matches all tags and values", async () => { 165 | givenTagsIsDefined("tag1=value1,tag2=value2"); 166 | const handler = require("./subscribe").newLogGroups; 167 | await handler(getEvent()); 168 | 169 | expect(mockPutSubscriptionFilter).toBeCalled(); 170 | expect(mockListTagsLogGroup).toBeCalled(); 171 | }); 172 | }); 173 | 174 | test("log group is not subscribed if it doesn't contain any matching tag", async () => { 175 | givenTagsIsDefined("tag3,tag4"); 176 | const handler = require("./subscribe").newLogGroups; 177 | await handler(getEvent()); 178 | 179 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 180 | expect(mockListTagsLogGroup).toBeCalled(); 181 | }); 182 | 183 | test("log group is not subscribed if its tag value doesn't match", async () => { 184 | givenTagsIsDefined("tag1=value2"); 185 | const handler = require("./subscribe").newLogGroups; 186 | await handler(getEvent()); 187 | 188 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 189 | expect(mockListTagsLogGroup).toBeCalled(); 190 | }); 191 | 192 | test("log group is not subscribed if it contains matching tag but wrong prefix", async () => { 193 | givenTagsIsDefined("tag2,tag3"); 194 | givenPrefixIsDefined(); 195 | const handler = require("./subscribe").newLogGroups; 196 | await handler(getEvent("/api/gateway/test-me")); 197 | 198 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 199 | expect(mockListTagsLogGroup).not.toBeCalled(); 200 | }); 201 | }); 202 | 203 | describe("add Lambda permission", () => { 204 | test("if encounters Lambda permission error, then attempts to add permission before retrying", async () => { 205 | givenPutFilterFailsWith( 206 | "InvalidParameterException", 207 | "Could not execute the lambda function. Make sure you have given CloudWatch Logs permission to execute your function." 208 | ); 209 | 210 | // succeed when retried 211 | mockPutSubscriptionFilter.mockReturnValueOnce({ 212 | promise: () => Promise.resolve() 213 | }); 214 | 215 | const handler = require("./subscribe").newLogGroups; 216 | await handler(getEvent()); 217 | 218 | expect(mockAddPermission).toBeCalled(); 219 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 220 | }); 221 | }); 222 | 223 | describe("error handling", () => { 224 | test("it should not handle any errors", async () => { 225 | mockPutSubscriptionFilter.mockReset(); 226 | mockPutSubscriptionFilter.mockReturnValue({ 227 | promise: () => Promise.reject(new AwsError("boo", "hoo")) 228 | }); 229 | 230 | const handler = require("./subscribe").newLogGroups; 231 | await expect(handler(getEvent())).rejects.toThrow(); 232 | 233 | expect(mockPutSubscriptionFilter).toBeCalled(); 234 | }); 235 | }); 236 | 237 | describe("exclude prefix", () => { 238 | test("should ignore groups that match the exclude prefix", async () => { 239 | givenExcludePrefixIsDefined(); 240 | 241 | const handler = require("./subscribe").newLogGroups; 242 | await handler(getEvent("/aws/lambda/exclude-me")); 243 | 244 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 245 | }); 246 | }); 247 | 248 | describe("exclude tags", () => { 249 | beforeEach(() => { 250 | mockListTagsLogGroup.mockReturnValueOnce({ 251 | promise: () => Promise.resolve({ 252 | tags: { 253 | tag1: "value1", 254 | tag2: "value2" 255 | } 256 | }) 257 | }); 258 | }); 259 | 260 | describe("when TAGS_MODE is OR", () => { 261 | beforeEach(() => { 262 | process.env.EXCLUDE_TAGS_MODE = "OR"; 263 | }); 264 | 265 | test("log group is not subscribed if it contains at least one matching exclude tag", async () => { 266 | givenExcludeTagsIsDefined("tag2,tag3"); 267 | const handler = require("./subscribe").newLogGroups; 268 | await handler(getEvent()); 269 | 270 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 271 | expect(mockListTagsLogGroup).toBeCalled(); 272 | }); 273 | 274 | test("log group is not subscribed if it contains at least one matching exclude tag AND value", async () => { 275 | givenExcludeTagsIsDefined("tag2=value2,tag3"); 276 | const handler = require("./subscribe").newLogGroups; 277 | await handler(getEvent()); 278 | 279 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 280 | expect(mockListTagsLogGroup).toBeCalled(); 281 | }); 282 | 283 | test("log group is not subscribed if it matches all exclude tags and values", async () => { 284 | givenExcludeTagsIsDefined("tag1=value1,tag2=value2"); 285 | const handler = require("./subscribe").newLogGroups; 286 | await handler(getEvent()); 287 | 288 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 289 | expect(mockListTagsLogGroup).toBeCalled(); 290 | }); 291 | }); 292 | 293 | describe("when TAGS_MODE is AND", () => { 294 | beforeEach(() => { 295 | process.env.EXCLUDE_TAGS_MODE = "AND"; 296 | }); 297 | 298 | test("log group is subscribed if it contains only one matching exclude tag", async () => { 299 | givenExcludeTagsIsDefined("tag2,tag3"); 300 | const handler = require("./subscribe").newLogGroups; 301 | await handler(getEvent()); 302 | 303 | expect(mockPutSubscriptionFilter).toBeCalled(); 304 | expect(mockListTagsLogGroup).toBeCalled(); 305 | }); 306 | 307 | test("log group is subscribed if it contains only one matching exclude tag AND value", async () => { 308 | givenExcludeTagsIsDefined("tag2=value2,tag3"); 309 | const handler = require("./subscribe").newLogGroups; 310 | await handler(getEvent()); 311 | 312 | expect(mockPutSubscriptionFilter).toBeCalled(); 313 | expect(mockListTagsLogGroup).toBeCalled(); 314 | }); 315 | 316 | test("log group is not subscribed if it matches all tags and values", async () => { 317 | givenExcludeTagsIsDefined("tag1=value1,tag2=value2"); 318 | const handler = require("./subscribe").newLogGroups; 319 | await handler(getEvent()); 320 | 321 | expect(mockPutSubscriptionFilter).not.toBeCalled(); 322 | expect(mockListTagsLogGroup).toBeCalled(); 323 | }); 324 | }); 325 | 326 | test("log group is subscribed if it doesn't contain any matching exclude tag", async () => { 327 | givenExcludeTagsIsDefined("tag3,tag4"); 328 | const handler = require("./subscribe").newLogGroups; 329 | await handler(getEvent()); 330 | 331 | expect(mockPutSubscriptionFilter).toBeCalled(); 332 | expect(mockListTagsLogGroup).toBeCalled(); 333 | }); 334 | 335 | test("log group is subscribed if its exclude tag value doesn't match", async () => { 336 | givenExcludeTagsIsDefined("tag1=value2"); 337 | const handler = require("./subscribe").newLogGroups; 338 | await handler(getEvent()); 339 | 340 | expect(mockPutSubscriptionFilter).toBeCalled(); 341 | expect(mockListTagsLogGroup).toBeCalled(); 342 | }); 343 | }); 344 | }); 345 | 346 | describe("existing log group", () => { 347 | test("when prefix is not specified, logGroupNamePrefix is ignored in describe log groups", async () => { 348 | givenDescribeLogGroupsReturns(["/aws/lambda/group1"]); 349 | 350 | givenDescribeFiltersReturns(destinationArn); // group1 (ignored) 351 | 352 | const handler = require("./subscribe").existingLogGroups; 353 | await handler(); 354 | 355 | expect(mockDescribeLogGroups).toBeCalledWith({ 356 | nextToken: undefined 357 | }); 358 | }); 359 | 360 | test("should replace filters that are different", async () => { 361 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 362 | givenDescribeLogGroupsReturns(["/aws/lambda/group3"]); 363 | 364 | givenDescribeFiltersReturns(destinationArn, "ship-logs", "[event]"); // group1 (replaced) 365 | givenDescribeFiltersReturns("some-other-arn"); // group2 (replaced) 366 | givenDescribeFiltersReturns(); // group3 (replaced) 367 | 368 | const handler = require("./subscribe").existingLogGroups; 369 | await handler(); 370 | 371 | expect(mockPutSubscriptionFilter).toBeCalledTimes(3); 372 | const isReplaced = (logGroupName) => { 373 | expect(mockPutSubscriptionFilter).toBeCalledWith({ 374 | destinationArn: destinationArn, 375 | logGroupName, 376 | filterName: "ship-logs", 377 | filterPattern: "" 378 | }); 379 | }; 380 | 381 | isReplaced("/aws/lambda/group1"); 382 | isReplaced("/aws/lambda/group2"); 383 | isReplaced("/aws/lambda/group3"); 384 | }); 385 | 386 | test("should ignore groups that match the exclude prefix", async () => { 387 | givenExcludePrefixIsDefined(); 388 | 389 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 390 | givenDescribeLogGroupsReturns(["/aws/lambda/exclude1", "/aws/lambda/exclude2"]); 391 | 392 | givenDescribeFiltersReturns("some-other-arn"); // group1 (replaced) 393 | givenDescribeFiltersReturns("some-other-arn"); // group2 (replaced) 394 | givenDescribeFiltersReturns("some-other-arn"); // exclude1 (ignored) 395 | givenDescribeFiltersReturns("some-other-arn"); // exclude2 (ignored) 396 | 397 | const handler = require("./subscribe").existingLogGroups; 398 | await handler(); 399 | 400 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 401 | }); 402 | 403 | test("should subscribe groups that match prefix", async () => { 404 | givenPrefixIsDefined(); 405 | 406 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 407 | givenDescribeLogGroupsReturns(["/api/gateway/group1", "/api/gateway/group2"]); 408 | 409 | givenDescribeFiltersReturns("some-other-arn"); // group1 (replaced) 410 | givenDescribeFiltersReturns("some-other-arn"); // group2 (replaced) 411 | givenDescribeFiltersReturns("some-other-arn"); // exclude1 (ignored) 412 | givenDescribeFiltersReturns("some-other-arn"); // exclude2 (ignored) 413 | 414 | const handler = require("./subscribe").existingLogGroups; 415 | await handler(); 416 | 417 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 418 | }); 419 | 420 | test("should subscribe groups that match both prefix and tags", async () => { 421 | givenPrefixIsDefined(); 422 | givenTagsIsDefined("tag2,tag3"); 423 | 424 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 425 | givenDescribeLogGroupsReturns(["/api/gateway/group1", "/api/gateway/group2"]); 426 | 427 | givenListTagsReturns({ 428 | tag2: "value2" 429 | }); // group1 (replaced) 430 | givenListTagsReturns({ 431 | tag2: "value2", 432 | tag3: "value3" 433 | }); // group2 (replaced) 434 | givenListTagsReturns({ 435 | tag1: "value1" 436 | }); // api group1 (ignored) 437 | givenListTagsReturns({ 438 | }); // api group2 (ignored) 439 | 440 | givenDescribeFiltersReturns("some-other-arn"); // group1 (replaced) 441 | givenDescribeFiltersReturns("some-other-arn"); // group2 (replaced) 442 | givenDescribeFiltersReturns("some-other-arn"); // exclude1 (ignored) 443 | givenDescribeFiltersReturns("some-other-arn"); // exclude2 (ignored) 444 | 445 | const handler = require("./subscribe").existingLogGroups; 446 | await handler(); 447 | 448 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 449 | }); 450 | 451 | test("when there are no prefix nor suffix, everything is subscribed", async () => { 452 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"]); 453 | 454 | givenDescribeFiltersReturns(); // group1 (replaced) 455 | givenDescribeFiltersReturns(); // group2 (replaced) 456 | 457 | const handler = require("./subscribe").existingLogGroups; 458 | await handler(); 459 | 460 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 461 | }); 462 | 463 | describe("error handling", () => { 464 | test("it should retry retryable errors when listing log groups", async () => { 465 | givenDescribeLogGroupsFailsWith("ThrottlingException", "Rate exceeded"); 466 | givenDescribeLogGroupsReturns([]); 467 | 468 | const handler = require("./subscribe").existingLogGroups; 469 | await handler(); 470 | 471 | expect(mockDescribeLogGroups).toBeCalledTimes(2); 472 | }); 473 | 474 | test("it should not retry non-retryable errors when listing log groups", async () => { 475 | givenDescribeLogGroupsFailsWith("Foo", "Bar", false); 476 | 477 | const handler = require("./subscribe").existingLogGroups; 478 | await expect(handler()).rejects; 479 | 480 | expect(mockDescribeLogGroups).toBeCalledTimes(1); 481 | }); 482 | 483 | test("it should retry retryable errors when getting a log group's tags", async () => { 484 | givenTagsIsDefined("tag1=value1"); 485 | 486 | givenDescribeLogGroupsReturns(["/aws/lambda/group1"]); 487 | 488 | givenListTagsFailsWith("ThrottlingException", "Rate exceeded"); 489 | givenListTagsReturns({ tag1: "value1" }); 490 | 491 | const handler = require("./subscribe").existingLogGroups; 492 | await handler(); 493 | 494 | expect(mockListTagsLogGroup).toBeCalledTimes(2); 495 | }); 496 | 497 | test("it should not retry non-retryable errors when getting a log group's tags", async () => { 498 | givenTagsIsDefined("tag1=value1"); 499 | 500 | givenDescribeLogGroupsReturns(["/aws/lambda/group1"]); 501 | 502 | givenListTagsFailsWith("Foo", "Bar", false); 503 | 504 | const handler = require("./subscribe").existingLogGroups; 505 | await expect(handler()).resolves.toEqual(undefined); 506 | 507 | expect(mockListTagsLogGroup).toBeCalledTimes(1); 508 | }); 509 | 510 | test("it should retry retryable errors when putting subscription filter", async () => { 511 | givenDescribeLogGroupsReturns(["/aws/lambda/group1"]); 512 | givenDescribeFiltersReturns(); 513 | 514 | mockPutSubscriptionFilter.mockReset(); 515 | givenPutSubscriptionFilterFailsWith("ThrottlingException", "Rate exceeded"); 516 | givenPutSubscriptionFilterSucceeds(); 517 | 518 | const handler = require("./subscribe").existingLogGroups; 519 | await handler(); 520 | 521 | expect(mockPutSubscriptionFilter).toBeCalledTimes(2); 522 | }); 523 | 524 | test("it should not retry non-retryable errors when putting subscription filter", async () => { 525 | givenDescribeLogGroupsReturns(["/aws/lambda/group1"]); 526 | givenDescribeFiltersReturns(); 527 | 528 | mockPutSubscriptionFilter.mockReset(); 529 | givenPutSubscriptionFilterFailsWith("Foo", "Bar", false); 530 | 531 | const handler = require("./subscribe").existingLogGroups; 532 | await expect(handler()).resolves.toEqual(undefined); 533 | 534 | expect(mockPutSubscriptionFilter).toBeCalledTimes(1); 535 | }); 536 | }); 537 | }); 538 | 539 | describe("unsubscribe", () => { 540 | test("should ignore groups that match the exclude prefix", async () => { 541 | givenExcludePrefixIsDefined(); 542 | 543 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 544 | givenDescribeLogGroupsReturns(["/aws/lambda/exclude1", "/aws/lambda/exclude2"]); 545 | 546 | const handler = require("./subscribe").undo; 547 | await handler(); 548 | 549 | expect(mockDeleteSubscriptionFilter).toBeCalledTimes(2); 550 | }); 551 | 552 | test("should unsubscribe groups that match prefix", async () => { 553 | givenPrefixIsDefined(); 554 | 555 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 556 | givenDescribeLogGroupsReturns(["/api/gateway/group1", "/api/gateway/group2"]); 557 | 558 | const handler = require("./subscribe").undo; 559 | await handler(); 560 | 561 | expect(mockDeleteSubscriptionFilter).toBeCalledTimes(2); 562 | }); 563 | 564 | test("should unsubscribe groups that match both prefix and tags", async () => { 565 | givenPrefixIsDefined(); 566 | givenTagsIsDefined("tag2,tag3"); 567 | 568 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"], true); 569 | givenDescribeLogGroupsReturns(["/api/gateway/group1", "/api/gateway/group2"]); 570 | 571 | givenListTagsReturns({ 572 | tag2: "value2" 573 | }); // group1 (replaced) 574 | givenListTagsReturns({ 575 | tag2: "value2", 576 | tag3: "value3" 577 | }); // group2 (replaced) 578 | givenListTagsReturns({ 579 | tag1: "value1" 580 | }); // api group1 (ignored) 581 | givenListTagsReturns({ 582 | }); // api group2 (ignored) 583 | 584 | const handler = require("./subscribe").undo; 585 | await handler(); 586 | 587 | expect(mockDeleteSubscriptionFilter).toBeCalledTimes(2); 588 | }); 589 | 590 | test("when there are no prefix nor suffix, everything is unsubscribed", async () => { 591 | givenDescribeLogGroupsReturns(["/aws/lambda/group1", "/aws/lambda/group2"]); 592 | 593 | const handler = require("./subscribe").undo; 594 | await handler(); 595 | 596 | expect(mockDeleteSubscriptionFilter).toBeCalledTimes(2); 597 | }); 598 | }); 599 | 600 | const givenPutFilterFailsWith = (code, message) => { 601 | mockPutSubscriptionFilter.mockReturnValueOnce({ 602 | promise: () => Promise.reject(new AwsError(code, message)) 603 | }); 604 | }; 605 | 606 | const givenDescribeLogGroupsFailsWith = (code, message, retryable = true) => { 607 | mockDescribeLogGroups.mockReturnValueOnce({ 608 | promise: () => Promise.reject(new AwsError(code, message, retryable)) 609 | }); 610 | }; 611 | 612 | const givenListTagsReturns = (tags) => { 613 | mockListTagsLogGroup.mockReturnValueOnce({ 614 | promise: () => Promise.resolve({ 615 | tags 616 | }) 617 | }); 618 | }; 619 | 620 | const givenListTagsFailsWith = (code, message, retryable = true) => { 621 | mockListTagsLogGroup.mockReturnValueOnce({ 622 | promise: () => Promise.reject(new AwsError(code, message, retryable)) 623 | }); 624 | }; 625 | 626 | const givenDescribeLogGroupsReturns = (logGroups, hasMore = false) => { 627 | mockDescribeLogGroups.mockReturnValueOnce({ 628 | promise: () => Promise.resolve({ 629 | logGroups: logGroups.map(x => ({ logGroupName: x })), 630 | nextToken: hasMore ? "more" : undefined 631 | }) 632 | }); 633 | }; 634 | 635 | const givenDescribeFiltersReturns = (arn, filterName = "ship-logs", filterPattern = "[]") => { 636 | const subscriptionFilters = arn 637 | ? [{ destinationArn: arn, filterName, filterPattern }] 638 | : []; 639 | 640 | mockDescribeSubscriptionFilters.mockReturnValueOnce({ 641 | promise: () => Promise.resolve({ 642 | subscriptionFilters: subscriptionFilters 643 | }) 644 | }); 645 | }; 646 | 647 | const givenPutSubscriptionFilterSucceeds = () => { 648 | mockPutSubscriptionFilter.mockReturnValueOnce({ 649 | promise: () => Promise.resolve() 650 | }); 651 | }; 652 | 653 | const givenPutSubscriptionFilterFailsWith = (code, message, retryable = true) => { 654 | mockPutSubscriptionFilter.mockReturnValueOnce({ 655 | promise: () => Promise.reject(new AwsError(code, message, retryable)) 656 | }); 657 | }; 658 | 659 | class AwsError extends Error { 660 | constructor (code, message, retryable) { 661 | super(message); 662 | 663 | this.code = code; 664 | this.retryable = retryable; 665 | } 666 | } 667 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatch-logs-auto-subscribe", 3 | "private": true, 4 | "version": "1.0.0", 5 | "description": "", 6 | "main": "index.js", 7 | "scripts": { 8 | "test": "jest", 9 | "package": "sam package --template-file template.yml --output-template-file packaged.yml --s3-bucket sar-logging-subscribe", 10 | "publish": "sam publish --template packaged.yml --region us-east-1" 11 | }, 12 | "author": "Yan Cui", 13 | "license": "MIT", 14 | "devDependencies": { 15 | "jest": "^24.8.0" 16 | }, 17 | "dependencies": { 18 | "@dazn/lambda-powertools-logger": "^1.9.0", 19 | "@hapi/joi": "^15.1.1", 20 | "async-retry": "^1.2.3", 21 | "aws-sdk": "^2.814.0", 22 | "uuid": "^3.3.2" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /packages/cloudwatch-logs-auto-subscribe/template.yml: -------------------------------------------------------------------------------- 1 | AWSTemplateFormatVersion: 2010-09-09 2 | Transform: AWS::Serverless-2016-10-31 3 | 4 | Metadata: 5 | AWS::ServerlessRepo::Application: 6 | Name: auto-subscribe-log-group-to-arn 7 | Description: Subscribes new and existing CloudWatch log groups to Lambda/Kinesis/Firehose by ARN. 8 | Author: Lumigo 9 | SpdxLicenseId: Apache-2.0 10 | LicenseUrl: LICENSE 11 | ReadmeUrl: README.md 12 | Labels: ['cloudwatch', 'logs', 'ops', 'devops'] 13 | HomePageUrl: https://github.com/lumigo/SAR-Logging 14 | SemanticVersion: 1.15.0 15 | SourceCodeUrl: https://github.com/lumigo/SAR-Logging 16 | 17 | Globals: 18 | Function: 19 | Runtime: nodejs18.x 20 | MemorySize: 128 21 | Environment: 22 | Variables: 23 | PREFIX: !Ref Prefix 24 | TAGS: !Ref Tags 25 | TAGS_MODE: !Ref TagsMode 26 | EXCLUDE_TAGS: !Ref ExcludeTags 27 | EXCLUDE_TAGS_MODE: !Ref ExcludeTagsMode 28 | EXCLUDE_PREFIX: !Ref ExcludePrefix 29 | DESTINATION_ARN: !Ref DestinationArn 30 | FILTER_NAME: !Ref FilterName 31 | FILTER_PATTERN: !Ref FilterPattern 32 | ROLE_ARN: !GetAtt CloudWatchToKinesisRole.Arn 33 | LOG_LEVEL: INFO 34 | 35 | Conditions: 36 | Unsubscribe: !Equals [!Ref UnsubscribeOnDelete, 'true'] 37 | 38 | Resources: 39 | SubscribeNewLogGroups: 40 | Type: AWS::Serverless::Function 41 | Properties: 42 | Handler: functions/subscribe.newLogGroups 43 | Description: Subscribes a newly create CloudWatch log group to the specified ARN 44 | Timeout: !Ref TimeOut 45 | Policies: 46 | - Statement: 47 | Effect: Allow 48 | Action: 49 | - logs:PutSubscriptionFilter 50 | - logs:ListTagsLogGroup 51 | Resource: '*' 52 | - Statement: 53 | Effect: Allow 54 | Action: lambda:AddPermission 55 | Resource: '*' 56 | - Statement: # required for subscribing to a Kinesis stream 57 | Effect: Allow 58 | Action: iam:PassRole 59 | Resource: '*' 60 | Events: 61 | SubscribeEvent: 62 | Type: CloudWatchEvent 63 | Properties: 64 | Pattern: 65 | source: 66 | - aws.logs 67 | detail-type: 68 | - AWS API Call via CloudTrail 69 | detail: 70 | eventSource: 71 | - logs.amazonaws.com 72 | eventName: 73 | - CreateLogGroup 74 | 75 | SubscribeNewLogGroupsLogGroup: 76 | Type: AWS::Logs::LogGroup 77 | Properties: 78 | LogGroupName: !Sub /aws/lambda/${SubscribeNewLogGroups} 79 | 80 | SubscribeExistingLogGroups: 81 | Type: AWS::Serverless::Function 82 | Properties: 83 | Handler: functions/subscribe.existingLogGroups 84 | Description: Subscribes existing log groups to the specified destination ARN. 85 | Timeout: !Ref TimeOut 86 | Policies: 87 | - Statement: 88 | Effect: Allow 89 | Action: 90 | - logs:PutSubscriptionFilter 91 | - logs:DeleteSubscriptionFilter 92 | - logs:DescribeLogGroups 93 | - logs:DescribeSubscriptionFilters 94 | - logs:ListTagsLogGroup 95 | Resource: '*' 96 | - Statement: 97 | Effect: Allow 98 | Action: lambda:AddPermission 99 | Resource: '*' 100 | - Statement: # required for subscribing to a Kinesis stream 101 | Effect: Allow 102 | Action: iam:PassRole 103 | Resource: '*' 104 | 105 | SubscribeExistingLogGroupsLogGroup: 106 | Type: AWS::Logs::LogGroup 107 | Properties: 108 | LogGroupName: !Sub /aws/lambda/${SubscribeExistingLogGroups} 109 | 110 | UnsubscribeExistingLogGroups: 111 | Type: AWS::Serverless::Function 112 | Condition: Unsubscribe 113 | Properties: 114 | Handler: functions/subscribe.undo 115 | Description: Undo the subscription filter for existing log groups. 116 | Timeout: !Ref TimeOut 117 | Policies: 118 | - Statement: 119 | Effect: Allow 120 | Action: 121 | - logs:DeleteSubscriptionFilter 122 | - logs:DescribeLogGroups 123 | - logs:DescribeSubscriptionFilters 124 | - logs:ListTagsLogGroup 125 | Resource: '*' 126 | 127 | UnsubscribeExistingLogGroupsLogGroup: 128 | Type: AWS::Logs::LogGroup 129 | Condition: Unsubscribe 130 | Properties: 131 | LogGroupName: !Sub /aws/lambda/${UnsubscribeExistingLogGroups} 132 | 133 | CloudWatchToKinesisRole: 134 | Type: AWS::IAM::Role 135 | Properties: 136 | AssumeRolePolicyDocument: 137 | Statement: 138 | Effect: Allow 139 | Action: sts:AssumeRole 140 | Principal: 141 | Service: !Sub logs.${AWS::Region}.amazonaws.com 142 | Policies: 143 | - PolicyName: root 144 | PolicyDocument: 145 | Version : '2012-10-17' 146 | Statement: 147 | - Effect: Allow 148 | Action: 149 | - kinesis:put* 150 | - firehose:put* 151 | Resource: '*' 152 | 153 | LambdaInvocationCustomResource: 154 | Type: AWS::Serverless::Application 155 | Properties: 156 | Location: 157 | ApplicationId: arn:aws:serverlessrepo:us-east-1:374852340823:applications/lambda-invocation-cfn-custom-resource 158 | SemanticVersion: 1.6.0 159 | 160 | # custom resource to invoke the SubscribeExistingLogGroups function during deployment 161 | InvokeSubscribeExistingLogGroups: 162 | Type: Custom::LambdaInvocation 163 | DependsOn: 164 | - SubscribeExistingLogGroups 165 | - LambdaInvocationCustomResource 166 | Properties: 167 | ServiceToken: !GetAtt LambdaInvocationCustomResource.Outputs.FunctionArn 168 | FunctionName: !Ref SubscribeExistingLogGroups 169 | InvocationType: Event # don't wait for the subscribe existing function to finish 170 | # this ties this custom resource to changes to the function's configuration 171 | # so that when CloudFormation would detect this custom resource as "changed" 172 | # and triggers "Update" event during deployment 173 | Configurations: 174 | PREFIX: !Ref Prefix 175 | TAGS: !Ref Tags 176 | TAGS_MODE: !Ref TagsMode 177 | EXCLUDE_TAGS: !Ref ExcludeTags 178 | EXCLUDE_TAGS_MODE: !Ref ExcludeTagsMode 179 | EXCLUDE_PREFIX: !Ref ExcludePrefix 180 | DESTINATION_ARN: !Ref DestinationArn 181 | FILTER_NAME: !Ref FilterName 182 | FILTER_PATTERN: !Ref FilterPattern 183 | ROLE_ARN: !GetAtt CloudWatchToKinesisRole.Arn 184 | LOG_LEVEL: INFO 185 | 186 | # custom resource to invoke the UnsubscribeExistingLogGroups function during deployment 187 | InvokeUnsubscribeExistingLogGroups: 188 | Type: Custom::LambdaInvocation 189 | Condition: Unsubscribe 190 | DependsOn: 191 | - UnsubscribeExistingLogGroups 192 | - LambdaInvocationCustomResource 193 | Properties: 194 | ServiceToken: !GetAtt LambdaInvocationCustomResource.Outputs.FunctionArn 195 | FunctionName: !Ref UnsubscribeExistingLogGroups 196 | InvocationType: RequestResponse # wait for the unsubscribe function to finish 197 | When: Delete # only run this during delete 198 | 199 | Parameters: 200 | DestinationArn: 201 | Type: String 202 | Description: > 203 | The ARN of the Lambda function or Kinesis stream to subscribe a newly created CloudWatch log group to. 204 | Prefix: 205 | Type: String 206 | Default: '' 207 | Description: > 208 | (Optional) if specified then only log groups with the prefix will be subscribed. 209 | E.g. '/aws/lambda/' will subscribe only Lambda function logs 210 | Tags: 211 | Type: String 212 | Default: '' 213 | Description: > 214 | (Optional) specify a common separated list of tags to whitelist log groups with, e.g. 'tag1=value1,tag2,tag3' 215 | Which can test for the existence of a tag (e.g. 'tag2'), and can also test the value of the tag too 216 | (e.g. 'tag1=value1'). 217 | Whether the tests are treated with 'AND' or 'OR' semantics depends on the TagMode parameter. 218 | TagsMode: 219 | Type: String 220 | Default: 'OR' 221 | AllowedValues: 222 | - 'AND' 223 | - 'OR' 224 | Description: > 225 | (Optional) controls how to combine the different tag tests, whether to join them using AND or OR semantic. 226 | Allowed values are "AND" or "OR". Defaults to "OR". 227 | ExcludeTags: 228 | Type: String 229 | Default: '' 230 | Description: > 231 | (Optional) specify a common separated list of tags to blacklist log groups with, e.g. 'tag1=value1,tag2,tag3' 232 | Which can test for the existence of a tag (e.g. 'tag2'), and can also test the value of the tag too 233 | (e.g. 'tag1=value1'). 234 | Whether the tests are treated with 'AND' or 'OR' semantics depends on the ExcludeTagMode parameter. 235 | ExcludeTagsMode: 236 | Type: String 237 | Default: 'OR' 238 | AllowedValues: 239 | - 'AND' 240 | - 'OR' 241 | Description: > 242 | (Optional) controls how to combine the different exclude tag tests, whether to join them using AND or OR semantic. 243 | Allowed values are "AND" or "OR". Defaults to "OR". 244 | FilterName: 245 | Type: String 246 | Default: 'ship-logs' 247 | Description: > 248 | (Optional) if specified, will override the filter name for the subscription. 249 | FilterPattern: 250 | Type: String 251 | Default: '[timestamp=*Z, request_id="*-*", event]' 252 | Description: > 253 | (Optional) if specified, will override the filter pattern used to create the subscription. 254 | ExcludePrefix: 255 | Type: String 256 | Default: '' 257 | Description: > 258 | (Optional) if specified then log groups that match the prefix will not be subscribed. 259 | E.g. '/aws/lambda/my-function-' will exclude Lambda function logs for functions that start with 'my-function-' 260 | UnsubscribeOnDelete: 261 | Type: String 262 | Default: "false" 263 | AllowedValues: 264 | - "true" 265 | - "false" 266 | Description: > 267 | (optional) whether to remove the subscription filters that were added by this app. 268 | Defaults to "false", allowed values are "true" or "false". 269 | TimeOut: 270 | Type: Number 271 | Default: 6 272 | Description: > 273 | (optional) whether to increase the default lambda timeout". 274 | --------------------------------------------------------------------------------