├── .github └── workflows │ └── release.yaml ├── .gitignore ├── .gitlab-ci.yml ├── .make-release-support ├── .release ├── Dockerfile.lambda ├── LICENSE ├── Makefile ├── Makefile.mk ├── Pipfile ├── Pipfile.lock ├── README.md ├── cloudformation └── aws-cloudwatch-log-minder.yaml ├── pyproject.toml ├── requirements.txt ├── setup.py ├── src └── aws_cloudwatch_log_minder │ ├── __init__.py │ ├── __main__.py │ ├── delete_empty_log_groups.py │ ├── delete_empty_log_streams.py │ ├── logger.py │ └── set_log_retention.py └── test-requirements.txt /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: release 2 | on: 3 | push: 4 | tags: 5 | - '*' 6 | jobs: 7 | build: 8 | name: snapshot 9 | runs-on: ubuntu-latest 10 | env: 11 | AWS_ACCESS_KEY_ID: ${{ secrets.aws_access_key_id }} 12 | AWS_SECRET_ACCESS_KEY: ${{ secrets.aws_secret_access_key }} 13 | AWS_DEFAULT_REGION: eu-central-1 14 | steps: 15 | - uses: actions/checkout@v2 16 | - uses: actions/setup-python@v1 17 | with: 18 | python-version: 3.9 19 | - name: checkout 20 | run: git fetch --prune --unshallow 21 | - name: pipenv 22 | run: pip install pipenv && pipenv sync -d 23 | - name: deploy all regions 24 | run: make deploy-all-regions 25 | - name: show workspace 26 | run: find . -type f 27 | - name: distribute application 28 | uses: pypa/gh-action-pypi-publish@release/v1 29 | with: 30 | user: __token__ 31 | password: ${{secrets.twine_password }} 32 | packages_dir: dist/ 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .pytest_cache/ 2 | .py 3 | .idea/workspace.xml 4 | .idea/tasks.xml 5 | .noseids 6 | *.zip 7 | deps/ 8 | .vscode/ 9 | # Byte-compiled / optimized / DLL files 10 | __pycache__/ 11 | *.py[cod] 12 | *$py.class 13 | 14 | # C extensions 15 | *.so 16 | 17 | # Distribution / packaging 18 | .Python 19 | env/ 20 | build/ 21 | develop-eggs/ 22 | dist/ 23 | downloads/ 24 | eggs/ 25 | .eggs/ 26 | lib/ 27 | lib64/ 28 | parts/ 29 | sdist/ 30 | var/ 31 | wheels/ 32 | *.egg-info/ 33 | .installed.cfg 34 | *.egg 35 | 36 | # PyInstaller 37 | # Usually these files are written by a python script from a template 38 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 39 | *.manifest 40 | *.spec 41 | 42 | # Installer logs 43 | pip-log.txt 44 | pip-delete-this-directory.txt 45 | 46 | # Unit test / coverage reports 47 | htmlcov/ 48 | .tox/ 49 | .coverage 50 | .coverage.* 51 | .cache 52 | nosetests.xml 53 | coverage.xml 54 | *.cover 55 | .hypothesis/ 56 | 57 | # Translations 58 | *.mo 59 | *.pot 60 | 61 | # Django stuff: 62 | *.log 63 | local_settings.py 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # pyenv 82 | .python-version 83 | 84 | # celery beat schedule file 85 | celerybeat-schedule 86 | 87 | # SageMath parsed files 88 | *.sage.py 89 | 90 | # dotenv 91 | .env 92 | 93 | # virtualenv 94 | .venv 95 | venv/ 96 | ENV/ 97 | 98 | # Spyder project settings 99 | .spyderproject 100 | .spyproject 101 | 102 | # Rope project settings 103 | .ropeproject 104 | 105 | # mkdocs documentation 106 | /site 107 | 108 | # mypy 109 | .mypy_cache/ 110 | /.idea/ 111 | -------------------------------------------------------------------------------- /.gitlab-ci.yml: -------------------------------------------------------------------------------- 1 | --- 2 | build: 3 | stage: build 4 | script: 5 | - make S3_BUCKET_PREFIX=${LAMBDA_BUCKET_PREFIX:-binxio-public} AWS_REGION=${LAMBDA_BUCKET_REGION:-eu-central-1} deploy 6 | -------------------------------------------------------------------------------- /.make-release-support: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Copyright 2015 Xebia Nederland B.V. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | function hasChanges() { 18 | test -n "$(git status -s .)" 19 | } 20 | 21 | function getRelease() { 22 | awk -F= '/^release=/{print $2}' .release 23 | } 24 | 25 | function getBaseTag() { 26 | sed -n -e "s/^tag=\(.*\)$(getRelease)\$/\1/p" .release 27 | } 28 | 29 | function getTag() { 30 | if [ -z "$1" ] ; then 31 | awk -F= '/^tag/{print $2}' .release 32 | else 33 | echo "$(getBaseTag)$1" 34 | fi 35 | } 36 | 37 | function setRelease() { 38 | if [ -n "$1" ] ; then 39 | sed -i.x -e "s/^tag=.*/tag=$(getTag $1)/" .release 40 | sed -i.x -e "s/^release=.*/release=$1/g" .release 41 | rm -f .release.x 42 | runPreTagCommand "$1" 43 | else 44 | echo "ERROR: missing release version parameter " >&2 45 | return 1 46 | fi 47 | } 48 | 49 | function runPreTagCommand() { 50 | if [ -n "$1" ] ; then 51 | COMMAND=$(sed -n -e "s/@@RELEASE@@/$1/g" -e 's/^pre_tag_command=\(.*\)/\1/p' .release) 52 | if [ -n "$COMMAND" ] ; then 53 | if ! OUTPUT=$(bash -c "$COMMAND" 2>&1) ; then echo $OUTPUT >&2 && exit 1 ; fi 54 | fi 55 | else 56 | echo "ERROR: missing release version parameter " >&2 57 | return 1 58 | fi 59 | } 60 | 61 | function tagExists() { 62 | tag=${1:-$(getTag)} 63 | test -n "$tag" && test -n "$(git tag | grep "^$tag\$")" 64 | } 65 | 66 | function differsFromRelease() { 67 | tag=$(getTag) 68 | ! tagExists $tag || test -n "$(git diff --shortstat -r $tag .)" 69 | } 70 | 71 | function getVersion() { 72 | result=$(getRelease) 73 | 74 | if differsFromRelease; then 75 | result="$result-$(git log -n 1 --format=%h .)" 76 | fi 77 | 78 | if hasChanges ; then 79 | result="$result-dirty" 80 | fi 81 | echo $result 82 | } 83 | 84 | function nextPatchLevel() { 85 | version=${1:-$(getRelease)} 86 | major_and_minor=$(echo $version | cut -d. -f1,2) 87 | patch=$(echo $version | cut -d. -f3) 88 | version=$(printf "%s.%d" $major_and_minor $(($patch + 1))) 89 | echo $version 90 | } 91 | 92 | function nextMinorLevel() { 93 | version=${1:-$(getRelease)} 94 | major=$(echo $version | cut -d. -f1); 95 | minor=$(echo $version | cut -d. -f2); 96 | version=$(printf "%d.%d.0" $major $(($minor + 1))) ; 97 | echo $version 98 | } 99 | 100 | function nextMajorLevel() { 101 | version=${1:-$(getRelease)} 102 | major=$(echo $version | cut -d. -f1); 103 | version=$(printf "%d.0.0" $(($major + 1))) 104 | echo $version 105 | } 106 | -------------------------------------------------------------------------------- /.release: -------------------------------------------------------------------------------- 1 | release=0.6.1 2 | tag=v0.6.1 3 | pre_tag_command=sed -i '' -e 's/lambdas\/aws-cloudwatch-log-minder.*\.zip/lambdas\/aws-cloudwatch-log-minder-@@RELEASE@@.zip/g' cloudformation/aws-cloudwatch-log-minder.yaml README.md && sed -i '' -e 's/version="[^"]*"/version="@@RELEASE@@"/g' setup.py 4 | -------------------------------------------------------------------------------- /Dockerfile.lambda: -------------------------------------------------------------------------------- 1 | FROM public.ecr.aws/lambda/python:3.9 2 | RUN yum install -y zip 3 | WORKDIR /lambda 4 | 5 | ADD requirements.txt /tmp 6 | RUN pip install -t /lambda -r /tmp/requirements.txt 7 | 8 | ADD src/ /lambda/ 9 | RUN find /lambda -type d | xargs -n 1 -I {} chmod ugo+rx "{}" && \ 10 | find /lambda -type f | xargs -n 1 -I {} chmod ugo+r "{}" 11 | 12 | RUN python -m compileall -q /lambda 13 | 14 | ARG ZIPFILE=lambda.zip 15 | RUN zip --quiet -9r /${ZIPFILE} . 16 | 17 | FROM scratch 18 | ARG ZIPFILE 19 | COPY --from=0 /${ZIPFILE} / 20 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2019 binx.io B.V. 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | include Makefile.mk 2 | 3 | NAME=aws-cloudwatch-log-minder 4 | AWS_REGION=eu-central-1 5 | S3_BUCKET_PREFIX=binxio-public 6 | S3_BUCKET=$(S3_BUCKET_PREFIX)-$(AWS_REGION) 7 | 8 | ALL_REGIONS=$(shell aws --region $(AWS_REGION) \ 9 | ec2 describe-regions \ 10 | --query 'join(`\n`, Regions[?RegionName != `$(AWS_REGION)`].RegionName)' \ 11 | --output text) 12 | 13 | help: 14 | @echo 'make - builds a zip file to target/.' 15 | @echo 'make release - builds a zip file and deploys it to s3.' 16 | @echo 'make clean - the workspace.' 17 | @echo 'make test - execute the tests, requires a working AWS connection.' 18 | @echo 'make deploy - lambda to bucket $(S3_BUCKET)' 19 | @echo 'make deploy-all-regions - lambda to all regions with bucket prefix $(S3_BUCKET_PREFIX)' 20 | @echo 'make deploy-lambda - deploys the manager.' 21 | @echo 'make delete-lambda - deletes the manager.' 22 | @echo 'make demo - deploys the provider and the demo cloudformation stack.' 23 | @echo 'make delete-demo - deletes the demo cloudformation stack.' 24 | 25 | deploy-all-regions: deploy 26 | @for REGION in $(ALL_REGIONS); do \ 27 | echo "copying to region $$REGION.." ; \ 28 | aws s3 --region $$REGION \ 29 | cp --acl public-read \ 30 | s3://$(S3_BUCKET_PREFIX)-$(AWS_REGION)/lambdas/$(NAME)-$(VERSION).zip \ 31 | s3://$(S3_BUCKET_PREFIX)-$$REGION/lambdas/$(NAME)-$(VERSION).zip; \ 32 | aws s3 --region $$REGION \ 33 | cp --acl public-read \ 34 | s3://$(S3_BUCKET_PREFIX)-$$REGION/lambdas/$(NAME)-$(VERSION).zip \ 35 | s3://$(S3_BUCKET_PREFIX)-$$REGION/lambdas/$(NAME)-latest.zip; \ 36 | done 37 | 38 | do-push: deploy 39 | 40 | 41 | do-build: Pipfile.lock target/$(NAME)-$(VERSION).zip 42 | 43 | upload-dist: Pipfile.lock 44 | pipenv run twine upload dist/* 45 | 46 | target/$(NAME)-$(VERSION).zip: setup.py src/*/*.py requirements.txt Dockerfile.lambda 47 | mkdir -p target 48 | rm -rf dist/* target/* 49 | pipenv run python setup.py check 50 | pipenv run python setup.py build 51 | pipenv run python setup.py sdist 52 | docker build --build-arg ZIPFILE=$(NAME)-$(VERSION).zip -t $(NAME)-lambda:$(VERSION) -f Dockerfile.lambda . && \ 53 | ID=$$(docker create $(NAME)-lambda:$(VERSION) /bin/true) && \ 54 | docker export $$ID | (cd target && tar -xvf - $(NAME)-$(VERSION).zip) && \ 55 | docker rm -f $$ID && \ 56 | chmod ugo+r target/$(NAME)-$(VERSION).zip 57 | 58 | Pipfile.lock: Pipfile requirements.txt test-requirements.txt setup.py 59 | pipenv update -d 60 | 61 | clean: 62 | rm -rf venv target 63 | find . -name \*.pyc | xargs rm 64 | 65 | test: Pipfile.lock 66 | for i in $$PWD/cloudformation/*; do \ 67 | aws cloudformation validate-template --template-body file://$$i > /dev/null || exit 1; \ 68 | done 69 | [ -z "$(shell ls -1 tests/test*.py 2>/dev/null)" ] || PYTHONPATH=$(PWD)/src pipenv run pytest ./tests/test*.py 70 | 71 | fmt: 72 | black $(shell find src -name \*.py) tests/*.py 73 | 74 | deploy: target/$(NAME)-$(VERSION).zip 75 | aws s3 --region $(AWS_REGION) \ 76 | cp --acl \ 77 | public-read target/$(NAME)-$(VERSION).zip \ 78 | s3://$(S3_BUCKET)/lambdas/$(NAME)-$(VERSION).zip 79 | aws s3 --region $(AWS_REGION) \ 80 | cp --acl public-read \ 81 | s3://$(S3_BUCKET)/lambdas/$(NAME)-$(VERSION).zip \ 82 | s3://$(S3_BUCKET)/lambdas/$(NAME)-latest.zip 83 | 84 | deploy-lambda: deploy target/$(NAME)-$(VERSION).zip 85 | aws cloudformation deploy \ 86 | --capabilities CAPABILITY_IAM \ 87 | --stack-name $(NAME) \ 88 | --template-file ./cloudformation/aws-cloudwatch-log-minder.yaml \ 89 | --parameter-override CFNCustomProviderZipFileName=lambdas/$(NAME)-$(VERSION).zip 90 | 91 | delete-lambda: 92 | aws cloudformation delete-stack --stack-name $(NAME) 93 | aws cloudformation wait stack-delete-complete --stack-name $(NAME) 94 | 95 | -------------------------------------------------------------------------------- /Makefile.mk: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2015 Xebia Nederland B.V. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | REGISTRY_HOST=docker.io 17 | USERNAME=$(USER) 18 | NAME=$(shell basename $(PWD)) 19 | 20 | RELEASE_SUPPORT := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))/.make-release-support 21 | IMAGE=$(REGISTRY_HOST)/$(USERNAME)/$(NAME) 22 | 23 | VERSION=$(shell . $(RELEASE_SUPPORT) ; getVersion) 24 | TAG=$(shell . $(RELEASE_SUPPORT); getTag) 25 | 26 | SHELL=/bin/bash 27 | 28 | .PHONY: pre-build do-build post-build build release patch-release minor-release major-release tag check-status check-release showver \ 29 | push do-push post-push 30 | 31 | build: pre-build do-build post-build 32 | 33 | pre-build: 34 | 35 | 36 | post-build: 37 | 38 | 39 | post-push: 40 | 41 | 42 | do-build: .release 43 | @if [[ -f Dockerfile ]]; then docker build -t $(IMAGE):$(VERSION) . ; else echo "INFO: No Dockerfile found." >/dev/null ; fi 44 | @if [[ -f Dockerfile ]]; then \ 45 | DOCKER_MAJOR=$(shell docker -v | sed -e 's/.*version //' -e 's/,.*//' | cut -d\. -f1) ; \ 46 | DOCKER_MINOR=$(shell docker -v | sed -e 's/.*version //' -e 's/,.*//' | cut -d\. -f2) ; \ 47 | if [ $$DOCKER_MAJOR -eq 1 ] && [ $$DOCKER_MINOR -lt 10 ] ; then \ 48 | echo docker tag -f $(IMAGE):$(VERSION) $(IMAGE):latest ;\ 49 | docker tag -f $(IMAGE):$(VERSION) $(IMAGE):latest ;\ 50 | else \ 51 | echo docker tag $(IMAGE):$(VERSION) $(IMAGE):latest ;\ 52 | docker tag $(IMAGE):$(VERSION) $(IMAGE):latest ; \ 53 | fi ; \ 54 | else \ 55 | echo 'No Dockerfile found.' > /dev/null ;\ 56 | fi 57 | 58 | .release: 59 | @echo "release=0.0.0" > .release 60 | @echo "tag=$(NAME)-0.0.0" >> .release 61 | @echo INFO: .release created 62 | @cat .release 63 | 64 | 65 | release: check-status check-release build push 66 | 67 | 68 | push: do-push post-push 69 | 70 | do-push: 71 | @if [[ -f Dockerfile ]]; then \ 72 | docker push $(IMAGE):$(VERSION) ; \ 73 | docker push $(IMAGE):latest ; \ 74 | else \ 75 | echo > /dev/null ; \ 76 | fi 77 | 78 | snapshot: build push 79 | 80 | showver: .release 81 | @. $(RELEASE_SUPPORT); getVersion 82 | 83 | tag-patch-release: VERSION := $(shell . $(RELEASE_SUPPORT); nextPatchLevel) 84 | tag-patch-release: .release tag 85 | 86 | tag-minor-release: VERSION := $(shell . $(RELEASE_SUPPORT); nextMinorLevel) 87 | tag-minor-release: .release tag 88 | 89 | tag-major-release: VERSION := $(shell . $(RELEASE_SUPPORT); nextMajorLevel) 90 | tag-major-release: .release tag 91 | 92 | patch-release: tag-patch-release release 93 | @echo $(VERSION) 94 | 95 | minor-release: tag-minor-release release 96 | @echo $(VERSION) 97 | 98 | major-release: tag-major-release release 99 | @echo $(VERSION) 100 | 101 | 102 | tag: TAG=$(shell . $(RELEASE_SUPPORT); getTag $(VERSION)) 103 | tag: check-status 104 | @. $(RELEASE_SUPPORT) ; ! tagExists $(TAG) || (echo "ERROR: tag $(TAG) for version $(VERSION) already tagged in git" >&2 && exit 1) ; 105 | @. $(RELEASE_SUPPORT) ; setRelease $(VERSION) 106 | git add . 107 | git commit -m "bumped to version $(VERSION)" ; 108 | git tag $(TAG) ; 109 | @if [ -n "$(shell git remote -v)" ] ; then git push && git push --tags ; fi 110 | 111 | check-status: 112 | @. $(RELEASE_SUPPORT) ; ! hasChanges || (echo "ERROR: there are still outstanding changes" >&2 && exit 1) ; 113 | 114 | check-release: .release 115 | @. $(RELEASE_SUPPORT) ; tagExists $(TAG) || (echo "ERROR: version not yet tagged in git. make [minor,major,patch]-release." >&2 && exit 1) ; 116 | @. $(RELEASE_SUPPORT) ; ! differsFromRelease $(TAG) || (echo "ERROR: current directory differs from tagged $(TAG). make [minor,major,patch]-release." ; exit 1) 117 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | twine = "*" 8 | pytest = "*" 9 | botostubs = "*" 10 | 11 | [packages] 12 | boto3 = "*" 13 | click = "*" 14 | 15 | 16 | [requires] 17 | python_version = "3.7" 18 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "dcf67a64ef166a829c632e04e0f12b5532674ec42a8a8a8e8455ef887f9799a2" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.7" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "boto3": { 20 | "hashes": [ 21 | "sha256:31c0adf71e4bd19a5428580bb229d7ea3b5795eecaa0847a85385df00c026116", 22 | "sha256:4f493a2aed71cee93e626de4f67ce58dd82c0473480a0fc45b131715cd8f4f30" 23 | ], 24 | "index": "pypi", 25 | "version": "==1.26.16" 26 | }, 27 | "botocore": { 28 | "hashes": [ 29 | "sha256:271b599e6cfe214405ed50d41cd967add1d5d469383dd81ff583bc818b47f59b", 30 | "sha256:8cfcc10f2f1751608c3cec694f2d6b5e16ebcd50d0a104f9914d5616227c62e9" 31 | ], 32 | "markers": "python_version >= '3.7'", 33 | "version": "==1.29.16" 34 | }, 35 | "click": { 36 | "hashes": [ 37 | "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e", 38 | "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48" 39 | ], 40 | "index": "pypi", 41 | "version": "==8.1.3" 42 | }, 43 | "importlib-metadata": { 44 | "hashes": [ 45 | "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b", 46 | "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313" 47 | ], 48 | "markers": "python_version < '3.8'", 49 | "version": "==5.1.0" 50 | }, 51 | "jmespath": { 52 | "hashes": [ 53 | "sha256:02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980", 54 | "sha256:90261b206d6defd58fdd5e85f478bf633a2901798906be2ad389150c5c60edbe" 55 | ], 56 | "markers": "python_version >= '3.7'", 57 | "version": "==1.0.1" 58 | }, 59 | "python-dateutil": { 60 | "hashes": [ 61 | "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86", 62 | "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9" 63 | ], 64 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 65 | "version": "==2.8.2" 66 | }, 67 | "s3transfer": { 68 | "hashes": [ 69 | "sha256:06176b74f3a15f61f1b4f25a1fc29a4429040b7647133a463da8fa5bd28d5ecd", 70 | "sha256:2ed07d3866f523cc561bf4a00fc5535827981b117dd7876f036b0c1aca42c947" 71 | ], 72 | "markers": "python_version >= '3.7'", 73 | "version": "==0.6.0" 74 | }, 75 | "six": { 76 | "hashes": [ 77 | "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", 78 | "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" 79 | ], 80 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 81 | "version": "==1.16.0" 82 | }, 83 | "typing-extensions": { 84 | "hashes": [ 85 | "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", 86 | "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" 87 | ], 88 | "markers": "python_version < '3.8'", 89 | "version": "==4.4.0" 90 | }, 91 | "urllib3": { 92 | "hashes": [ 93 | "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", 94 | "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" 95 | ], 96 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", 97 | "version": "==1.26.13" 98 | }, 99 | "zipp": { 100 | "hashes": [ 101 | "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", 102 | "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" 103 | ], 104 | "markers": "python_version >= '3.7'", 105 | "version": "==3.11.0" 106 | } 107 | }, 108 | "develop": { 109 | "attrs": { 110 | "hashes": [ 111 | "sha256:29adc2665447e5191d0e7c568fde78b21f9672d344281d0c6e1ab085429b22b6", 112 | "sha256:86efa402f67bf2df34f51a335487cf46b1ec130d02b8d39fd248abfd30da551c" 113 | ], 114 | "markers": "python_version >= '3.5'", 115 | "version": "==22.1.0" 116 | }, 117 | "bleach": { 118 | "hashes": [ 119 | "sha256:085f7f33c15bd408dd9b17a4ad77c577db66d76203e5984b1bd59baeee948b2a", 120 | "sha256:0d03255c47eb9bd2f26aa9bb7f2107732e7e8fe195ca2f64709fcf3b0a4a085c" 121 | ], 122 | "markers": "python_version >= '3.7'", 123 | "version": "==5.0.1" 124 | }, 125 | "botostubs": { 126 | "hashes": [ 127 | "sha256:c36f7cd408125d203c69744d995aa11f32a3c78225d1448fc350c943f56e7322", 128 | "sha256:cf1b8e01a0e87145be6214a3106fd633632f4dff211f808ad240728f567631de" 129 | ], 130 | "index": "pypi", 131 | "version": "==0.15.1.23.10" 132 | }, 133 | "certifi": { 134 | "hashes": [ 135 | "sha256:0d9c601124e5a6ba9712dbc60d9c53c21e34f5f641fe83002317394311bdce14", 136 | "sha256:90c1a32f1d68f940488354e36370f6cca89f0f106db09518524c88d6ed83f382" 137 | ], 138 | "markers": "python_version >= '3.6'", 139 | "version": "==2022.9.24" 140 | }, 141 | "charset-normalizer": { 142 | "hashes": [ 143 | "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", 144 | "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" 145 | ], 146 | "markers": "python_full_version >= '3.6.0'", 147 | "version": "==2.1.1" 148 | }, 149 | "commonmark": { 150 | "hashes": [ 151 | "sha256:452f9dc859be7f06631ddcb328b6919c67984aca654e5fefb3914d54691aed60", 152 | "sha256:da2f38c92590f83de410ba1a3cbceafbc74fee9def35f9251ba9a971d6d66fd9" 153 | ], 154 | "version": "==0.9.1" 155 | }, 156 | "docutils": { 157 | "hashes": [ 158 | "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6", 159 | "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc" 160 | ], 161 | "markers": "python_version >= '3.7'", 162 | "version": "==0.19" 163 | }, 164 | "exceptiongroup": { 165 | "hashes": [ 166 | "sha256:542adf9dea4055530d6e1279602fa5cb11dab2395fa650b8674eaec35fc4a828", 167 | "sha256:bd14967b79cd9bdb54d97323216f8fdf533e278df937aa2a90089e7d6e06e5ec" 168 | ], 169 | "markers": "python_version < '3.11'", 170 | "version": "==1.0.4" 171 | }, 172 | "idna": { 173 | "hashes": [ 174 | "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", 175 | "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" 176 | ], 177 | "markers": "python_version >= '3.5'", 178 | "version": "==3.4" 179 | }, 180 | "importlib-metadata": { 181 | "hashes": [ 182 | "sha256:d5059f9f1e8e41f80e9c56c2ee58811450c31984dfa625329ffd7c0dad88a73b", 183 | "sha256:d84d17e21670ec07990e1044a99efe8d615d860fd176fc29ef5c306068fda313" 184 | ], 185 | "markers": "python_version < '3.8'", 186 | "version": "==5.1.0" 187 | }, 188 | "iniconfig": { 189 | "hashes": [ 190 | "sha256:011e24c64b7f47f6ebd835bb12a743f2fbe9a26d4cecaa7f53bc4f35ee9da8b3", 191 | "sha256:bc3af051d7d14b2ee5ef9969666def0cd1a000e121eaea580d4a313df4b37f32" 192 | ], 193 | "version": "==1.1.1" 194 | }, 195 | "jaraco.classes": { 196 | "hashes": [ 197 | "sha256:2353de3288bc6b82120752201c6b1c1a14b058267fa424ed5ce5984e3b922158", 198 | "sha256:89559fa5c1d3c34eff6f631ad80bb21f378dbcbb35dd161fd2c6b93f5be2f98a" 199 | ], 200 | "markers": "python_version >= '3.7'", 201 | "version": "==3.2.3" 202 | }, 203 | "keyring": { 204 | "hashes": [ 205 | "sha256:3dd30011d555f1345dec2c262f0153f2f0ca6bca041fb1dc4588349bb4c0ac1e", 206 | "sha256:ad192263e2cdd5f12875dedc2da13534359a7e760e77f8d04b50968a821c2361" 207 | ], 208 | "markers": "python_version >= '3.7'", 209 | "version": "==23.11.0" 210 | }, 211 | "more-itertools": { 212 | "hashes": [ 213 | "sha256:250e83d7e81d0c87ca6bd942e6aeab8cc9daa6096d12c5308f3f92fa5e5c1f41", 214 | "sha256:5a6257e40878ef0520b1803990e3e22303a41b5714006c32a3fd8304b26ea1ab" 215 | ], 216 | "markers": "python_version >= '3.7'", 217 | "version": "==9.0.0" 218 | }, 219 | "packaging": { 220 | "hashes": [ 221 | "sha256:dd47c42927d89ab911e606518907cc2d3a1f38bbd026385970643f9c5b8ecfeb", 222 | "sha256:ef103e05f519cdc783ae24ea4e2e0f508a9c99b2d4969652eed6a2e1ea5bd522" 223 | ], 224 | "markers": "python_version >= '3.6'", 225 | "version": "==21.3" 226 | }, 227 | "pkginfo": { 228 | "hashes": [ 229 | "sha256:848865108ec99d4901b2f7e84058b6e7660aae8ae10164e015a6dcf5b242a594", 230 | "sha256:a84da4318dd86f870a9447a8c98340aa06216bfc6f2b7bdc4b8766984ae1867c" 231 | ], 232 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", 233 | "version": "==1.8.3" 234 | }, 235 | "pluggy": { 236 | "hashes": [ 237 | "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159", 238 | "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3" 239 | ], 240 | "markers": "python_version >= '3.6'", 241 | "version": "==1.0.0" 242 | }, 243 | "pygments": { 244 | "hashes": [ 245 | "sha256:56a8508ae95f98e2b9bdf93a6be5ae3f7d8af858b43e02c5a2ff083726be40c1", 246 | "sha256:f643f331ab57ba3c9d89212ee4a2dabc6e94f117cf4eefde99a0574720d14c42" 247 | ], 248 | "markers": "python_version >= '3.6'", 249 | "version": "==2.13.0" 250 | }, 251 | "pyparsing": { 252 | "hashes": [ 253 | "sha256:2b020ecf7d21b687f219b71ecad3631f644a47f01403fa1d1036b0c6416d70fb", 254 | "sha256:5026bae9a10eeaefb61dab2f09052b9f4307d44aee4eda64b309723d8d206bbc" 255 | ], 256 | "markers": "python_full_version >= '3.6.8'", 257 | "version": "==3.0.9" 258 | }, 259 | "pytest": { 260 | "hashes": [ 261 | "sha256:892f933d339f068883b6fd5a459f03d85bfcb355e4981e146d2c7616c21fef71", 262 | "sha256:c4014eb40e10f11f355ad4e3c2fb2c6c6d1919c73f3b5a433de4708202cade59" 263 | ], 264 | "index": "pypi", 265 | "version": "==7.2.0" 266 | }, 267 | "readme-renderer": { 268 | "hashes": [ 269 | "sha256:cd653186dfc73055656f090f227f5cb22a046d7f71a841dfa305f55c9a513273", 270 | "sha256:f67a16caedfa71eef48a31b39708637a6f4664c4394801a7b0d6432d13907343" 271 | ], 272 | "markers": "python_version >= '3.7'", 273 | "version": "==37.3" 274 | }, 275 | "requests": { 276 | "hashes": [ 277 | "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", 278 | "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" 279 | ], 280 | "markers": "python_version >= '3.7' and python_version < '4'", 281 | "version": "==2.28.1" 282 | }, 283 | "requests-toolbelt": { 284 | "hashes": [ 285 | "sha256:18565aa58116d9951ac39baa288d3adb5b3ff975c4f25eee78555d89e8f247f7", 286 | "sha256:62e09f7ff5ccbda92772a29f394a49c3ad6cb181d568b1337626b2abb628a63d" 287 | ], 288 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 289 | "version": "==0.10.1" 290 | }, 291 | "rfc3986": { 292 | "hashes": [ 293 | "sha256:50b1502b60e289cb37883f3dfd34532b8873c7de9f49bb546641ce9cbd256ebd", 294 | "sha256:97aacf9dbd4bfd829baad6e6309fa6573aaf1be3f6fa735c8ab05e46cecb261c" 295 | ], 296 | "markers": "python_version >= '3.7'", 297 | "version": "==2.0.0" 298 | }, 299 | "rich": { 300 | "hashes": [ 301 | "sha256:a4eb26484f2c82589bd9a17c73d32a010b1e29d89f1604cd9bf3a2097b81bb5e", 302 | "sha256:ba3a3775974105c221d31141f2c116f4fd65c5ceb0698657a11e9f295ec93fd0" 303 | ], 304 | "markers": "python_full_version >= '3.6.3' and python_full_version < '4.0.0'", 305 | "version": "==12.6.0" 306 | }, 307 | "six": { 308 | "hashes": [ 309 | "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926", 310 | "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254" 311 | ], 312 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", 313 | "version": "==1.16.0" 314 | }, 315 | "tomli": { 316 | "hashes": [ 317 | "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc", 318 | "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f" 319 | ], 320 | "markers": "python_version < '3.11'", 321 | "version": "==2.0.1" 322 | }, 323 | "twine": { 324 | "hashes": [ 325 | "sha256:42026c18e394eac3e06693ee52010baa5313e4811d5a11050e7d48436cf41b9e", 326 | "sha256:96b1cf12f7ae611a4a40b6ae8e9570215daff0611828f5fe1f37a16255ab24a0" 327 | ], 328 | "index": "pypi", 329 | "version": "==4.0.1" 330 | }, 331 | "typing-extensions": { 332 | "hashes": [ 333 | "sha256:1511434bb92bf8dd198c12b1cc812e800d4181cfcb867674e0f8279cc93087aa", 334 | "sha256:16fa4864408f655d35ec496218b85f79b3437c829e93320c7c9215ccfd92489e" 335 | ], 336 | "markers": "python_version < '3.8'", 337 | "version": "==4.4.0" 338 | }, 339 | "urllib3": { 340 | "hashes": [ 341 | "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", 342 | "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" 343 | ], 344 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", 345 | "version": "==1.26.13" 346 | }, 347 | "webencodings": { 348 | "hashes": [ 349 | "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78", 350 | "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923" 351 | ], 352 | "version": "==0.5.1" 353 | }, 354 | "zipp": { 355 | "hashes": [ 356 | "sha256:83a28fcb75844b5c0cdaf5aa4003c2d728c77e05f5aeabe8e95e56727005fbaa", 357 | "sha256:a7a22e05929290a67401440b39690ae6563279bced5f314609d9d03798f56766" 358 | ], 359 | "markers": "python_version >= '3.7'", 360 | "version": "==3.11.0" 361 | } 362 | } 363 | } 364 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS Cloudwatch Log minder 2 | AWS CloudWatch logs is an useful logging system, but it has two quircks. It does not allow you too set a default 3 | retention period for newly created log groups, and it does not delete empty log streams that are older than 4 | the retention period. This utility: 5 | 6 | 1. sets a default retention period on log groups without a period set. 7 | 1. removes empty log streams older than the retention period of the log group 8 | 9 | You can use it as a command line utility. You can also install it as an AWS Lambda function and have your 10 | logs kept in order, NoOps style! 11 | 12 | ## install the log minder 13 | to install the log minder, type: 14 | 15 | ```sh 16 | pip install aws-cloudwatch-log-minder 17 | ``` 18 | 19 | ## set default retention period 20 | to set the default retention period on log groups without one, type: 21 | ```sh 22 | cwlog-minder --dry-run set-log-retention --days 30 23 | ``` 24 | This will show you which log groups will have its retention period set. Remove the `--dry-run` and 25 | it the retention period will be set. If you wish to set the retention of all log groups to the same 26 | value, type: 27 | ```sh 28 | cwlog-minder --dry-run set-log-retention --days 30 --overwrite 29 | ``` 30 | 31 | ## delete empty log streams 32 | To delete empty log streams older than the retention period, type: 33 | ```sh 34 | cwlog-minder --dry-run delete-empty-log-streams 35 | ``` 36 | This will show you which empty log streams will be deleted. Remove the `--dry-run` and 37 | these stream will be deleted. 38 | 39 | 40 | ## deploy the log minder 41 | To deploy the log minder as an AWS Lambda, type: 42 | 43 | ```sh 44 | git clone https://github.com/binxio/aws-cloudwatch-log-minder.git 45 | cd aws-cloudwatch-log-minder 46 | aws cloudformation deploy \ 47 | --capabilities CAPABILITY_IAM \ 48 | --stack-name aws-cloudwatch-log-minder \ 49 | --template-file ./cloudformation/aws-cloudwatch-log-minder.yaml \ 50 | --parameter-overrides LogRetentionInDays=30 51 | ``` 52 | This will install the log minder in your AWS account and run every hour. 53 | 54 | ## delete empty log groups 55 | To delete empty log groups, type: 56 | ```sh 57 | cwlog-minder --dry-run delete-empty-log-groups 58 | ``` 59 | This will show you which empty log groups will be deleted. Remove the `--dry-run` and 60 | these groups will be deleted. Do not use this command, if your log groups are 61 | managed by CloudFormation or Terraform. 62 | 63 | ## verbose 64 | 65 | ```sh 66 | export LOG_LEVEL=DEBUG 67 | cwlog-minder ... 68 | ``` 69 | 70 | ## region and profile selection 71 | AWS regions and credential profiles can be selected via command line 72 | arguments or environment variables. 73 | 74 | ### region via parameter 75 | ```sh 76 | cwlog-minder --region eu-west-1 ... 77 | ``` 78 | 79 | ### region via environment 80 | ```sh 81 | export AWS_DEFAULT_REGION=eu-west-1 82 | cwlog-minder ... 83 | ``` 84 | 85 | ### profile via parameter 86 | ```sh 87 | cwlog-minder --profile dev ... 88 | ``` 89 | 90 | ### profile via environment 91 | 92 | ```sh 93 | export AWS_PROFILE=dev 94 | cwlog-minder ... 95 | ``` 96 | -------------------------------------------------------------------------------- /cloudformation/aws-cloudwatch-log-minder.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | AWSTemplateFormatVersion: '2010-09-09' 3 | 4 | Parameters: 5 | LambdaS3Bucket: 6 | Type: String 7 | Default: '' 8 | CFNCustomProviderZipFileName: 9 | Type: String 10 | Default: lambdas/aws-cloudwatch-log-minder-0.6.1.zip 11 | LogRetentionInDays: 12 | Description: 'to apply to log groups without retention' 13 | Type: Number 14 | Default: 30 15 | MinValue: 1 16 | DeleteLogGroups: 17 | Description: 'Set to true to activate log group deletion' 18 | Type: String 19 | Default: false 20 | AllowedValues: 21 | - true 22 | - false 23 | 24 | Conditions: 25 | UsePublicBucket: !Equals 26 | - !Ref 'LambdaS3Bucket' 27 | - '' 28 | CreateLogGroupDeletion: !Equals 29 | - !Ref DeleteLogGroups 30 | - true 31 | 32 | Resources: 33 | DeleteLogGroupPolicy: 34 | Type: AWS::IAM::ManagedPolicy 35 | Condition: CreateLogGroupDeletion 36 | Properties: 37 | PolicyDocument: 38 | Version: 2012-10-17 39 | Statement: 40 | - Effect: Allow 41 | Action: 42 | - logs:DescribeLogGroups 43 | - logs:DescribeLogStreams 44 | - logs:PutRetentionPolicy 45 | - logs:DeleteLogGroup 46 | - logs:GetLogStream 47 | - logs:GetLogEvents 48 | Resource: "*" 49 | - Effect: Allow 50 | Action: 51 | - lambda:InvokeFunction 52 | Resource: 53 | - !Sub 'arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:aws-cloudwatch-delete-empty-log-groups' 54 | 55 | DeleteLogStreamPolicy: 56 | Type: AWS::IAM::ManagedPolicy 57 | Properties: 58 | PolicyDocument: 59 | Version: 2012-10-17 60 | Statement: 61 | - Effect: Allow 62 | Action: 63 | - logs:DescribeLogGroups 64 | - logs:DescribeLogStreams 65 | - logs:PutRetentionPolicy 66 | - logs:DeleteLogStream 67 | - logs:GetLogStream 68 | - logs:GetLogEvents 69 | Resource: "*" 70 | - Effect: Allow 71 | Action: 72 | - lambda:InvokeFunction 73 | Resource: 74 | - !Sub 'arn:${AWS::Partition}:lambda:${AWS::Region}:${AWS::AccountId}:function:aws-cloudwatch-delete-empty-log-streams' 75 | 76 | DeleteLogGroupLambdaRole: 77 | Type: AWS::IAM::Role 78 | Condition: CreateLogGroupDeletion 79 | Properties: 80 | ManagedPolicyArns: 81 | - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 82 | - !Ref DeleteLogGroupPolicy 83 | AssumeRolePolicyDocument: 84 | Version: '2012-10-17' 85 | Statement: 86 | - Effect: Allow 87 | Principal: 88 | Service: 89 | - lambda.amazonaws.com 90 | Action: sts:AssumeRole 91 | 92 | DeleteLogStreamLambdaRole: 93 | Type: AWS::IAM::Role 94 | Properties: 95 | ManagedPolicyArns: 96 | - !Sub arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole 97 | - !Ref DeleteLogStreamPolicy 98 | AssumeRolePolicyDocument: 99 | Version: '2012-10-17' 100 | Statement: 101 | - Effect: Allow 102 | Principal: 103 | Service: 104 | - lambda.amazonaws.com 105 | Action: sts:AssumeRole 106 | 107 | DeleteEmptyLogGroups: 108 | Type: AWS::Lambda::Function 109 | Condition: CreateLogGroupDeletion 110 | Properties: 111 | Description: Delete empty log groups 112 | FunctionName: aws-cloudwatch-delete-empty-log-groups 113 | Code: 114 | S3Bucket: !If 115 | - UsePublicBucket 116 | - !Sub 'binxio-public-${AWS::Region}' 117 | - !Ref 'LambdaS3Bucket' 118 | S3Key: !Ref 'CFNCustomProviderZipFileName' 119 | Handler: aws_cloudwatch_log_minder.delete_empty_log_groups.handle 120 | Role: !GetAtt DeleteLogGroupLambdaRole.Arn 121 | Runtime: python3.9 122 | Timeout: 900 123 | 124 | DeleteEmptyLogStreams: 125 | Type: AWS::Lambda::Function 126 | Properties: 127 | Description: Delete empty log streams older than retention period 128 | FunctionName: aws-cloudwatch-delete-empty-log-streams 129 | Code: 130 | S3Bucket: !If 131 | - UsePublicBucket 132 | - !Sub 'binxio-public-${AWS::Region}' 133 | - !Ref 'LambdaS3Bucket' 134 | S3Key: !Ref 'CFNCustomProviderZipFileName' 135 | Handler: aws_cloudwatch_log_minder.delete_empty_log_streams.handle 136 | Role: !GetAtt DeleteLogStreamLambdaRole.Arn 137 | Runtime: python3.9 138 | Timeout: 900 139 | 140 | SetLogRetention: 141 | Type: AWS::Lambda::Function 142 | Properties: 143 | Description: set default log retention period to groups without one 144 | FunctionName: aws-cloudwatch-set-log-retention 145 | Code: 146 | S3Bucket: !If 147 | - UsePublicBucket 148 | - !Sub 'binxio-public-${AWS::Region}' 149 | - !Ref 'LambdaS3Bucket' 150 | S3Key: !Ref 'CFNCustomProviderZipFileName' 151 | Handler: aws_cloudwatch_log_minder.set_log_retention.handle 152 | Role: !GetAtt DeleteLogStreamLambdaRole.Arn 153 | Runtime: python3.9 154 | Timeout: 900 155 | Environment: 156 | Variables: 157 | DEFAULT_LOG_RETENTION_IN_DAYS: !Ref 'LogRetentionInDays' 158 | 159 | DeleteEmptyLogGroupsSchedulePermission: 160 | Type: AWS::Lambda::Permission 161 | Condition: CreateLogGroupDeletion 162 | Properties: 163 | Action: "lambda:InvokeFunction" 164 | FunctionName: !GetAtt DeleteEmptyLogGroups.Arn 165 | Principal: events.amazonaws.com 166 | 167 | DeleteEmptyLogStreamsSchedulePermission: 168 | Type: AWS::Lambda::Permission 169 | Properties: 170 | Action: "lambda:InvokeFunction" 171 | FunctionName: !GetAtt DeleteEmptyLogStreams.Arn 172 | Principal: events.amazonaws.com 173 | 174 | SetLogRetentionSchedulePermission: 175 | Type: AWS::Lambda::Permission 176 | Properties: 177 | Action: "lambda:InvokeFunction" 178 | FunctionName: !GetAtt SetLogRetention.Arn 179 | Principal: events.amazonaws.com 180 | 181 | SetLogRetentionSchedule: 182 | Type: AWS::Events::Rule 183 | Properties: 184 | Name: aws-cloudwatch-set-log-retention 185 | Description: set log retention on log streams 186 | ScheduleExpression: 'cron(0 * * * ? *)' 187 | State: ENABLED 188 | Targets: 189 | - Id: aws-cloudwatch-set-log-retention 190 | Arn: !GetAtt SetLogRetention.Arn 191 | Input: '{"dry_run": false}' 192 | 193 | DeleteEmptyLogStreamsSchedule: 194 | Type: AWS::Events::Rule 195 | Properties: 196 | Name: aws-cloudwatch-delete-empty-log-streams 197 | Description: delete empty log streams 198 | ScheduleExpression: 'cron(15 * * * ? *)' 199 | State: ENABLED 200 | Targets: 201 | - Id: aws-cloudwatch-delete-empty-log-streams 202 | Arn: !GetAtt DeleteEmptyLogStreams.Arn 203 | Input: '{"dry_run": false}' 204 | 205 | DeleteEmptyLogGroupsSchedule: 206 | Type: AWS::Events::Rule 207 | Condition: CreateLogGroupDeletion 208 | Properties: 209 | Name: aws-cloudwatch-delete-empty-log-groups 210 | Description: delete empty log groups 211 | ScheduleExpression: 'cron(30 * * * ? *)' 212 | State: ENABLED 213 | Targets: 214 | - Id: aws-cloudwatch-delete-empty-log-groups 215 | Arn: !GetAtt DeleteEmptyLogGroups.Arn 216 | Input: '{"dry_run": false}' 217 | 218 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools", "wheel"] 3 | build-backend = "setuptools.build_meta:__legacy__" -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binxio/aws-cloudwatch-log-minder/6195cff5e4e4ee3510f6734ed962ebbc89e7239d/requirements.txt -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """ 2 | The AWS Cloudwatch Log minder 3 | """ 4 | from setuptools import find_packages, setup 5 | 6 | dependencies = ['boto3', 'click'] 7 | 8 | from os import path 9 | this_directory = path.abspath(path.dirname(__file__)) 10 | with open(path.join(this_directory, 'README.md'), encoding='utf-8') as f: 11 | long_description = f.read() 12 | 13 | setup( 14 | name='aws-cloudwatch-log-minder', 15 | version="0.6.1", 16 | url='https://github.com/binxio/aws-cloudwatch-log-minder', 17 | license='BSD', 18 | author='Mark van Holsteijn', 19 | author_email='mark@binx.io', 20 | description='Maintain AWS CloudWatch log streams', 21 | long_description=long_description, 22 | long_description_content_type='text/markdown', 23 | package_dir={'': 'src'}, 24 | packages=find_packages(where='src'), 25 | include_package_data=True, 26 | zip_safe=False, 27 | platforms='any', 28 | install_requires=dependencies, 29 | setup_requires=[], 30 | tests_require=dependencies + ['pytest', 'botostubs', 'pytest-runner'], 31 | test_suite='tests', 32 | entry_points={ 33 | 'console_scripts': [ 34 | 'cwlog-minder = aws_cloudwatch_log_minder.__main__:main' 35 | ], 36 | }, 37 | classifiers=[ 38 | # As from http://pypi.python.org/pypi?%3Aaction=list_classifiers 39 | # 'Development Status :: 1 - Planning', 40 | # 'Development Status :: 2 - Pre-Alpha', 41 | # 'Development Status :: 3 - Alpha', 42 | 'Development Status :: 4 - Beta', 43 | #'Development Status :: 5 - Production/Stable', 44 | # 'Development Status :: 6 - Mature', 45 | # 'Development Status :: 7 - Inactive', 46 | 'Environment :: Console', 47 | 'Intended Audience :: Developers', 48 | 'License :: OSI Approved :: BSD License', 49 | 'Operating System :: POSIX', 50 | 'Operating System :: MacOS', 51 | 'Operating System :: Unix', 52 | 'Operating System :: Microsoft :: Windows', 53 | 'Programming Language :: Python', 54 | 'Programming Language :: Python :: 3', 55 | 'Topic :: Software Development :: Libraries :: Python Modules', 56 | ] 57 | ) 58 | -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/binxio/aws-cloudwatch-log-minder/6195cff5e4e4ee3510f6734ed962ebbc89e7239d/src/aws_cloudwatch_log_minder/__init__.py -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/__main__.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | import click 4 | from .delete_empty_log_streams import delete_empty_log_streams 5 | from .delete_empty_log_groups import delete_empty_log_groups 6 | from .set_log_retention import set_log_retention 7 | 8 | 9 | @click.group() 10 | @click.pass_context 11 | @click.option( 12 | "--dry-run", 13 | is_flag=True, 14 | default=False, 15 | help="do not change anything, just show what is going to happen", 16 | ) 17 | @click.option("--region", help="aws region to connect to", required=False) 18 | @click.option("--profile", help="aws profile to use to connect", required=False) 19 | def main(ctx, dry_run, region, profile): 20 | logging.basicConfig(level=os.getenv("LOG_LEVEL", "WARN")) 21 | ctx.obj = ctx.params 22 | 23 | 24 | @main.command(name="set-log-retention") 25 | @click.pass_context 26 | @click.option("--days", type=int, required=False, default=30, help="retention period") 27 | @click.option( 28 | "--overwrite", is_flag=True, default=False, help="existing retention periods" 29 | ) 30 | @click.option( 31 | "--log-group-name-prefix", 32 | type=str, 33 | required=False, 34 | help="of selected log group only", 35 | ) 36 | def set_log_retention_command(ctx, log_group_name_prefix, days, overwrite): 37 | set_log_retention( 38 | log_group_name_prefix, 39 | days, 40 | overwrite, 41 | ctx.obj["dry_run"], 42 | ctx.obj["region"], 43 | ctx.obj["profile"], 44 | ) 45 | 46 | 47 | @main.command(name="delete-empty-log-streams") 48 | @click.pass_context 49 | @click.option( 50 | "--log-group-name-prefix", 51 | type=str, 52 | required=False, 53 | help="of selected log group only", 54 | ) 55 | @click.option( 56 | "--purge-non-empty", 57 | is_flag=True, 58 | default=False, 59 | help="purge non empty streams older than retention period too", 60 | ) 61 | def delete_empty_log_streams_command(ctx, log_group_name_prefix, purge_non_empty): 62 | delete_empty_log_streams( 63 | log_group_name_prefix, 64 | purge_non_empty, 65 | ctx.obj["dry_run"], 66 | ctx.obj["region"], 67 | ctx.obj["profile"], 68 | ) 69 | 70 | 71 | @main.command(name="delete-empty-log-groups") 72 | @click.pass_context 73 | @click.option( 74 | "--log-group-name-prefix", 75 | type=str, 76 | required=False, 77 | help="of selected log group only", 78 | ) 79 | def delete_empty_log_groups_command(ctx, log_group_name_prefix): 80 | delete_empty_log_groups( 81 | log_group_name_prefix, 82 | ctx.obj["dry_run"], 83 | ctx.obj["region"], 84 | ctx.obj["profile"], 85 | ) 86 | 87 | 88 | if __name__ == "__main__": 89 | main() 90 | -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/delete_empty_log_groups.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime, timedelta 3 | from typing import List 4 | 5 | import boto3 6 | from botocore.config import Config 7 | 8 | from .logger import log 9 | 10 | cw_logs = None 11 | 12 | 13 | def delete_empty_log_groups( 14 | log_group_name_prefix: str = None, 15 | dry_run: bool = False, 16 | region: str = None, 17 | profile: str = None, 18 | ): 19 | global cw_logs 20 | 21 | boto_session = boto3.Session(region_name=region, profile_name=profile) 22 | cw_logs = boto_session.client("logs", config=Config(retries=dict(max_attempts=10))) 23 | 24 | kwargs = {"PaginationConfig": {"PageSize": 50}} 25 | if log_group_name_prefix: 26 | kwargs["logGroupNamePrefix"] = log_group_name_prefix 27 | log.info("finding log groups with prefix %r", log_group_name_prefix) 28 | 29 | for response in cw_logs.get_paginator("describe_log_groups").paginate(**kwargs): 30 | for group in response["logGroups"]: 31 | log_group_name = group["logGroupName"] 32 | response = cw_logs.describe_log_streams(logGroupName=log_group_name) 33 | if len(response["logStreams"]) == 0: 34 | log.debug( 35 | "%s deleting empty log group %s", 36 | ("dry run" if dry_run else ""), 37 | log_group_name, 38 | ) 39 | if dry_run: 40 | continue 41 | cw_logs.delete_log_group(logGroupName=log_group_name) 42 | else: 43 | log.warn( 44 | "%s keeping log group %s as it is not empty", 45 | ("dry run" if dry_run else ""), 46 | log_group_name, 47 | ) 48 | 49 | 50 | def get_all_log_group_names() -> List[str]: 51 | result: List[str] = [] 52 | for response in cw_logs.get_paginator("describe_log_groups").paginate( 53 | PaginationConfig={"PageSize": 50} 54 | ): 55 | result.extend(list(map(lambda g: g["logGroupName"], response["logGroups"]))) 56 | return result 57 | 58 | 59 | def fan_out(function_arn: str, log_group_names: List[str], dry_run: bool): 60 | awslambda = boto3.client("lambda") 61 | log.info( 62 | "recursively invoking %s to delete empty groups from %d log groups", 63 | function_arn, 64 | len(log_group_names), 65 | ) 66 | for log_group_name in log_group_names: 67 | payload = json.dumps( 68 | { 69 | "log_group_name_prefix": log_group_name, 70 | "dry_run": dry_run, 71 | } 72 | ) 73 | awslambda.invoke( 74 | FunctionName=function_arn, InvocationType="Event", Payload=payload 75 | ) 76 | 77 | 78 | def handle(request, context): 79 | global cw_logs 80 | 81 | cw_logs = boto3.client("logs", config=Config(retries=dict(max_attempts=10))) 82 | 83 | dry_run = request.get("dry_run", False) 84 | if "dry_run" in request and not isinstance(dry_run, bool): 85 | raise ValueError(f"'dry_run' is not a boolean value, {request}") 86 | 87 | log_group_name_prefix = request.get("log_group_name_prefix") 88 | if log_group_name_prefix: 89 | delete_empty_log_groups(log_group_name_prefix, dry_run) 90 | else: 91 | fan_out( 92 | context.invoked_function_arn, 93 | get_all_log_group_names(), 94 | dry_run, 95 | ) 96 | -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/delete_empty_log_streams.py: -------------------------------------------------------------------------------- 1 | import json 2 | from datetime import datetime, timedelta 3 | from typing import List 4 | 5 | import boto3 6 | from botocore.config import Config 7 | from botocore.exceptions import ClientError 8 | 9 | from .logger import log 10 | 11 | cw_logs = None 12 | 13 | 14 | def ms_to_datetime(ms: int) -> datetime: 15 | return datetime(1970, 1, 1) + timedelta(milliseconds=ms) 16 | 17 | 18 | def _delete_empty_log_streams( 19 | group: dict, purge_non_empty: bool = False, dry_run: bool = False 20 | ): 21 | now = datetime.utcnow().replace(hour=0, minute=0, second=0, microsecond=0) 22 | log_group_name = group["logGroupName"] 23 | retention_in_days = group.get("retentionInDays", 0) 24 | if not retention_in_days: 25 | log.info( 26 | "skipping log group %s as it has no retention period set", log_group_name 27 | ) 28 | return 29 | 30 | log.debug( 31 | "%s deleting streams from log group %s older than the retention period of %s days", 32 | ("dry run" if dry_run else ""), 33 | log_group_name, 34 | retention_in_days, 35 | ) 36 | 37 | kwargs = { 38 | "logGroupName": log_group_name, 39 | "orderBy": "LastEventTime", 40 | "descending": False, 41 | "PaginationConfig": {"PageSize": 50}, 42 | } 43 | 44 | for response in cw_logs.get_paginator("describe_log_streams").paginate(**kwargs): 45 | for stream in response["logStreams"]: 46 | log_stream_name = stream["logStreamName"] 47 | last_event = ms_to_datetime( 48 | stream.get("lastEventTimestamp", stream.get("creationTime")) 49 | ) 50 | if ( 51 | last_event > (now - timedelta(days=retention_in_days)) 52 | and "lastEventTimestamp" not in stream 53 | ): 54 | log.debug( 55 | "keeping group %s, empty log stream %s, created on %s", 56 | log_group_name, 57 | log_stream_name, 58 | last_event, 59 | ) 60 | continue 61 | elif last_event > (now - timedelta(days=retention_in_days)): 62 | log.info( 63 | "there are no log streams from group %s older than the retention period of %s days", 64 | log_group_name, 65 | retention_in_days, 66 | ) 67 | return 68 | 69 | if not purge_non_empty: 70 | try: 71 | response = cw_logs.get_log_events( 72 | logGroupName=log_group_name, 73 | logStreamName=log_stream_name, 74 | startFromHead=False, 75 | limit=2, 76 | ) 77 | if response["events"]: 78 | log.warn( 79 | "keeping group %s, log stream %s, as it is non empty. Last event stored on %s", 80 | log_group_name, 81 | log_stream_name, 82 | last_event, 83 | ) 84 | continue 85 | except ClientError as e: 86 | if e.response["Error"]["Code"] == "ResourceNotFoundException": 87 | log.warn( 88 | "log stream %s from group %s no longer present in cloudwatch", 89 | log_stream_name, 90 | log_group_name, 91 | ) 92 | 93 | log.debug( 94 | "%s deleting from group %s, log stream %s, with %s bytes last event stored on %s", 95 | ("dry run" if dry_run else ""), 96 | log_group_name, 97 | log_stream_name, 98 | stream["storedBytes"], 99 | last_event, 100 | ) 101 | if dry_run: 102 | continue 103 | 104 | try: 105 | cw_logs.delete_log_stream( 106 | logGroupName=log_group_name, logStreamName=log_stream_name 107 | ) 108 | except ClientError as e: 109 | if e.response["Error"]["Code"] == "ResourceNotFoundException": 110 | log.warn( 111 | "log stream %s from group %s already deleted", 112 | log_stream_name, 113 | log_group_name, 114 | ) 115 | else: 116 | log.error( 117 | "failed to delete log stream %s from group %s, %s", 118 | log_stream_name, 119 | log_group_name, 120 | e, 121 | ) 122 | 123 | 124 | def delete_empty_log_streams( 125 | log_group_name_prefix: str = None, 126 | purge_non_empty: bool = False, 127 | dry_run: bool = False, 128 | region: str = None, 129 | profile: str = None, 130 | ): 131 | global cw_logs 132 | 133 | boto_session = boto3.Session(region_name=region, profile_name=profile) 134 | cw_logs = boto_session.client("logs", config=Config(retries=dict(max_attempts=10))) 135 | 136 | kwargs = {"PaginationConfig": {"PageSize": 50}} 137 | if log_group_name_prefix: 138 | kwargs["logGroupNamePrefix"] = log_group_name_prefix 139 | log.info("finding log groups with prefix %r", log_group_name_prefix) 140 | 141 | for response in cw_logs.get_paginator("describe_log_groups").paginate(**kwargs): 142 | for group in response["logGroups"]: 143 | _delete_empty_log_streams(group, purge_non_empty, dry_run) 144 | 145 | 146 | def get_all_log_group_names() -> List[str]: 147 | result: List[str] = [] 148 | for response in cw_logs.get_paginator("describe_log_groups").paginate( 149 | PaginationConfig={"PageSize": 50} 150 | ): 151 | result.extend(list(map(lambda g: g["logGroupName"], response["logGroups"]))) 152 | return result 153 | 154 | 155 | def fan_out( 156 | function_arn: str, log_group_names: List[str], purge_non_empty: bool, dry_run: bool 157 | ): 158 | awslambda = boto3.client("lambda") 159 | log.info( 160 | "recursively invoking %s to delete empty log streams from %d log groups", 161 | function_arn, 162 | len(log_group_names), 163 | ) 164 | for log_group_name in log_group_names: 165 | payload = json.dumps( 166 | { 167 | "log_group_name_prefix": log_group_name, 168 | "purge_non_empty": purge_non_empty, 169 | "dry_run": dry_run, 170 | } 171 | ) 172 | awslambda.invoke( 173 | FunctionName=function_arn, InvocationType="Event", Payload=payload 174 | ) 175 | 176 | 177 | def handle(request, context): 178 | global cw_logs 179 | 180 | cw_logs = boto3.client("logs", config=Config(retries=dict(max_attempts=10))) 181 | 182 | dry_run = request.get("dry_run", False) 183 | if "dry_run" in request and not isinstance(dry_run, bool): 184 | raise ValueError(f"'dry_run' is not a boolean value, {request}") 185 | 186 | purge_non_empty = request.get("purge_non_empty", False) 187 | if "purge_non_empty" in request and not isinstance(dry_run, bool): 188 | raise ValueError(f"'purge_non_empty' is not a boolean value, {request}") 189 | 190 | log_group_name_prefix = request.get("log_group_name_prefix") 191 | if log_group_name_prefix: 192 | delete_empty_log_streams(log_group_name_prefix, purge_non_empty, dry_run) 193 | else: 194 | fan_out( 195 | context.invoked_function_arn, 196 | get_all_log_group_names(), 197 | purge_non_empty, 198 | dry_run, 199 | ) 200 | -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/logger.py: -------------------------------------------------------------------------------- 1 | import os 2 | import logging 3 | 4 | if len(logging.getLogger().handlers) == 0: 5 | logging.basicConfig( 6 | format="%(levelname)s: %(message)s", level=os.getenv("LOG_LEVEL", "INFO") 7 | ) 8 | log = logging.getLogger() 9 | log.setLevel(os.getenv("LOG_LEVEL", "INFO")) 10 | -------------------------------------------------------------------------------- /src/aws_cloudwatch_log_minder/set_log_retention.py: -------------------------------------------------------------------------------- 1 | import os 2 | import boto3 3 | from botocore.exceptions import ClientError 4 | from .logger import log 5 | 6 | 7 | cw_logs = None 8 | 9 | 10 | def set_log_retention( 11 | log_group_name_prefix: str = None, 12 | retention_in_days: int = 30, 13 | overwrite: bool = False, 14 | dry_run: bool = False, 15 | region: str = None, 16 | profile: str = None, 17 | ): 18 | global cw_logs 19 | 20 | boto_session = boto3.Session(region_name=region, profile_name=profile) 21 | cw_logs = boto_session.client("logs") 22 | 23 | kwargs = {"PaginationConfig": {"PageSize": 50}} 24 | if log_group_name_prefix: 25 | kwargs["logGroupNamePrefix"] = log_group_name_prefix 26 | log.info("finding log groups with prefix %r", log_group_name_prefix) 27 | 28 | for response in cw_logs.get_paginator("describe_log_groups").paginate(**kwargs): 29 | for group in response["logGroups"]: 30 | log_group_name = group["logGroupName"] 31 | current_retention = group.get("retentionInDays") 32 | if not current_retention or ( 33 | overwrite and int(current_retention) != retention_in_days 34 | ): 35 | try: 36 | if current_retention: 37 | log.info( 38 | "%s overwriting current retention period of %s of log stream %s to %s", 39 | ("dry run" if dry_run else ""), 40 | current_retention, 41 | log_group_name, 42 | retention_in_days, 43 | ) 44 | else: 45 | log.info( 46 | "%s setting default retention period of log stream %s to %s", 47 | ("dry run" if dry_run else ""), 48 | log_group_name, 49 | retention_in_days, 50 | ) 51 | if dry_run: 52 | continue 53 | cw_logs.put_retention_policy( 54 | logGroupName=log_group_name, retentionInDays=retention_in_days 55 | ) 56 | except ClientError as e: 57 | log.error( 58 | "failed to set retention period of log stream %s to %s, %s", 59 | log_group_name, 60 | retention_in_days, 61 | e, 62 | ) 63 | else: 64 | log.debug( 65 | "retention period on %s already set to %s", 66 | log_group_name, 67 | current_retention, 68 | ) 69 | 70 | 71 | def handle(request, context): 72 | global cw_logs 73 | 74 | cw_logs = boto3.client("logs") 75 | 76 | dry_run = request.get("dry_run", False) 77 | if "dry_run" in request and not isinstance(dry_run, bool): 78 | raise ValueError(f"'dry_run' is not a boolean value, {request}") 79 | 80 | overwrite = request.get("overwrite", False) 81 | if "overwrite" in request and not isinstance(overwrite, bool): 82 | raise ValueError(f"'overwrite' is not a boolean value, {request}") 83 | 84 | default_log_retention = int(os.getenv("DEFAULT_LOG_RETENTION_IN_DAYS", "30")) 85 | days = request.get("days", default_log_retention) 86 | if "days" in request and not isinstance(days, int): 87 | raise ValueError(f"'days' is not a integer value, {request}") 88 | 89 | log_group_name_prefix = request.get("log_group_name_prefix") 90 | 91 | set_log_retention( 92 | log_group_name_prefix=log_group_name_prefix, 93 | retention_in_days=days, 94 | overwrite=overwrite, 95 | dry_run=dry_run, 96 | ) 97 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | boto3 2 | pytest 3 | botostubs 4 | --------------------------------------------------------------------------------