├── .dockerignore ├── .gitignore ├── Dockerfile ├── Instructions.md ├── LICENSE ├── Makefile ├── NOTICE ├── README.md ├── aws_setup.sh ├── cloud ├── aws.go ├── billing │ ├── aws.go │ ├── billing.go │ ├── gcp.go │ └── pricing.go ├── bucket.go ├── cloud.go ├── filter │ ├── filter.go │ ├── filter_test.go │ ├── helpers.go │ ├── rules.go │ └── rules_test.go ├── gcp.go ├── image.go ├── instance.go ├── resource.go ├── snapshot.go └── volume.go ├── cloudsweeper ├── cleanup │ └── cleanup.go ├── find │ ├── aws.go │ └── find.go ├── notify │ ├── helpers.go │ ├── notify.go │ └── templates.go ├── organization.go └── setup │ ├── aws.go │ └── setup.go ├── cmd └── cloudsweeper │ ├── config.go │ └── main.go ├── config.conf ├── mailer └── mailer.go └── organization.json /.dockerignore: -------------------------------------------------------------------------------- 1 | aws_setup.sh 2 | config.conf 3 | Dockerfile 4 | Instructions.md 5 | LICENSE 6 | Makefile 7 | NOTICE 8 | organization.json 9 | README.md 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | # Created by https://www.gitignore.io/api/macos,linux,windows,intellij,sublimetext,visualstudio,visualstudiocode 3 | 4 | ### Intellij ### 5 | # Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio and Webstorm 6 | # Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 7 | 8 | # User-specific stuff: 9 | .idea/**/workspace.xml 10 | .idea/**/tasks.xml 11 | .idea/dictionaries 12 | 13 | # Sensitive or high-churn files: 14 | .idea/**/dataSources/ 15 | .idea/**/dataSources.ids 16 | .idea/**/dataSources.xml 17 | .idea/**/dataSources.local.xml 18 | .idea/**/sqlDataSources.xml 19 | .idea/**/dynamic.xml 20 | .idea/**/uiDesigner.xml 21 | 22 | # Gradle: 23 | .idea/**/gradle.xml 24 | .idea/**/libraries 25 | 26 | # CMake 27 | cmake-build-debug/ 28 | 29 | # Mongo Explorer plugin: 30 | .idea/**/mongoSettings.xml 31 | 32 | ## File-based project format: 33 | *.iws 34 | 35 | ## Plugin-specific files: 36 | 37 | # IntelliJ 38 | /out/ 39 | 40 | # mpeltonen/sbt-idea plugin 41 | .idea_modules/ 42 | 43 | # JIRA plugin 44 | atlassian-ide-plugin.xml 45 | 46 | # Cursive Clojure plugin 47 | .idea/replstate.xml 48 | 49 | # Ruby plugin and RubyMine 50 | /.rakeTasks 51 | 52 | # Crashlytics plugin (for Android Studio and IntelliJ) 53 | com_crashlytics_export_strings.xml 54 | crashlytics.properties 55 | crashlytics-build.properties 56 | fabric.properties 57 | 58 | ### Intellij Patch ### 59 | # Comment Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-215987721 60 | 61 | # *.iml 62 | # modules.xml 63 | # .idea/misc.xml 64 | # *.ipr 65 | 66 | # Sonarlint plugin 67 | .idea/sonarlint 68 | 69 | ### Linux ### 70 | *~ 71 | 72 | # temporary files which can be created if a process still has a handle open of a deleted file 73 | .fuse_hidden* 74 | 75 | # KDE directory preferences 76 | .directory 77 | 78 | # Linux trash folder which might appear on any partition or disk 79 | .Trash-* 80 | 81 | # .nfs files are created when an open file is removed but is still being accessed 82 | .nfs* 83 | 84 | ### macOS ### 85 | *.DS_Store 86 | .AppleDouble 87 | .LSOverride 88 | 89 | # Icon must end with two \r 90 | Icon 91 | 92 | # Thumbnails 93 | ._* 94 | 95 | # Files that might appear in the root of a volume 96 | .DocumentRevisions-V100 97 | .fseventsd 98 | .Spotlight-V100 99 | .TemporaryItems 100 | .Trashes 101 | .VolumeIcon.icns 102 | .com.apple.timemachine.donotpresent 103 | 104 | # Directories potentially created on remote AFP share 105 | .AppleDB 106 | .AppleDesktop 107 | Network Trash Folder 108 | Temporary Items 109 | .apdisk 110 | 111 | ### SublimeText ### 112 | # cache files for sublime text 113 | *.tmlanguage.cache 114 | *.tmPreferences.cache 115 | *.stTheme.cache 116 | 117 | # workspace files are user-specific 118 | *.sublime-workspace 119 | 120 | # project files should be checked into the repository, unless a significant 121 | # proportion of contributors will probably not be using SublimeText 122 | # *.sublime-project 123 | 124 | # sftp configuration file 125 | sftp-config.json 126 | 127 | # Package control specific files 128 | Package Control.last-run 129 | Package Control.ca-list 130 | Package Control.ca-bundle 131 | Package Control.system-ca-bundle 132 | Package Control.cache/ 133 | Package Control.ca-certs/ 134 | Package Control.merged-ca-bundle 135 | Package Control.user-ca-bundle 136 | oscrypto-ca-bundle.crt 137 | bh_unicode_properties.cache 138 | 139 | # Sublime-github package stores a github token in this file 140 | # https://packagecontrol.io/packages/sublime-github 141 | GitHub.sublime-settings 142 | 143 | ### VisualStudioCode ### 144 | .vscode/* 145 | !.vscode/settings.json 146 | !.vscode/tasks.json 147 | !.vscode/launch.json 148 | !.vscode/extensions.json 149 | .history 150 | 151 | ### Windows ### 152 | # Windows thumbnail cache files 153 | Thumbs.db 154 | ehthumbs.db 155 | ehthumbs_vista.db 156 | 157 | # Folder config file 158 | Desktop.ini 159 | 160 | # Recycle Bin used on file shares 161 | $RECYCLE.BIN/ 162 | 163 | # Windows Installer files 164 | *.cab 165 | *.msi 166 | *.msm 167 | *.msp 168 | 169 | # Windows shortcuts 170 | *.lnk 171 | 172 | ### VisualStudio ### 173 | ## Ignore Visual Studio temporary files, build results, and 174 | ## files generated by popular Visual Studio add-ons. 175 | ## 176 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore 177 | 178 | # User-specific files 179 | *.suo 180 | *.user 181 | *.userosscache 182 | *.sln.docstates 183 | 184 | # User-specific files (MonoDevelop/Xamarin Studio) 185 | *.userprefs 186 | 187 | # Build results 188 | [Dd]ebug/ 189 | [Dd]ebugPublic/ 190 | [Rr]elease/ 191 | [Rr]eleases/ 192 | x64/ 193 | x86/ 194 | bld/ 195 | [Bb]in/ 196 | [Oo]bj/ 197 | [Ll]og/ 198 | 199 | # Visual Studio 2015 cache/options directory 200 | .vs/ 201 | # Uncomment if you have tasks that create the project's static files in wwwroot 202 | #wwwroot/ 203 | 204 | # MSTest test Results 205 | [Tt]est[Rr]esult*/ 206 | [Bb]uild[Ll]og.* 207 | 208 | # NUNIT 209 | *.VisualState.xml 210 | TestResult.xml 211 | 212 | # Build Results of an ATL Project 213 | [Dd]ebugPS/ 214 | [Rr]eleasePS/ 215 | dlldata.c 216 | 217 | # .NET Core 218 | project.lock.json 219 | project.fragment.lock.json 220 | artifacts/ 221 | **/Properties/launchSettings.json 222 | 223 | *_i.c 224 | *_p.c 225 | *_i.h 226 | *.ilk 227 | *.meta 228 | *.obj 229 | *.pch 230 | *.pdb 231 | *.pgc 232 | *.pgd 233 | *.rsp 234 | *.sbr 235 | *.tlb 236 | *.tli 237 | *.tlh 238 | *.tmp 239 | *.tmp_proj 240 | *.log 241 | *.vspscc 242 | *.vssscc 243 | .builds 244 | *.pidb 245 | *.svclog 246 | *.scc 247 | 248 | # Chutzpah Test files 249 | _Chutzpah* 250 | 251 | # Visual C++ cache files 252 | ipch/ 253 | *.aps 254 | *.ncb 255 | *.opendb 256 | *.opensdf 257 | *.sdf 258 | *.cachefile 259 | *.VC.db 260 | *.VC.VC.opendb 261 | 262 | # Visual Studio profiler 263 | *.psess 264 | *.vsp 265 | *.vspx 266 | *.sap 267 | 268 | # TFS 2012 Local Workspace 269 | $tf/ 270 | 271 | # Guidance Automation Toolkit 272 | *.gpState 273 | 274 | # ReSharper is a .NET coding add-in 275 | _ReSharper*/ 276 | *.[Rr]e[Ss]harper 277 | *.DotSettings.user 278 | 279 | # JustCode is a .NET coding add-in 280 | .JustCode 281 | 282 | # TeamCity is a build add-in 283 | _TeamCity* 284 | 285 | # DotCover is a Code Coverage Tool 286 | *.dotCover 287 | 288 | # Visual Studio code coverage results 289 | *.coverage 290 | *.coveragexml 291 | 292 | # NCrunch 293 | _NCrunch_* 294 | .*crunch*.local.xml 295 | nCrunchTemp_* 296 | 297 | # MightyMoose 298 | *.mm.* 299 | AutoTest.Net/ 300 | 301 | # Web workbench (sass) 302 | .sass-cache/ 303 | 304 | # Installshield output folder 305 | [Ee]xpress/ 306 | 307 | # DocProject is a documentation generator add-in 308 | DocProject/buildhelp/ 309 | DocProject/Help/*.HxT 310 | DocProject/Help/*.HxC 311 | DocProject/Help/*.hhc 312 | DocProject/Help/*.hhk 313 | DocProject/Help/*.hhp 314 | DocProject/Help/Html2 315 | DocProject/Help/html 316 | 317 | # Click-Once directory 318 | publish/ 319 | 320 | # Publish Web Output 321 | *.[Pp]ublish.xml 322 | *.azurePubxml 323 | # TODO: Uncomment the next line to ignore your web deploy settings. 324 | # By default, sensitive information, such as encrypted password 325 | # should be stored in the .pubxml.user file. 326 | #*.pubxml 327 | *.pubxml.user 328 | *.publishproj 329 | 330 | # Microsoft Azure Web App publish settings. Comment the next line if you want to 331 | # checkin your Azure Web App publish settings, but sensitive information contained 332 | # in these scripts will be unencrypted 333 | PublishScripts/ 334 | 335 | # NuGet Packages 336 | *.nupkg 337 | # The packages folder can be ignored because of Package Restore 338 | **/packages/* 339 | # except build/, which is used as an MSBuild target. 340 | !**/packages/build/ 341 | # Uncomment if necessary however generally it will be regenerated when needed 342 | #!**/packages/repositories.config 343 | # NuGet v3's project.json files produces more ignorable files 344 | *.nuget.props 345 | *.nuget.targets 346 | 347 | # Microsoft Azure Build Output 348 | csx/ 349 | *.build.csdef 350 | 351 | # Microsoft Azure Emulator 352 | ecf/ 353 | rcf/ 354 | 355 | # Windows Store app package directories and files 356 | AppPackages/ 357 | BundleArtifacts/ 358 | Package.StoreAssociation.xml 359 | _pkginfo.txt 360 | 361 | # Visual Studio cache files 362 | # files ending in .cache can be ignored 363 | *.[Cc]ache 364 | # but keep track of directories ending in .cache 365 | !*.[Cc]ache/ 366 | 367 | # Others 368 | ClientBin/ 369 | ~$* 370 | *.dbmdl 371 | *.dbproj.schemaview 372 | *.jfm 373 | *.pfx 374 | *.publishsettings 375 | orleans.codegen.cs 376 | 377 | # Since there are multiple workflows, uncomment next line to ignore bower_components 378 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) 379 | #bower_components/ 380 | 381 | # RIA/Silverlight projects 382 | Generated_Code/ 383 | 384 | # Backup & report files from converting an old project file 385 | # to a newer Visual Studio version. Backup files are not needed, 386 | # because we have git ;-) 387 | _UpgradeReport_Files/ 388 | Backup*/ 389 | UpgradeLog*.XML 390 | UpgradeLog*.htm 391 | 392 | # SQL Server files 393 | *.mdf 394 | *.ldf 395 | *.ndf 396 | 397 | # Business Intelligence projects 398 | *.rdl.data 399 | *.bim.layout 400 | *.bim_*.settings 401 | 402 | # Microsoft Fakes 403 | FakesAssemblies/ 404 | 405 | # GhostDoc plugin setting file 406 | *.GhostDoc.xml 407 | 408 | # Node.js Tools for Visual Studio 409 | .ntvs_analysis.dat 410 | node_modules/ 411 | 412 | # Typescript v1 declaration files 413 | typings/ 414 | 415 | # Visual Studio 6 build log 416 | *.plg 417 | 418 | # Visual Studio 6 workspace options file 419 | *.opt 420 | 421 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.) 422 | *.vbw 423 | 424 | # Visual Studio LightSwitch build output 425 | **/*.HTMLClient/GeneratedArtifacts 426 | **/*.DesktopClient/GeneratedArtifacts 427 | **/*.DesktopClient/ModelManifest.xml 428 | **/*.Server/GeneratedArtifacts 429 | **/*.Server/ModelManifest.xml 430 | _Pvt_Extensions 431 | 432 | # Paket dependency manager 433 | .paket/paket.exe 434 | paket-files/ 435 | 436 | # FAKE - F# Make 437 | .fake/ 438 | 439 | # JetBrains Rider 440 | .idea/ 441 | *.sln.iml 442 | 443 | # CodeRush 444 | .cr/ 445 | 446 | # Python Tools for Visual Studio (PTVS) 447 | __pycache__/ 448 | *.pyc 449 | 450 | # Cake - Uncomment if you are using it 451 | # tools/** 452 | # !tools/packages.config 453 | 454 | # Telerik's JustMock configuration file 455 | *.jmconfig 456 | 457 | # BizTalk build output 458 | *.btp.cs 459 | *.btm.cs 460 | *.odx.cs 461 | *.xsd.cs 462 | 463 | ### VisualStudio Patch ### 464 | # By default, sensitive information, such as encrypted password 465 | # should be stored in the .pubxml.user file. 466 | 467 | # End of https://www.gitignore.io/api/macos,linux,windows,intellij,sublimetext,visualstudio,visualstudiocode 468 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # STEP 1 build executable binary 2 | FROM golang:1.10-alpine3.7 as builder 3 | 4 | ADD . $GOPATH/src/github.com/cloudtools/cloudsweeper 5 | WORKDIR $GOPATH/src/github.com/cloudtools/cloudsweeper 6 | 7 | RUN apk -U upgrade && \ 8 | apk add --no-cache -U git && \ 9 | apk add --no-cache -U ca-certificates && \ 10 | update-ca-certificates && \ 11 | go get ./... && \ 12 | go test -cover ./... && \ 13 | CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -installsuffix cgo -ldflags="-w -s" -o /cs cmd/cloudsweeper/*.go 14 | 15 | FROM scratch 16 | COPY --from=builder /cs /cs 17 | COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt 18 | COPY --from=builder /usr/share/ca-certificates/* /usr/share/ca-certificates/ 19 | ENTRYPOINT [ "/cs" ] 20 | -------------------------------------------------------------------------------- /Instructions.md: -------------------------------------------------------------------------------- 1 | # Cloudsweeper Instructions 2 | 3 | ## Content 4 | 5 | - [Overview](#Overview) 6 | - [Account Setup](#account-setup) 7 | - [Master Setup](#master-setup) 8 | - [Slave Setup](#slave-setup) 9 | - [Organization definition](#organization-definition) 10 | - [Configuration](#configuration) 11 | - [Building](#building) 12 | - [Functionality](#functionality) 13 | 14 | ## Overview 15 | What is Cloudsweeper? Cloudsweeper is a tool that, once properly setup, can monitor and take action in multiple cloud accounts (supporting both AWS and GCP). The problem it solved when first created was to make sure that all cloud accounts in the compnay were used as effectively as possible, not leaving any unused resources sticking around. 16 | 17 | At Bracket Computing, Inc. (where it was originally created), Cloudsweeper was ran daily to perform different operations that monitored and cleaned up all employees' accounts. 18 | 19 | ## Account Setup 20 | There are two parts to properly setting up Cloudsweeper; setting up the _master_, and setting up all employee/slave accounts. 21 | 22 | ### Master setup 23 | The _master_ is the machine/server where Cloudsweeper will run from. This will be responsible for _assuming_ into the slave accounts to perform monitoring and/or cleanup. 24 | 25 | Typically, the master will always run from the same machine or system — perhaps form a Jenkins instance? It would be recommended to setup the master to run at some defined interval, performing the actions you want. 26 | 27 | For AWS, you should have the master always run from the same account, using the same role. This is important to let the slave accounts only need to allow role assumption from a single ARN. Start by deciding which account to run Cloudsweeper with, note its account ID. Then create an IAM user in this account that can assume into other accounts, note its name. When you have done this, you need the ARN, which will look like: 28 | 29 | ``` 30 | arn:aws:iam:::user/ 31 | ``` 32 | 33 | Let's call this the _master ARN_. 34 | 35 | ### Slave setup 36 | The slaves are the accounts that Cloudsweeper will monitor. These need to be setup so that the master can access them. 37 | 38 | For AWS, the easiest way to setup a slave account is to run either the built-in Cloudsweeper setup command or to run the `aws_setup.sh` script. Both of these methods assumes that the machine they are being run on are setup with AWS (i.e. having set `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`). This will setup an IAM role in the account with the name `Cloudsweeper` that has an IAM policy with the name `CloudsweeperPolicy` — these names are important and can not be modified. 39 | 40 | When the `Cloudsweeper` role is being created, an assume policy is attached to it which allows the _master ARN_ (see above) to assume into the role. If using any of the two automatic setup methods, make sure to first configure them to use your unique master ARN. If using the `aws_setup.sh` script, you must set the environment variable `CS_MASTER_ARN`. If using the built-in setup command you must either specify the flag `--aws-master-arn` when running the command or add the ARN to the `config.conf` file. 41 | 42 | This setup must be done for all accounts that should be monitored by Cloudsweeper. 43 | 44 | ## Organization definition 45 | Cloudsweeper uses a central organization definition file (see `example-org.json` for an example) to figure out which slaves/employees it should monitor. Every employee can own multiple different AWS and GCP accounts, and can have a manager assigned to them. 46 | 47 | - `managers` is a list of all managers 48 | - `departments` is a list of all departments 49 | - `employees` is a list of all employees 50 | 51 | All managers should also be definied in the list of employees with the same username. The username should preferably match with the person's email alias, as this will be used by Cloudsweeper to send out mail (it should just be the alias, i.e. the part before the `@`, as the domain part is configured). To enable cloudsweeper in an employee's account, it's important to specify `cloudsweeper_enabled: true`, as it defaults to `false` otherwise. 52 | 53 | **NOTE:** Employees obviously don't need to be actual employees, they can be anything. An _employee_ could be the Production account for example, and another could be Stage. 54 | 55 | ## Configuration 56 | In order for Cloudsweeper to work properly, besides the previously mentioned setup, it needs to be configured. The recommended way to configure Cloudsweeper is to use the `config.conf` file, however, all configuration can also be made through command line flags. Flags take precedence over the `config.conf` file, so it can be used to override anything specified in that file. The flags themselves can be discovered by either running `./cloudsweeper --help` or by looking in the `cmd/cloudsweeper/main.go` file. 57 | 58 | The `config.conf` file contains descriptions of all configuration options. 59 | 60 | ## Building 61 | Cloudsweeper was built using Go 1.10. In order to complile it, you need to either install Go or Docker. For building with Go, simply run: 62 | ``` 63 | $ go get ./... 64 | $ go build -o cs cmd/cloudsweeper/*.go 65 | ``` 66 | Which will create a binary called `cs` which you can then execute (e.g `./cs setup`). 67 | 68 | If using Docker, the easiest way to build is by using the make target `make build`, which will build a container `cloudsweeper:latest`. 69 | 70 | **IMPORTANT:** If building with Docker, modify the `Dockerfile` to use your own organization JSON file (it defaults to `example-org.json`). This could also reference a remote file in e.g. an S3 bucket. 71 | 72 | ## Functionality 73 | The best way to explore what Cloudsweeper can do is to look at the source code. It is however divided into some different parts. A good way to start exploring is to look at the `notify` and the `cleanup` packages within the `cloudsweeper`. For example, the `cleanup` command will run the `PerformCleanup` function in the `cleanup` package. This function in turn delegates to other functions, and this would be an ideal place for you to add more things to be clean up if you wanted to extend Cloudsweeper. 74 | 75 | All notification and cleanup functions leverages a filtering system which makes it really easy to first filter out the resources you wanna operate on and then perform some action (e.g. clean them up) — you could draw parallels to map-reduce. -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 VMware, Inc. All rights reserved. 2 | 3 | Redistribution and use in source and binary forms, with or without 4 | modification, are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 2. Redistributions in binary form must reproduce the above copyright notice, 9 | this list of conditions and the following disclaimer in the documentation 10 | and/or other materials provided with the distribution. 11 | 12 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 13 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 14 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 15 | ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE 16 | LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 17 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 18 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 19 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 20 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 21 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 22 | POSSIBILITY OF SUCH DAMAGE. 23 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ORG_FILE := organization.json 2 | CONF_FILE := config.conf 3 | WARNING_HOURS := 48 4 | DOCKER_GOOGLE_FLAG := $(shell echo $${GOOGLE_APPLICATION_CREDENTIALS:+-v ${GOOGLE_APPLICATION_CREDENTIALS}:/google-creds -e GOOGLE_APPLICATION_CREDENTIALS=/google-creds}) 5 | CONTAINER_TAG := cloudsweeper 6 | 7 | build: 8 | docker build -t $(CONTAINER_TAG) . 9 | 10 | clean-build: 11 | docker image rm $(CONTAINER_TAG) 12 | 13 | push: build 14 | docker push $(CONTAINER_TAG):latest 15 | 16 | run: build 17 | docker run \ 18 | -e AWS_ACCESS_KEY_ID \ 19 | -e AWS_SECRET_ACCESS_KEY \ 20 | $(DOCKER_GOOGLE_FLAG) \ 21 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 22 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 23 | --rm $(CONTAINER_TAG) 24 | 25 | cleanup: build 26 | docker run \ 27 | -e AWS_ACCESS_KEY_ID \ 28 | -e AWS_SECRET_ACCESS_KEY \ 29 | $(DOCKER_GOOGLE_FLAG) \ 30 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 31 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 32 | --rm $(CONTAINER_TAG) cleanup 33 | 34 | reset: build 35 | docker run \ 36 | -e AWS_ACCESS_KEY_ID \ 37 | -e AWS_SECRET_ACCESS_KEY \ 38 | $(DOCKER_GOOGLE_FLAG) \ 39 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 40 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 41 | --rm $(CONTAINER_TAG) reset 42 | 43 | review: build 44 | docker run \ 45 | -e AWS_ACCESS_KEY_ID \ 46 | -e AWS_SECRET_ACCESS_KEY \ 47 | $(DOCKER_GOOGLE_FLAG) \ 48 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 49 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 50 | --rm $(CONTAINER_TAG) review 51 | 52 | mark: build 53 | docker run \ 54 | -e AWS_ACCESS_KEY_ID \ 55 | -e AWS_SECRET_ACCESS_KEY \ 56 | $(DOCKER_GOOGLE_FLAG) \ 57 | --rm $(CONTAINER_TAG) mark-for-cleanup 58 | 59 | warn: build 60 | docker run \ 61 | -e AWS_ACCESS_KEY_ID \ 62 | -e AWS_SECRET_ACCESS_KEY \ 63 | $(DOCKER_GOOGLE_FLAG) \ 64 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 65 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 66 | --rm $(CONTAINER_TAG) warn 67 | 68 | untagged: build 69 | docker run \ 70 | -e AWS_ACCESS_KEY_ID \ 71 | -e AWS_SECRET_ACCESS_KEY \ 72 | $(DOCKER_GOOGLE_FLAG) \ 73 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 74 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 75 | --rm $(CONTAINER_TAG) find-untagged 76 | 77 | billing-report: build 78 | docker run \ 79 | -e AWS_ACCESS_KEY_ID \ 80 | -e AWS_SECRET_ACCESS_KEY \ 81 | $(DOCKER_GOOGLE_FLAG) \ 82 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 83 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 84 | --rm $(CONTAINER_TAG) billing-report 85 | 86 | find: build 87 | docker run \ 88 | -e AWS_ACCESS_KEY_ID \ 89 | -e AWS_SECRET_ACCESS_KEY \ 90 | $(DOCKER_GOOGLE_FLAG) \ 91 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 92 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 93 | --rm $(CONTAINER_TAG) --resource-id=$(RESOURCE_ID) find-resource 94 | 95 | setup: build 96 | docker run \ 97 | -e AWS_ACCESS_KEY_ID \ 98 | -e AWS_SECRET_ACCESS_KEY \ 99 | $(DOCKER_GOOGLE_FLAG) \ 100 | -v $(shell pwd)/$(ORG_FILE):/$(ORG_FILE) \ 101 | -v $(shell pwd)/$(CONF_FILE):/$(CONF_FILE) \ 102 | --rm -it $(CONTAINER_TAG) setup 103 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | 3 | This product is licensed to you under the BSD-2 license (the "License"). 4 | You may not use this product except in compliance with the BSD-2 License. 5 | 6 | This product may include a number of subcomponents with separate 7 | copyright notices and license terms. Your use of these subcomponents 8 | is subject to the terms and conditions of the subcomponent's license, 9 | as noted in the LICENSE file. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Cloudsweeper 2 | 3 | ## Overview 4 | Cloudsweeper was created because developers have better things to do than to constantly monitor their cloud accounts for things they left around. The tool monitors developer accounts for old items that might have been forgotten, and then sends reports to you or others to let them know what will be deleted (and to whitelist anything they do not want deleted). It also can let you know when you haven't been tagging things properly. 5 | 6 | Detailed instructions can be found in `Instructions.md`. 7 | 8 | ## Setup 9 | To setup Cloudsweeper to work with your account, you must create a role in AWS to allow access. This is easily done using the `--setup` flag of Cloudsweeper. It will handle everything for you. Then all you need to do is to make sure you're in the list of accounts that will be checked. 10 | 11 | It's also possible to run `aws_setup.sh`, if you have the `aws` CLI installed and properly setup. 12 | 13 | **IMPORTANT:** When running any of these setup methods, an environemnt variable named `CS_MASTER_ARN` must be set. This should be the AWS ARN of a role in a specific account that should have permission to assume into your account. This should match up with the role and account used to run Cloudsweeper, as Cloudsweeper will attempt to assume into your account from the role and account specified with this ARN. An example of valid input it: 14 | ``` 15 | arn:aws:iam::123456789123:user/cloudsweeper-master 16 | ``` 17 | 18 | ## Usage 19 | The program relies on having a list of accounts to actually check. This list can either be provided manually, or through other scripts. 20 | 21 | The recommended way of using Cloudsweeper is through Docker. For the most common use cases, there are make targets (take a look in the `Makefile`). 22 | 23 | ## Modes 24 | Below are the different modes that Cloudsweeper runs in. 25 | 26 | ### Review - `make review` 27 | The review target will look for really old resources that Cloudsweeper is too unsure about to automatically cleanup. These resources are filtered based on some rules 28 | The defaults are: 29 | 30 | - Resource is older than 30 days 31 | - A whitelisted resource is older than 6 months 32 | - An instance marked with do-not-delete is older than a week 33 | 34 | The account owner will get an email with these resources listed. 35 | 36 | These thresholds may be modified to your own preference. 37 | 38 | ### Warning - `make warn` 39 | The warning target will look for resources that are about to be automatically cleaned up by Cloudsweeper (not resources that the owner explicitly said should be deleted) and warn the owner about this. 40 | 41 | ### Marking - `make mark` 42 | Marking will go through resources in the a users account and look for those that match a certain set of rules. If a resource matches, it will be marked for deletion. Deletion is set a few days in the future, so the user has time to whitelist anything that shouldn't be deleted. Resources are matched using the following rules: 43 | - unattached volumes > 30 days old 44 | - unused/unaccessed buckets > 120 days old 45 | - non-whitelisted AMIs > 6 months 46 | - non-whitelisted snapshots > 6 months 47 | - non-whitelisted volumes > 6 months 48 | - untagged resources > 30 days (this should take care of instances) 49 | 50 | The resources will be marked with a tag with key `cloudsweeper-delete-at` and the value be a RFC3339 encoded timestamp. 51 | 52 | ### Finding resources - `RESOURCE_ID= make find` 53 | Cloudsweeper can be used to find out more details about a specified resource in AWS. This is useful to quickly get some more details if all you have is a resource ID. If using the make target, the `RESOURCE_ID` variable must be set. If running the command directly, use the `--resource-id` flag. 54 | 55 | ### Cleanup - `make cleanup` 56 | The cleanup target will look through resources and delete those that should be cleaned up. This is determined by looking at tags of the resources. 57 | There are certain thresholds that can be configured for this target. You can get more information on what those are by looking at the `--help` flag in the executable or by looking at the `config.conf` file 58 | There are three requirements for this deletion: 59 | #### Lifetime 60 | A resource can have a lifetime. This is specified with the tag `Key: cloudsweeper-lifetime, Value: days-X`, where `X` is the number of days to keep the resource after its creation date. If the current date is after a resource's creation date + the lifetime it will get cleaned up. 61 | #### Expiry 62 | A resource can have an expiry date. This is specified with the tag `Key: cloudsweeper-expiry, Value: YYYY-MM-DD`, where `YYYY-MM-DD` e.g. `2018-01-29`. If the current date is after the expiry date, the resource will be cleaned up. 63 | #### Delete at 64 | If cloudsweeper has automatically marked a resource for deletion, it will have a tag with the key `cloudsweeper-delete-at`, and the value will be an RFC3339 encoded timestamp. If the current time is after that timestamp, the resource will get cleaned up. 65 | 66 | ## LICENSE 67 | CloudSweeper is licensed under the BSD 2-clause licenses. Originally written 68 | at Bracket Computing, it was made open source by VMware to enable further 69 | development by the original authors. 70 | -------------------------------------------------------------------------------- /aws_setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | POLICY_NAME="CloudsweeperPolicy" 4 | ROLE_NAME="Cloudsweeper" 5 | 6 | CLOUDSWEEPER_POLICY='{ 7 | "Version": "2012-10-17", 8 | "Statement": [ 9 | { 10 | "Effect": "Allow", 11 | "Action": [ 12 | "ec2:DescribeInstances", 13 | "ec2:DescribeInstanceAttribute", 14 | "ec2:DescribeSnapshots", 15 | "ec2:DescribeVolumeStatus", 16 | "ec2:DescribeVolumes", 17 | "ec2:DescribeInstanceStatus", 18 | "ec2:DescribeTags", 19 | "ec2:DescribeVolumeAttribute", 20 | "ec2:DescribeImages", 21 | "ec2:DescribeSnapshotAttribute", 22 | "ec2:DeregisterImage", 23 | "ec2:DeleteSnapshot", 24 | "ec2:DeleteTags", 25 | "ec2:ModifyImageAttribute", 26 | "ec2:DeleteVolume", 27 | "ec2:TerminateInstances", 28 | "ec2:CreateTags", 29 | "ec2:StopInstances", 30 | "s3:GetBucketTagging", 31 | "s3:ListBucket", 32 | "s3:GetObject", 33 | "s3:ListAllMyBuckets", 34 | "s3:GetBucketLocation", 35 | "s3:PutBucketTagging", 36 | "s3:DeleteObject", 37 | "s3:DeleteBucket", 38 | "cloudwatch:GetMetricStatistics" 39 | ], 40 | "Resource": [ 41 | "*" 42 | ] 43 | } 44 | ] 45 | }' 46 | 47 | ASSUME_POLICY_DOCUMENT_TEMPLATE='{ 48 | "Version": "2012-10-17", 49 | "Statement": [ 50 | { 51 | "Effect": "Allow", 52 | "Principal": { 53 | "AWS": "%s" 54 | }, 55 | "Action": "sts:AssumeRole" 56 | } 57 | ] 58 | }' 59 | 60 | ASSUME_POLICY_DOCUMENT=$(printf "$ASSUME_POLICY_DOCUMENT_TEMPLATE" "$CS_MASTER_ARN") 61 | 62 | account=$(aws sts get-caller-identity --output text --query 'Account') 63 | 64 | echo "Creating policy" 65 | aws iam create-policy --policy-name=$POLICY_NAME --policy-document="$CLOUDSWEEPER_POLICY" 66 | echo "Creating role" 67 | aws iam create-role --role-name=$ROLE_NAME --assume-role-policy-document="$ASSUME_POLICY_DOCUMENT" 68 | echo "Attaching policy to role" 69 | aws iam attach-role-policy --role-name=$ROLE_NAME --policy-arn=arn:aws:iam::${account}:policy/$POLICY_NAME 70 | -------------------------------------------------------------------------------- /cloud/billing/aws.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package billing 5 | 6 | import ( 7 | "archive/zip" 8 | "encoding/csv" 9 | "errors" 10 | "fmt" 11 | "io" 12 | "log" 13 | "os" 14 | "path/filepath" 15 | "strconv" 16 | "strings" 17 | "time" 18 | 19 | "github.com/aws/aws-sdk-go/aws" 20 | "github.com/cloudtools/cloudsweeper/cloud" 21 | 22 | "github.com/aws/aws-sdk-go/aws/session" 23 | "github.com/aws/aws-sdk-go/service/s3" 24 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 25 | ) 26 | 27 | const ( 28 | awsCSVDateFormat = "2006-01-02" 29 | awsCSVNameFormat = "%s-aws-billing-detailed-line-items-%d-%02d.csv.zip" 30 | awsCSVNameFormatWithTags = "%s-aws-billing-detailed-line-items-with-resources-and-tags-%d-%02d.csv.zip" 31 | ) 32 | 33 | type awsReporter struct { 34 | csp cloud.CSP 35 | billingAccount string 36 | billingBucket string 37 | billingBucketRegion string 38 | sortByTag string 39 | } 40 | 41 | func (r *awsReporter) GenerateReport(start time.Time) Report { 42 | report := Report{} 43 | report.CSP = r.csp 44 | 45 | var name string 46 | if r.sortByTag == "" { 47 | name = fmt.Sprintf(awsCSVNameFormat, r.billingAccount, start.Year(), start.Month()) 48 | } else { 49 | name = fmt.Sprintf(awsCSVNameFormatWithTags, r.billingAccount, start.Year(), start.Month()) 50 | } 51 | 52 | csvFile, err := r.getCSVFromS3(name) 53 | if err != nil { 54 | log.Println("Failed to get", name, ":", err) 55 | } 56 | err = r.processAwsCsv(&report, csvFile, true) 57 | if err != nil { 58 | log.Println("Failed to process CSV", name) 59 | } 60 | 61 | return report 62 | } 63 | 64 | func (r *awsReporter) processAwsCsv(report *Report, csvFile *csv.Reader, allowFailed bool) error { 65 | csvHeaders := make(map[string]int) 66 | line := 0 67 | for { 68 | record, err := csvFile.Read() 69 | if err == io.EOF { 70 | return nil 71 | } 72 | if err != nil { 73 | if allowFailed { 74 | log.Printf("Failed reading line %d, continuing...\n%s", line, err) 75 | } else { 76 | return err 77 | } 78 | } 79 | if line == 0 { 80 | csvHeaders = updateCsvHeaders(record) 81 | line++ 82 | continue 83 | } 84 | if record[csvHeaders["RecordType"]] != "LineItem" { 85 | // Ignore lines with AccountTotal (so we don't count it twice) 86 | line++ 87 | continue 88 | } 89 | 90 | reportItem := ReportItem{} 91 | reportItem.Owner = record[csvHeaders["LinkedAccountId"]] 92 | reportItem.Description = record[csvHeaders["ItemDescription"]] 93 | cost := record[csvHeaders["UnBlendedCost"]] 94 | cost = strings.Replace(cost, ",", "", -1) 95 | costNumber, err := strconv.ParseFloat(cost, 64) 96 | if err != nil { 97 | if allowFailed { 98 | log.Println("Could not convert cost to float:", cost) 99 | } else { 100 | return err 101 | } 102 | } 103 | reportItem.Cost = costNumber 104 | if r.sortByTag != "" { 105 | if idx, exist := csvHeaders[fmt.Sprintf("user:%s", r.sortByTag)]; exist { 106 | reportItem.sortTagValue = record[idx] 107 | } else if idx, exist := csvHeaders[fmt.Sprintf("aws:%s", r.sortByTag)]; exist { 108 | reportItem.sortTagValue = record[idx] 109 | } else if !allowFailed { 110 | return fmt.Errorf("Could not find tag %s in report", r.sortByTag) 111 | } 112 | } 113 | report.Items = append(report.Items, reportItem) 114 | line++ 115 | } 116 | } 117 | 118 | func (r *awsReporter) getCSVFromS3(name string) (*csv.Reader, error) { 119 | tmpZip := filepath.Join(os.TempDir(), name) 120 | f, err := os.Create(tmpZip) 121 | if err != nil { 122 | log.Println("Could not create file in temp directory") 123 | return nil, err 124 | } 125 | sess := session.Must(session.NewSession()) 126 | sess.Config.Region = aws.String(r.billingBucketRegion) 127 | downloader := s3manager.NewDownloader(sess) 128 | input := &s3.GetObjectInput{ 129 | Bucket: aws.String(r.billingBucket), 130 | Key: aws.String(name), 131 | } 132 | _, err = downloader.Download(f, input) 133 | if err != nil { 134 | log.Println("Could not find bucket") 135 | return nil, err 136 | } 137 | reader, err := zip.OpenReader(tmpZip) 138 | if err != nil { 139 | log.Println("Could not read ZIP file") 140 | return nil, err 141 | } 142 | //defer reader.Close() 143 | if len(reader.File) == 0 { 144 | return nil, errors.New("Zip file was empty") 145 | } 146 | file := reader.File[0] 147 | log.Println("Using", file.Name) 148 | rc, err := file.Open() 149 | if err != nil { 150 | log.Println("Billing CSV is corrupt:", err) 151 | return nil, err 152 | } 153 | return csv.NewReader(rc), nil 154 | } 155 | -------------------------------------------------------------------------------- /cloud/billing/billing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package billing 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "sort" 10 | "time" 11 | 12 | "github.com/cloudtools/cloudsweeper/cloud" 13 | ) 14 | 15 | const ( 16 | dateFormatLayout = "2006-01-02" 17 | // MinimumTotalCost is also used in notify.MonthToDateReport 18 | MinimumTotalCost = 10.0 19 | // MinimumCost is also used in notify.MonthToDateReport 20 | MinimumCost = 5.0 21 | ) 22 | 23 | // ReportItem represent a single item in a report. This is usually 24 | // the cost for a specific service for a certain user in a certain 25 | // account/project. 26 | type ReportItem struct { 27 | Owner string 28 | Description string 29 | Cost float64 30 | sortTagValue string 31 | } 32 | 33 | // User represents an User and it's TotalCost 34 | // plus a CostList of all associated DetailedCosts 35 | type User struct { 36 | Name string 37 | TotalCost float64 38 | DetailedCosts CostList 39 | } 40 | 41 | // UserList respresents a list of Users 42 | type UserList []User 43 | 44 | func (l UserList) Len() int { return len(l) } 45 | func (l UserList) Less(i, j int) bool { return l[i].TotalCost < l[j].TotalCost } 46 | func (l UserList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 47 | 48 | // DetailedCost represents a Cost and Description for a Users expense 49 | type DetailedCost struct { 50 | Cost float64 51 | Description string 52 | } 53 | 54 | // CostList respresents a list of Costs 55 | type CostList []DetailedCost 56 | 57 | func (l CostList) Len() int { return len(l) } 58 | func (l CostList) Less(i, j int) bool { return l[i].Cost < l[j].Cost } 59 | func (l CostList) Swap(i, j int) { l[i], l[j] = l[j], l[i] } 60 | 61 | // Reporter is a general interface that can be implemented 62 | // for both AWS and GCP to generate expense reports. 63 | type Reporter interface { 64 | GenerateReport(start time.Time) Report 65 | } 66 | 67 | // NewReporterAWS will initialize a new Reporter for the AWS cloud. This 68 | // requires specifying the account which holds the billing information, 69 | // the bucket where the billing CSVs can be found as well as which region 70 | // this bucket is in. None of these arguments must be empty. 71 | func NewReporterAWS(billingAccount, bucket, bucketRegion, sortTag string) Reporter { 72 | if billingAccount == "" || bucket == "" || bucketRegion == "" { 73 | panic("Invalid arguments, must not be empty (\"\")") 74 | } 75 | return &awsReporter{ 76 | csp: cloud.AWS, 77 | billingAccount: billingAccount, 78 | billingBucket: bucket, 79 | billingBucketRegion: bucketRegion, 80 | sortByTag: sortTag, 81 | } 82 | } 83 | 84 | // NewReporterGCP initializes and returns a new Reporter for the GCP cloud. 85 | // This requires specifying a bucket where the billing CSVs can be found, as 86 | // well as the prefix of these CSV files. The prefix will be prepended to 87 | // the date and .csv suffix (e.g. -2018-10-09.csv). None of 88 | // these argument must be empty. 89 | func NewReporterGCP(bucket, csvPrefix string) Reporter { 90 | if bucket == "" || csvPrefix == "" { 91 | panic("Invalid argument, must not be empty") 92 | } 93 | return &gcpReporter{ 94 | csp: cloud.GCP, 95 | bucket: bucket, 96 | csvNamePrefix: csvPrefix, 97 | } 98 | } 99 | 100 | // Report contains a collection of items, and some metadata 101 | // about when the items were collected and which dates they 102 | // span. The report struct also has methods to help work with 103 | // all the items. 104 | type Report struct { 105 | CSP cloud.CSP 106 | Items []ReportItem 107 | } 108 | 109 | // TotalCost returns the total cost for all items 110 | func (r *Report) TotalCost() float64 { 111 | total := 0.0 112 | for i := range r.Items { 113 | total += r.Items[i].Cost 114 | } 115 | return total 116 | } 117 | 118 | // SortedUsersByTotalCost returns a sorted list of Users by TotalCost 119 | func (r *Report) SortedUsersByTotalCost() UserList { 120 | type tempUser struct { 121 | name string 122 | totalCost float64 123 | detailedCosts map[string]float64 124 | } 125 | userMap := make(map[string]*tempUser) 126 | // Go through all ReportItems 127 | for _, item := range r.Items { 128 | // Group by AccountId 129 | if user, ok := userMap[item.Owner]; ok { 130 | user.totalCost += item.Cost 131 | // Group by Description 132 | if cost, ok := user.detailedCosts[item.Description]; ok { 133 | user.detailedCosts[item.Description] = cost + item.Cost 134 | } else { 135 | user.detailedCosts[item.Description] = item.Cost 136 | } 137 | } else { 138 | costs := make(map[string]float64) 139 | costs[item.Description] = item.Cost 140 | userMap[item.Owner] = &tempUser{item.Owner, item.Cost, costs} 141 | } 142 | } 143 | 144 | userList := make(UserList, 0, len(userMap)) 145 | for _, user := range userMap { 146 | // omit users with low TotalCost 147 | if user.totalCost < MinimumTotalCost { 148 | continue 149 | } 150 | // convert detailedCosts into sorted CostLists 151 | detailedCostList := convertCostMapToSortedList(user.detailedCosts) 152 | // add generated User to userList 153 | userList = append(userList, User{user.name, user.totalCost, detailedCostList}) 154 | } 155 | 156 | sort.Sort(sort.Reverse(userList)) 157 | return userList 158 | } 159 | 160 | // SortedTagsByTotalCost returns a sorted list of grouped sort tag values, 161 | // sorted by their total cost. 162 | func (r *Report) SortedTagsByTotalCost() UserList { 163 | type tempTag struct { 164 | name string 165 | totalCost float64 166 | detailedCosts map[string]float64 167 | } 168 | tagMap := make(map[string]*tempTag) 169 | // Iterate through all report items 170 | for _, item := range r.Items { 171 | // Group by sort tag value 172 | if tag, ok := tagMap[item.sortTagValue]; ok { 173 | tag.totalCost += item.Cost 174 | // Group by Description 175 | if cost, ok := tag.detailedCosts[item.Description]; ok { 176 | tag.detailedCosts[item.Description] = cost + item.Cost 177 | } else { 178 | tag.detailedCosts[item.Description] = item.Cost 179 | } 180 | } else { 181 | costs := make(map[string]float64) 182 | costs[item.Description] = item.Cost 183 | tagMap[item.sortTagValue] = &tempTag{item.sortTagValue, item.Cost, costs} 184 | } 185 | } 186 | 187 | tagList := make(UserList, 0, len(tagMap)) 188 | for _, tag := range tagMap { 189 | // Omit tags with low total cost 190 | if tag.totalCost < MinimumTotalCost { 191 | continue 192 | } 193 | 194 | // Convert detailed costs into sorted cost lists 195 | detailedCostList := convertCostMapToSortedList(tag.detailedCosts) 196 | // Add generated tag to tag list 197 | tagList = append(tagList, User{tag.name, tag.totalCost, detailedCostList}) 198 | } 199 | 200 | sort.Sort(sort.Reverse(tagList)) 201 | return tagList 202 | } 203 | 204 | // FormatReport returns a simple version of the Month-to-date billing report. It 205 | // takes a mapping form account/project ID to employee username in order to 206 | // more easily distinguish the owner of a cost. 207 | func (r *Report) FormatReport(accountToUserMapping map[string]string, sortedByTags bool) string { 208 | b := new(bytes.Buffer) 209 | var sorted UserList 210 | if sortedByTags { 211 | sorted = r.SortedTagsByTotalCost() 212 | } else { 213 | sorted = r.SortedUsersByTotalCost() 214 | } 215 | 216 | fmt.Fprintln(b, "\n\nSummary:") 217 | fmt.Fprintln(b, "Name | Cost ($)") 218 | fmt.Fprintln(b, "----------------------------") 219 | for _, user := range sorted { 220 | name := user.Name 221 | if realName, exist := accountToUserMapping[name]; exist { 222 | name = realName 223 | } else { 224 | // Assume this is a support cost 225 | if name == "" { 226 | if sortedByTags { 227 | name = "" 228 | } else { 229 | name = "Support" 230 | } 231 | } 232 | } 233 | fmt.Fprintf(b, "%-12s | %8.2f\n", name, user.TotalCost) 234 | } 235 | 236 | fmt.Fprintf(b, "\nDetails:") 237 | for _, user := range sorted { 238 | name := user.Name 239 | if realName, exist := accountToUserMapping[name]; exist { 240 | name = realName 241 | } else { 242 | // Assume this is a support cost 243 | if name == "" { 244 | if sortedByTags { 245 | name = "" 246 | } else { 247 | name = "support" 248 | } 249 | } 250 | } 251 | fmt.Fprintf(b, "\n%s's costs:\n", name) 252 | fmt.Fprintln(b, "Cost ($) | Description") 253 | fmt.Fprintln(b, "---------------------------") 254 | for _, cost := range user.DetailedCosts { 255 | fmt.Fprintf(b, "%-8.2f | %s\n", cost.Cost, cost.Description) 256 | } 257 | } 258 | return b.String() 259 | } 260 | 261 | // GenerateReport generates a Month-to-date billing report for the current month 262 | func GenerateReport(reporter Reporter) Report { 263 | today := time.Now() 264 | start := time.Date(today.Year(), today.Month(), 1, 0, 0, 0, 0, time.Local) 265 | return reporter.GenerateReport(start) 266 | } 267 | 268 | func convertCostMapToSortedList(costMap map[string]float64) CostList { 269 | costList := make(CostList, 0, len(costMap)) 270 | for desc, cost := range costMap { 271 | if cost > MinimumCost { 272 | costList = append(costList, DetailedCost{Description: desc, Cost: cost}) 273 | } 274 | } 275 | sort.Sort(sort.Reverse(costList)) 276 | return costList 277 | } 278 | 279 | func updateCsvHeaders(record []string) map[string]int { 280 | csvHeaders := make(map[string]int) 281 | for i, column := range record { 282 | csvHeaders[column] = i 283 | } 284 | return csvHeaders 285 | } 286 | -------------------------------------------------------------------------------- /cloud/billing/gcp.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package billing 5 | 6 | import ( 7 | "context" 8 | "encoding/csv" 9 | "fmt" 10 | "io" 11 | "log" 12 | "os" 13 | "strconv" 14 | "time" 15 | 16 | "github.com/cloudtools/cloudsweeper/cloud" 17 | 18 | "cloud.google.com/go/storage" 19 | "google.golang.org/api/option" 20 | ) 21 | 22 | const ( 23 | gcpCSVNameFormat = "%s-%d-%02d-%02d.csv" 24 | ) 25 | 26 | type gcpReporter struct { 27 | csp cloud.CSP 28 | bucket string 29 | csvNamePrefix string 30 | } 31 | 32 | func (r *gcpReporter) GenerateReport(start time.Time) Report { 33 | report := Report{} 34 | report.CSP = r.csp 35 | 36 | ctx := context.Background() 37 | credsFilePath, exist := os.LookupEnv(cloud.GcpCredentialsFileKey) 38 | if !exist { 39 | log.Fatalln("No GCP credentials specified!") 40 | } 41 | if _, err := os.Stat(credsFilePath); os.IsNotExist(err) { 42 | log.Fatalln(credsFilePath, "is not a file!") 43 | } 44 | opt := option.WithServiceAccountFile(credsFilePath) 45 | client, err := storage.NewClient(ctx, opt) 46 | if err != nil { 47 | log.Printf("Could not initialize storage service:\n%s\n", err) 48 | return report 49 | } 50 | 51 | for d := start; d.Month() == start.Month(); d = d.AddDate(0, 0, 1) { 52 | name := fmt.Sprintf(gcpCSVNameFormat, r.csvNamePrefix, start.Year(), start.Month(), d.Day()) 53 | log.Println("Getting", name) 54 | obj := client.Bucket(r.bucket).Object(name) 55 | if err := processObjectHandle(ctx, obj, &report, true); err != nil { 56 | log.Println(err, "- skipping...") 57 | break 58 | } 59 | } 60 | return report 61 | } 62 | 63 | func processObjectHandle(ctx context.Context, obj *storage.ObjectHandle, report *Report, allowFailed bool) error { 64 | reader, err := obj.NewReader(ctx) 65 | if err != nil { 66 | return err 67 | } 68 | defer reader.Close() 69 | csvFile := csv.NewReader(reader) 70 | i := 0 71 | csvHeaders := make(map[string]int) 72 | for { 73 | record, err := csvFile.Read() 74 | if err == io.EOF { 75 | return nil 76 | } 77 | if err != nil { 78 | if allowFailed { 79 | log.Printf("Failed reading line %d, continuing...\n%s", i, err) 80 | } else { 81 | return err 82 | } 83 | } 84 | if i == 0 { 85 | csvHeaders = updateCsvHeaders(record) 86 | i++ 87 | continue 88 | } 89 | 90 | reportItem := ReportItem{} 91 | reportItem.Owner = record[csvHeaders["Project ID"]] 92 | reportItem.Description = record[csvHeaders["Description"]] 93 | cost := record[csvHeaders["Cost"]] 94 | costNumber, err := strconv.ParseFloat(cost, 64) 95 | if err != nil { 96 | if allowFailed { 97 | log.Println("Could not convert cost to float:", cost) 98 | } else { 99 | return err 100 | } 101 | } 102 | reportItem.Cost = costNumber 103 | report.Items = append(report.Items, reportItem) 104 | i++ 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /cloud/billing/pricing.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package billing 5 | 6 | import ( 7 | "encoding/json" 8 | "fmt" 9 | "log" 10 | "strconv" 11 | 12 | "github.com/aws/aws-sdk-go/private/protocol" 13 | 14 | "github.com/aws/aws-sdk-go/aws" 15 | "github.com/aws/aws-sdk-go/aws/credentials/stscreds" 16 | "github.com/aws/aws-sdk-go/aws/session" 17 | "github.com/aws/aws-sdk-go/service/pricing" 18 | "github.com/cloudtools/cloudsweeper/cloud" 19 | ) 20 | 21 | const ( 22 | gcpBucketPerGBMonth = 0.026 23 | 24 | assumeRoleARNTemplate = "arn:aws:iam::%s:role/Cloudsweeper" 25 | ) 26 | 27 | type instanceKeyPair struct { 28 | Region, InstanceType string 29 | } 30 | 31 | type priceMap map[instanceKeyPair]float64 32 | 33 | var ( 34 | awsPrices priceMap 35 | ) 36 | 37 | var generalInstanceFilters = []*pricing.Filter{ 38 | { 39 | Field: aws.String("operatingSystem"), 40 | Type: aws.String("TERM_MATCH"), 41 | Value: aws.String("Linux"), 42 | }, 43 | { 44 | Field: aws.String("operation"), 45 | Type: aws.String("TERM_MATCH"), 46 | Value: aws.String("RunInstances"), 47 | }, 48 | { 49 | Field: aws.String("capacitystatus"), 50 | Type: aws.String("TERM_MATCH"), 51 | Value: aws.String("Used"), 52 | }, 53 | { 54 | Field: aws.String("tenancy"), 55 | Type: aws.String("TERM_MATCH"), 56 | Value: aws.String("Shared"), 57 | }, 58 | } 59 | 60 | var awsRegionIDToNameMap = map[string]string{ 61 | "us-east-2": "US East (Ohio)", 62 | "us-east-1": "US East (N. Virginia)", 63 | "us-west-1": "US West (N. California)", 64 | "us-west-2": "US West (Oregon)", 65 | "ap-northeast-1": "Asia Pacific (Tokyo)", 66 | "ap-northeast-2": "Asia Pacific (Seoul)", 67 | "ap-northeast-3": "Asia Pacific (Osaka-Local)", 68 | "ap-south-1": "Asia Pacific (Mumbai)", 69 | "ap-southeast-1": "Asia Pacific (Singapore)", 70 | "ap-southeast-2": "Asia Pacific (Sydney)", 71 | "ca-central-1": "Canada (Central)", 72 | "cn-north-1": "China (Beijing)", 73 | "cn-northwest-1": "China (Ningxia)", 74 | "eu-central-1": "EU (Frankfurt)", 75 | "eu-west-1": "EU (Ireland)", 76 | "eu-west-2": "EU (London)", 77 | "eu-west-3": "EU (Paris)", 78 | "eu-north-1": "EU (Stockholm)", 79 | "sa-east-1": "South America (Sao Paulo)", 80 | "us-gov-east-1": "AWS GovCloud (US-East)", 81 | "us-gov-west-1": "AWS GovCloud (US-West)", 82 | } 83 | 84 | var awsS3StorageCostMap = map[string]float64{ 85 | "StandardStorage": 0.023, 86 | "IntelligentTieringFAStorage": 0.023, 87 | "IntelligentTieringIAStorage": 0.0125, 88 | "StandardIAStorage": 0.0125, 89 | "OneZoneIAStorage": 0.01, 90 | "ReducedRedundancyStorage": 0.023, // TODO: double check this 91 | "GlacierStorage": 0.004, 92 | } 93 | 94 | // Storage cost per GB per day 95 | var awsStorageCostMap = map[string]float64{ 96 | "standard": 0.05 / 30.0, 97 | "gp2": 0.1 / 30.0, 98 | "io1": 0.125 / 30.0, 99 | "st1": 0.045 / 30.0, 100 | "sc1": 0.025 / 30.0, 101 | "snapshot": 0.05 / 30.0, 102 | } 103 | 104 | // Storage cost per GB per day 105 | var gcpStorageCostGBDayMap = map[string]float64{ 106 | "pd-ssd": 0.170 / 30.0, 107 | "pd-standard": 0.040 / 30.0, 108 | "snapshot": 0.026 / 30.0, 109 | } 110 | 111 | var gcpInstanceCostPerHourMap = map[string]float64{ 112 | "n1-standard-1": 0.0475, 113 | "n1-standard-2": 0.0950, 114 | "n1-standard-4": 0.1900, 115 | "n1-standard-8": 0.3800, 116 | "n1-standard-16": 0.7600, 117 | "n1-standard-32": 1.5200, 118 | "n1-standard-64": 3.0400, 119 | "n1-standard-96": 4.5600, 120 | 121 | "n1-highmem-2": 0.1184, 122 | "n1-highmem-4": 0.2368, 123 | "n1-highmem-8": 0.4736, 124 | "n1-highmem-16": 0.9472, 125 | "n1-highmem-32": 1.8944, 126 | "n1-highmem-64": 3.7888, 127 | "n1-highmem-96": 5.6832, 128 | 129 | "n1-highcpu-2": 0.0709, 130 | "n1-highcpu-4": 0.1418, 131 | "n1-highcpu-8": 0.2836, 132 | "n1-highcpu-16": 0.5672, 133 | "n1-highcpu-32": 1.1344, 134 | "n1-highcpu-64": 2.2688, 135 | "n1-highcpu-96": 3.4020, 136 | 137 | "f1-micro": 0.0076, 138 | "g1-small": 0.0257, 139 | 140 | "n1-megamem-96": 10.6740, 141 | } 142 | 143 | // ResourceCostPerDay returns the daily cost of a resource in USD 144 | func ResourceCostPerDay(resource cloud.Resource) float64 { 145 | if inst, ok := resource.(cloud.Instance); ok { 146 | return InstancePricePerHour(inst) * 24.0 147 | } else if vol, ok := resource.(cloud.Volume); ok { 148 | return VolumeCostPerDay(vol) 149 | } else if img, ok := resource.(cloud.Image); ok { 150 | return ImageCostPerDay(img) 151 | } else if snap, ok := resource.(cloud.Snapshot); ok { 152 | return SnapshotCostPerDay(snap) 153 | } else { 154 | log.Println("Resource was neither instance, volume, image or snapshot") 155 | return 0.0 156 | } 157 | } 158 | 159 | // VolumeCostPerDay returns the daily cost in USD for a 160 | // certain volume 161 | func VolumeCostPerDay(volume cloud.Volume) float64 { 162 | if volume.CSP() == cloud.AWS { 163 | price, ok := awsStorageCostMap[volume.VolumeType()] 164 | if !ok { 165 | log.Fatalf("Could not find price for %s in AWS", volume.VolumeType()) 166 | return 0.0 167 | } 168 | return price * float64(volume.SizeGB()) 169 | } else if volume.CSP() == cloud.GCP { 170 | price, ok := gcpStorageCostGBDayMap[volume.VolumeType()] 171 | if !ok { 172 | log.Fatalf("Could not find price for %s in GCP", volume.VolumeType()) 173 | return 0.0 174 | } 175 | return price * float64(volume.SizeGB()) 176 | } 177 | log.Panicln("Unsupported CSP:", volume.CSP()) 178 | return 0.0 179 | } 180 | 181 | // SnapshotCostPerDay returns the daily cost in USD for a 182 | // certain snapshot 183 | func SnapshotCostPerDay(snapshot cloud.Snapshot) float64 { 184 | if snapshot.CSP() == cloud.AWS { 185 | return awsStorageCostMap["snapshot"] * float64(snapshot.SizeGB()) 186 | } else if snapshot.CSP() == cloud.GCP { 187 | price := gcpStorageCostGBDayMap["snapshot"] 188 | return price * float64(snapshot.SizeGB()) 189 | } 190 | log.Panicln("Unsupported CSP:", snapshot.CSP()) 191 | return 0.0 192 | } 193 | 194 | // ImageCostPerDay returns the daily cost in USD for a 195 | // certain image 196 | func ImageCostPerDay(image cloud.Image) float64 { 197 | if image.CSP() == cloud.AWS { 198 | return awsStorageCostMap["snapshot"] * float64(image.SizeGB()) 199 | } else if image.CSP() == cloud.GCP { 200 | price := gcpStorageCostGBDayMap["snapshot"] 201 | return price * float64(image.SizeGB()) 202 | } 203 | log.Panicln("Unsupported CSP:", image.CSP()) 204 | return 0.0 205 | } 206 | 207 | // InstancePricePerHour will return the hourly price in USD for a 208 | // specified instance. 209 | func InstancePricePerHour(instance cloud.Instance) float64 { 210 | if instance.CSP() == cloud.AWS { 211 | return awsInstancePricePerHour(instance) 212 | } else if instance.CSP() == cloud.GCP { 213 | price, ok := gcpInstanceCostPerHourMap[instance.InstanceType()] 214 | if !ok { 215 | log.Fatalf("Could not find price for %s in GCP", instance.InstanceType()) 216 | return 0.0 217 | } 218 | return price 219 | } 220 | log.Panicln("Unsupported CSP:", instance.CSP()) 221 | return 0.0 222 | } 223 | 224 | // BucketPricePerMonth will return the monthly price in USD for a 225 | // specified bucket. It will not take any account wide discounts 226 | // that might have been collected for using a certain amount of 227 | // storage every month. 228 | func BucketPricePerMonth(bucket cloud.Bucket) float64 { 229 | if bucket.CSP() == cloud.AWS { 230 | price := 0.0 231 | for storageType, size := range bucket.StorageTypeSizesGB() { 232 | price += awsS3StorageCostMap[storageType] * size 233 | } 234 | return price 235 | } else if bucket.CSP() == cloud.GCP { 236 | return gcpBucketPerGBMonth * bucket.TotalSizeGB() 237 | } 238 | log.Panicln("Unsupported CSP:", bucket.CSP()) 239 | return 0.0 240 | } 241 | 242 | // awsInstancePricePerHour will return the hourly price in USD for a 243 | // specified instance type in a specified AWS region. 244 | func awsInstancePricePerHour(instance cloud.Instance) float64 { 245 | if awsPrices == nil { 246 | awsPrices = make(priceMap) 247 | } 248 | // The price for this instance type/region has already been fetched before 249 | price, exist := awsPrices[instanceKeyPair{instance.Location(), instance.InstanceType()}] 250 | if exist { 251 | return price 252 | } 253 | 254 | sess := session.Must(session.NewSession()) 255 | creds := stscreds.NewCredentials(sess, fmt.Sprintf(assumeRoleARNTemplate, instance.Owner())) 256 | svc := pricing.New(sess, &aws.Config{ 257 | Credentials: creds, 258 | Region: aws.String("us-east-1"), // pricing API is only available here 259 | }) 260 | 261 | specificFilters := []*pricing.Filter{ 262 | { 263 | Field: aws.String("instanceType"), 264 | Type: aws.String("TERM_MATCH"), 265 | Value: aws.String(instance.InstanceType()), 266 | }, 267 | { 268 | Field: aws.String("location"), 269 | Type: aws.String("TERM_MATCH"), 270 | Value: aws.String(awsRegionIDToNameMap[instance.Location()]), 271 | }, 272 | } 273 | filters := append(generalInstanceFilters, specificFilters...) 274 | input := &pricing.GetProductsInput{ 275 | ServiceCode: aws.String("AmazonEC2"), 276 | Filters: filters, 277 | FormatVersion: aws.String("aws_v1"), 278 | } 279 | result, err := svc.GetProducts(input) 280 | if err != nil { 281 | log.Fatalln(err.Error()) 282 | } 283 | 284 | var listPrice rawAWSPrice 285 | rawListPriceJSON, err := protocol.EncodeJSONValue(result.PriceList[0], protocol.NoEscape) 286 | if err != nil { 287 | log.Fatalln(err.Error()) 288 | } 289 | err = json.Unmarshal([]byte(rawListPriceJSON), &listPrice) 290 | if err != nil { 291 | log.Fatalln(err.Error()) 292 | } 293 | 294 | for _, term := range listPrice.Terms.OnDemand { 295 | for _, price := range term.PriceDimensions { 296 | key := instanceKeyPair{ 297 | Region: instance.Location(), 298 | InstanceType: instance.InstanceType(), 299 | } 300 | usd, err := strconv.ParseFloat(price.PricePerUnit.USD, 64) 301 | if err != nil { 302 | log.Fatalln("Could not convert price from AWS JSON", err) 303 | } 304 | if usd == 0.00 { 305 | log.Println("Price for", instance.InstanceType(), "in", instance.Location(), "is $0.00. Needs investigation!") 306 | } 307 | awsPrices[key] = usd 308 | continue 309 | } 310 | } 311 | 312 | price, exist = awsPrices[instanceKeyPair{instance.Location(), instance.InstanceType()}] 313 | if !exist { 314 | log.Fatalln("Could not fetch price for", instance.InstanceType(), "in", instance.Location()) 315 | } 316 | return price 317 | } 318 | 319 | // Helper structs for parsing the JSON from AWS 320 | type rawAWSPrice struct { 321 | Terms struct { 322 | OnDemand map[string]struct { 323 | PriceDimensions map[string]struct { 324 | PricePerUnit struct { 325 | USD string `json:"USD"` 326 | } `json:"pricePerUnit"` 327 | } `json:"priceDimensions"` 328 | } `json:"OnDemand"` 329 | } `json:"terms"` 330 | } 331 | -------------------------------------------------------------------------------- /cloud/bucket.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "log" 10 | "time" 11 | 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/credentials/stscreds" 14 | "github.com/aws/aws-sdk-go/aws/session" 15 | "github.com/aws/aws-sdk-go/service/s3" 16 | storage "google.golang.org/api/storage/v1" 17 | ) 18 | 19 | type baseBucket struct { 20 | baseResource 21 | lastModified time.Time 22 | objectCount int64 23 | totalSizeGB float64 24 | storageTypeSizesGB map[string]float64 25 | } 26 | 27 | func (b *baseBucket) LastModified() time.Time { 28 | return b.lastModified 29 | } 30 | 31 | func (b *baseBucket) ObjectCount() int64 { 32 | return b.objectCount 33 | } 34 | 35 | func (b *baseBucket) TotalSizeGB() float64 { 36 | return b.totalSizeGB 37 | } 38 | 39 | func (b *baseBucket) StorageTypeSizesGB() map[string]float64 { 40 | return b.storageTypeSizesGB 41 | } 42 | 43 | func cleanupBuckets(buckets []Bucket) error { 44 | resList := []Resource{} 45 | for i := range buckets { 46 | v, ok := buckets[i].(Resource) 47 | if !ok { 48 | return errors.New("Could not convert Bucket to Resource") 49 | } 50 | resList = append(resList, v) 51 | } 52 | return cleanupResources(resList) 53 | } 54 | 55 | // AWS 56 | 57 | type awsBucket struct { 58 | baseBucket 59 | } 60 | 61 | func (b *awsBucket) Cleanup() error { 62 | log.Printf("Cleaning up bucket %s in %s", b.ID(), b.Owner()) 63 | sess := session.Must(session.NewSession()) 64 | creds := stscreds.NewCredentials(sess, fmt.Sprintf(assumeRoleARNTemplate, b.Owner())) 65 | s3Client := s3.New(sess, &aws.Config{ 66 | Credentials: creds, 67 | Region: aws.String(b.Location()), 68 | }) 69 | 70 | var internalErr error 71 | err := s3Client.ListObjectsV2Pages(&s3.ListObjectsV2Input{ 72 | Bucket: aws.String(b.ID()), 73 | }, func(output *s3.ListObjectsV2Output, lastPage bool) bool { 74 | input := &s3.DeleteObjectsInput{ 75 | Bucket: aws.String(b.ID()), 76 | } 77 | delete := &s3.Delete{ 78 | Objects: []*s3.ObjectIdentifier{}, 79 | } 80 | for i := range output.Contents { 81 | delete.Objects = append(delete.Objects, &s3.ObjectIdentifier{Key: output.Contents[i].Key}) 82 | } 83 | input.Delete = delete 84 | if len(delete.Objects) == 0 { 85 | // A request with an empty list of objects is not allowed 86 | return true 87 | } 88 | out, e := s3Client.DeleteObjects(input) 89 | if e != nil { 90 | internalErr = e 91 | return false 92 | } 93 | if len(out.Errors) > 0 { 94 | for i := range out.Errors { 95 | log.Printf("ERROR: Could not delete '%s': %s\n", *out.Errors[i].Key, *out.Errors[i].Message) 96 | } 97 | internalErr = errors.New("Failed to delete one or more objects") 98 | return false 99 | } 100 | return !lastPage 101 | }) 102 | if err != nil { 103 | return err 104 | } 105 | if internalErr != nil { 106 | return internalErr 107 | } 108 | 109 | input := &s3.DeleteBucketInput{ 110 | Bucket: aws.String(b.ID()), 111 | } 112 | _, err = s3Client.DeleteBucket(input) 113 | return err 114 | } 115 | 116 | func (b *awsBucket) SetTag(key, value string, overwrite bool) error { 117 | _, exist := b.Tags()[key] 118 | if exist && !overwrite { 119 | return fmt.Errorf("Key %s already exist on %s", key, b.ID()) 120 | } 121 | sess := session.Must(session.NewSession()) 122 | creds := stscreds.NewCredentials(sess, fmt.Sprintf(assumeRoleARNTemplate, b.Owner())) 123 | s3Client := s3.New(sess, &aws.Config{ 124 | Credentials: creds, 125 | Region: aws.String(b.Location()), 126 | }) 127 | tagging := &s3.Tagging{ 128 | TagSet: []*s3.Tag{&s3.Tag{ 129 | Key: aws.String(key), 130 | Value: aws.String(value), 131 | }}, 132 | } 133 | input := &s3.PutBucketTaggingInput{ 134 | Bucket: aws.String(b.ID()), 135 | Tagging: tagging, 136 | } 137 | _, err := s3Client.PutBucketTagging(input) 138 | return err 139 | } 140 | 141 | // RemoveTag removes the specified tag from the bucket by first deleting all tags 142 | // and then adding back all the other tags. Note that this is potentially unsafe if 143 | // the program crashes in the middle of the function. Unfortunately there doesn't seem 144 | // to be an API call for removing a specific tag from a bucket... 145 | func (b *awsBucket) RemoveTag(tagToRemove string) error { 146 | sess := session.Must(session.NewSession()) 147 | creds := stscreds.NewCredentials(sess, fmt.Sprintf(assumeRoleARNTemplate, b.Owner())) 148 | s3Client := s3.New(sess, &aws.Config{ 149 | Credentials: creds, 150 | Region: aws.String(b.Location()), 151 | }) 152 | _, err := s3Client.DeleteBucketTagging(&s3.DeleteBucketTaggingInput{ 153 | Bucket: aws.String(b.ID()), 154 | }) 155 | if err != nil { 156 | return err 157 | } 158 | tagging := &s3.Tagging{ 159 | TagSet: []*s3.Tag{}, 160 | } 161 | for k, v := range b.Tags() { 162 | if k == tagToRemove { 163 | continue 164 | } 165 | tagging.TagSet = append(tagging.TagSet, &s3.Tag{ 166 | Key: aws.String(k), 167 | Value: aws.String(v), 168 | }) 169 | } 170 | input := &s3.PutBucketTaggingInput{ 171 | Bucket: aws.String(b.ID()), 172 | Tagging: tagging, 173 | } 174 | _, err = s3Client.PutBucketTagging(input) 175 | return err 176 | } 177 | 178 | // GCP 179 | 180 | type gcpBucket struct { 181 | baseBucket 182 | storage *storage.Service 183 | } 184 | 185 | func (b *gcpBucket) Cleanup() error { 186 | log.Printf("Cleaning up bucket %s in %s", b.ID(), b.Owner()) 187 | // TODO: Currently only works if bucket is empty, cleanup 188 | // the objects in the bucket too 189 | return b.storage.Buckets.Delete(b.ID()).Do() 190 | } 191 | 192 | func (b *gcpBucket) SetTag(key, value string, overwrite bool) error { 193 | log.Println("Bucket tagging not supported on GCP") 194 | return nil 195 | } 196 | 197 | func (b *gcpBucket) RemoveTag(key string) error { 198 | log.Println("Bucket tagging not supported on GCP") 199 | return nil 200 | } 201 | -------------------------------------------------------------------------------- /cloud/cloud.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "context" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "net/http" 12 | "os" 13 | "time" 14 | 15 | oauth2 "golang.org/x/oauth2/google" 16 | compute "google.golang.org/api/compute/v1" 17 | storage "google.golang.org/api/storage/v1" 18 | ) 19 | 20 | const ( 21 | // GcpCredentialsFileKey is the Env variable to store path 22 | // to service accounts credentials JSON file 23 | GcpCredentialsFileKey = "GOOGLE_APPLICATION_CREDENTIALS" 24 | 25 | scopeGCPCompute = "https://www.googleapis.com/auth/compute" 26 | scopeGCPStorage = "https://www.googleapis.com/auth/devstorage.read_write" 27 | ) 28 | 29 | // ResourceManager is used to manage the different resources on 30 | // a CSP. It can be used to get e.g. all instances for all accounts 31 | // in AWS. 32 | type ResourceManager interface { 33 | // Owners return a list of all owners the manager handle 34 | Owners() []string 35 | // BucketsPerAccount returns a mapping from account/project to 36 | // its associated buckets 37 | BucketsPerAccount() map[string][]Bucket 38 | // InstancesPerAccount returns a mapping from account/project 39 | // to its associated instances 40 | InstancesPerAccount() map[string][]Instance 41 | // ImagesPerAccount returns a mapping from account/project 42 | // to its associated images 43 | ImagesPerAccount() map[string][]Image 44 | // VolumesPerAccount returns a mapping from account/project 45 | // to its associated volumes 46 | VolumesPerAccount() map[string][]Volume 47 | // SnapshotsPerAccount returns a mapping from account/project 48 | // to its associated snaphots 49 | SnapshotsPerAccount() map[string][]Snapshot 50 | // AllResourcesPerAccount will return a mapping from account/project 51 | // to all of the resources associated with that account/project 52 | AllResourcesPerAccount() map[string]*ResourceCollection 53 | // CleanupInstances termiantes a list of instances, which is faster 54 | // than calling Cleanup() on every individual instance 55 | CleanupInstances([]Instance) error 56 | // CleanupImages de-registers a list of images 57 | CleanupImages([]Image) error 58 | // CleanupVolumes deletes a list of volumes 59 | CleanupVolumes([]Volume) error 60 | // CleanupSnapshots delete a list of snapshots 61 | CleanupSnapshots([]Snapshot) error 62 | // CleanupBuckets deletes the specified buckets 63 | CleanupBuckets([]Bucket) error 64 | } 65 | 66 | // Resource represents a generic resource in any CSP. It should be 67 | // concretizised further. 68 | type Resource interface { 69 | CSP() CSP 70 | Owner() string 71 | ID() string 72 | Tags() map[string]string 73 | Location() string 74 | Public() bool 75 | CreationTime() time.Time 76 | 77 | SetTag(key, value string, overwrite bool) error 78 | RemoveTag(key string) error 79 | Cleanup() error 80 | } 81 | 82 | // Instance composes the Resource interface, and descibes an instance 83 | // in any CSP. 84 | type Instance interface { 85 | Resource 86 | InstanceType() string 87 | } 88 | 89 | // Image composes the Resource interface, and descibe an image in 90 | // any CSP. Such as an AMI in AWS. 91 | type Image interface { 92 | Resource 93 | Name() string 94 | SizeGB() int64 95 | 96 | MakePrivate() error 97 | } 98 | 99 | // Volume composes the Resource interface, and describe a volume in 100 | // any CSP. 101 | type Volume interface { 102 | Resource 103 | SizeGB() int64 104 | Attached() bool 105 | Encrypted() bool 106 | VolumeType() string 107 | } 108 | 109 | // Snapshot composes the Resource interface, and describe a snapshot 110 | // in any CSP. 111 | type Snapshot interface { 112 | Resource 113 | Encrypted() bool 114 | InUse() bool 115 | SizeGB() int64 116 | } 117 | 118 | // Bucket represents a bucket in a CSP, such as an S3 bucket in AWS 119 | type Bucket interface { 120 | Resource 121 | LastModified() time.Time 122 | ObjectCount() int64 123 | TotalSizeGB() float64 124 | StorageTypeSizesGB() map[string]float64 125 | } 126 | 127 | // ResourceCollection encapsulates collections of multiple resources. Does not 128 | // include buckets. 129 | type ResourceCollection struct { 130 | Owner string 131 | Instances []Instance 132 | Images []Image 133 | Volumes []Volume 134 | Snapshots []Snapshot 135 | } 136 | 137 | // AllResourceCollection encapsulates collections of all resources, 138 | // including buckets 139 | type AllResourceCollection struct { 140 | Owner string 141 | Instances []Instance 142 | Images []Image 143 | Volumes []Volume 144 | Snapshots []Snapshot 145 | Buckets []Bucket 146 | } 147 | 148 | // CSP represent a cloud service provider, such as AWS 149 | type CSP string 150 | 151 | const ( 152 | // AWS is AWS 153 | AWS CSP = "AWS" 154 | // GCP is Google Cloud Platform 155 | GCP CSP = "GCP" 156 | ) 157 | 158 | // NewManager will build a new resource manager for the specified CSP 159 | func NewManager(c CSP, accounts ...string) (ResourceManager, error) { 160 | switch c { 161 | case AWS: 162 | log.Println("Initializing AWS Resource Manager") 163 | manager := &awsResourceManager{ 164 | accounts: accounts, 165 | } 166 | return manager, nil 167 | case GCP: 168 | log.Println("Initializing GCP Resource Manager") 169 | client, err := getGCPHttpClient() 170 | if err != nil { 171 | return nil, err 172 | } 173 | computeService, err := compute.New(client) 174 | if err != nil { 175 | return nil, fmt.Errorf("Could not initialize compute service: %s", err) 176 | } 177 | storageService, err := storage.New(client) 178 | if err != nil { 179 | return nil, fmt.Errorf("Coult not initialize storage service: %s", err) 180 | } 181 | manager := &gcpResourceManager{ 182 | projects: accounts, 183 | compute: computeService, 184 | storage: storageService, 185 | } 186 | return manager, nil 187 | default: 188 | return nil, fmt.Errorf("Invalid CSP specified: %s", c) 189 | } 190 | } 191 | 192 | func getGCPHttpClient() (*http.Client, error) { 193 | credsFile, exist := os.LookupEnv(GcpCredentialsFileKey) 194 | if !exist { 195 | log.Println("No GCP credentials specified, using default") 196 | return oauth2.DefaultClient(context.Background(), scopeGCPCompute, scopeGCPStorage) 197 | } 198 | creds, err := ioutil.ReadFile(credsFile) 199 | if err != nil { 200 | return nil, fmt.Errorf("Could not read GCP credentials JSON: %s", err) 201 | } 202 | conf, err := oauth2.JWTConfigFromJSON(creds, scopeGCPCompute, scopeGCPStorage) 203 | if err != nil { 204 | return nil, fmt.Errorf("Could not get GCP credentials: %s", err) 205 | } 206 | return conf.Client(context.Background()), nil 207 | } 208 | -------------------------------------------------------------------------------- /cloud/filter/filter.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package filter 5 | 6 | import ( 7 | "github.com/cloudtools/cloudsweeper/cloud" 8 | ) 9 | 10 | // New will create a new resource filter ready to use 11 | func New() *ResourceFilter { 12 | return &ResourceFilter{ 13 | generalRules: []func(cloud.Resource) bool{}, 14 | instanceRules: []func(cloud.Instance) bool{}, 15 | volumeRules: []func(cloud.Volume) bool{}, 16 | imageRules: []func(cloud.Image) bool{}, 17 | snapshotRules: []func(cloud.Snapshot) bool{}, 18 | bucketRules: []func(cloud.Bucket) bool{}, 19 | 20 | OverrideWhitelist: false, 21 | } 22 | } 23 | 24 | // ResourceFilter is a dynamic filter that can have any amount 25 | // of rules. The rules are used to determine which resources 26 | // are kept when performing the filtering 27 | type ResourceFilter struct { 28 | generalRules []func(cloud.Resource) bool 29 | instanceRules []func(cloud.Instance) bool 30 | imageRules []func(cloud.Image) bool 31 | volumeRules []func(cloud.Volume) bool 32 | snapshotRules []func(cloud.Snapshot) bool 33 | bucketRules []func(cloud.Bucket) bool 34 | 35 | OverrideWhitelist bool 36 | } 37 | 38 | // AddGeneralRule adds a generic resource rule, which is not specific to 39 | // any particular type of resource. 40 | func (f *ResourceFilter) AddGeneralRule(rule func(cloud.Resource) bool) { 41 | f.generalRules = append(f.generalRules, rule) 42 | } 43 | 44 | // AddInstanceRule adds an instance specific rule to the filter chain 45 | func (f *ResourceFilter) AddInstanceRule(rule func(cloud.Instance) bool) { 46 | f.instanceRules = append(f.instanceRules, rule) 47 | } 48 | 49 | // AddImageRule adds an image specific rule to the filter chain 50 | func (f *ResourceFilter) AddImageRule(rule func(cloud.Image) bool) { 51 | f.imageRules = append(f.imageRules, rule) 52 | } 53 | 54 | // AddVolumeRule adds a volume specific rule to the filter chain 55 | func (f *ResourceFilter) AddVolumeRule(rule func(cloud.Volume) bool) { 56 | f.volumeRules = append(f.volumeRules, rule) 57 | } 58 | 59 | // AddSnapshotRule adds a snapshot specific rule to the filter chain 60 | func (f *ResourceFilter) AddSnapshotRule(rule func(cloud.Snapshot) bool) { 61 | f.snapshotRules = append(f.snapshotRules, rule) 62 | } 63 | 64 | // AddBucketRule adds a bucket specific rule to the filter chain 65 | func (f *ResourceFilter) AddBucketRule(rule func(cloud.Bucket) bool) { 66 | f.bucketRules = append(f.bucketRules, rule) 67 | } 68 | 69 | // Instances will filter the specified instances using the specified filters and 70 | // return the instances which match. A boolean OR is performed between every specified 71 | // filter. 72 | func Instances(instances []cloud.Instance, filters ...*ResourceFilter) []cloud.Instance { 73 | resultList := []cloud.Instance{} 74 | for i := range instances { 75 | if or(instances[i], filters) { 76 | resultList = append(resultList, instances[i]) 77 | } 78 | } 79 | return resultList 80 | } 81 | 82 | // Images will filter the specified images using the specified filters and 83 | // return the images which match. A boolean OR is performed between every specified 84 | // filter. 85 | func Images(images []cloud.Image, filters ...*ResourceFilter) []cloud.Image { 86 | resultList := []cloud.Image{} 87 | for i := range images { 88 | if or(images[i], filters) { 89 | resultList = append(resultList, images[i]) 90 | } 91 | } 92 | return resultList 93 | } 94 | 95 | // Volumes will filter the specified volumes using the specified filters and 96 | // return the volumes which match. A boolean OR is performed between every specified 97 | // filter. 98 | func Volumes(volumes []cloud.Volume, filters ...*ResourceFilter) []cloud.Volume { 99 | resultList := []cloud.Volume{} 100 | for i := range volumes { 101 | if or(volumes[i], filters) { 102 | resultList = append(resultList, volumes[i]) 103 | } 104 | } 105 | return resultList 106 | } 107 | 108 | // Snapshots will filter the specified snapshots using the specified filters and 109 | // return the snapshots which match. A boolean OR is performed between every specified 110 | // filter. 111 | func Snapshots(snapshots []cloud.Snapshot, filters ...*ResourceFilter) []cloud.Snapshot { 112 | resultList := []cloud.Snapshot{} 113 | for i := range snapshots { 114 | if or(snapshots[i], filters) { 115 | resultList = append(resultList, snapshots[i]) 116 | } 117 | } 118 | return resultList 119 | } 120 | 121 | // Buckets will filter the specified buckets using the specified filters and 122 | // return the buckets which match. A boolean OR is performed between every specified 123 | // filter. 124 | func Buckets(buckets []cloud.Bucket, filters ...*ResourceFilter) []cloud.Bucket { 125 | resultList := []cloud.Bucket{} 126 | for i := range buckets { 127 | if or(buckets[i], filters) { 128 | resultList = append(resultList, buckets[i]) 129 | } 130 | } 131 | return resultList 132 | } 133 | -------------------------------------------------------------------------------- /cloud/filter/filter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package filter 5 | 6 | import ( 7 | "fmt" 8 | "testing" 9 | "time" 10 | 11 | "github.com/cloudtools/cloudsweeper/cloud" 12 | ) 13 | 14 | func TestAddingFilters(t *testing.T) { 15 | fil := New() 16 | fil.AddGeneralRule(func(r cloud.Resource) bool { return true }) 17 | if len(fil.generalRules) != 1 { 18 | t.Error("General rule not added") 19 | } 20 | fil.AddInstanceRule(func(r cloud.Instance) bool { return true }) 21 | if len(fil.instanceRules) != 1 { 22 | t.Error("Instance rule not added") 23 | } 24 | fil.AddVolumeRule(func(r cloud.Volume) bool { return true }) 25 | if len(fil.volumeRules) != 1 { 26 | t.Error("Volume rule not added") 27 | } 28 | fil.AddImageRule(func(r cloud.Image) bool { return true }) 29 | if len(fil.imageRules) != 1 { 30 | t.Error("Image rule not added") 31 | } 32 | fil.AddSnapshotRule(func(r cloud.Snapshot) bool { return true }) 33 | if len(fil.snapshotRules) != 1 { 34 | t.Error("Snapshot rule not added") 35 | } 36 | fil.AddBucketRule(func(r cloud.Bucket) bool { return true }) 37 | if len(fil.bucketRules) != 1 { 38 | t.Error("Bucket rule not added") 39 | } 40 | } 41 | 42 | type testInstance struct { 43 | testResource 44 | instType string 45 | } 46 | 47 | func (i *testInstance) InstanceType() string { 48 | return i.instType 49 | } 50 | 51 | // Testing using a single filter and multiple filters for the same 52 | // resource type is identical for all instance types, so the tests 53 | // here only do cloud.Instance, but should cover all resource types. 54 | // This does not cover the case of mixing different resource types. 55 | func TestSingleInstanceFilter(t *testing.T) { 56 | inst1 := &testInstance{} 57 | inst1.creationTime = time.Now().AddDate(0, 0, -5) 58 | inst2 := &testInstance{} 59 | inst2.creationTime = time.Now() 60 | 61 | fil := New() 62 | fil.AddGeneralRule(OlderThanXDays(2)) 63 | 64 | filtered := Instances([]cloud.Instance{inst1, inst2}, fil) 65 | if len(filtered) != 1 { 66 | t.Error("Failed filtering") 67 | } 68 | } 69 | 70 | func TestMultipleInstanceFilter(t *testing.T) { 71 | inst1 := &testInstance{} 72 | inst1.creationTime = time.Now().AddDate(0, 0, -5) 73 | 74 | inst2 := &testInstance{} 75 | inst2.creationTime = time.Now() 76 | inst2.instType = "instance-type" 77 | 78 | fil1 := New() 79 | fil1.AddGeneralRule(OlderThanXDays(2)) 80 | 81 | fil2 := New() 82 | fil2.AddInstanceRule(func(i cloud.Instance) bool { 83 | return i.InstanceType() == "instance-type" 84 | }) 85 | 86 | filtered := Instances([]cloud.Instance{inst1, inst2}, fil1, fil2) 87 | if len(filtered) != 2 { 88 | for _, inst := range filtered { 89 | fmt.Println(inst) 90 | } 91 | t.Error("Failed filtering with multiple filters") 92 | } 93 | } 94 | 95 | type testImg struct { 96 | testResource 97 | } 98 | 99 | func (i *testImg) Name() string { return "test-img" } 100 | func (i *testImg) SizeGB() int64 { return 10 } 101 | func (i *testImg) MakePrivate() error { return nil } 102 | 103 | // This will test the filters being used when marking resources for 104 | // cleanup. These are: 105 | // - unattached volumes > 30 days old 106 | // - unused/unaccessed buckets > 6 months (182 days) 107 | // - non-whitelisted AMIs > 6 months 108 | // - non-whitelisted snapshots > 6 months 109 | // - non-whitelisted volumes > 6 months 110 | // - untagged resources > 30 days (this should take care of instances) 111 | func TestCleanupRulesFilter(t *testing.T) { 112 | 113 | // Setup the filters used 114 | untaggedFilter := New() 115 | untaggedFilter.AddGeneralRule(func(r cloud.Resource) bool { 116 | return len(r.Tags()) == 0 117 | }) 118 | untaggedFilter.AddGeneralRule(OlderThanXDays(30)) 119 | untaggedFilter.AddSnapshotRule(IsNotInUse()) 120 | untaggedFilter.AddGeneralRule(Negate(TaggedForCleanup())) 121 | 122 | oldFilter := New() 123 | oldFilter.AddGeneralRule(OlderThanXMonths(6)) 124 | // Don't cleanup resources tagged for release 125 | oldFilter.AddGeneralRule(Negate(HasTag("Release"))) 126 | oldFilter.AddSnapshotRule(IsNotInUse()) 127 | oldFilter.AddGeneralRule(Negate(TaggedForCleanup())) 128 | 129 | unattachedFilter := New() 130 | unattachedFilter.AddVolumeRule(IsUnattached()) 131 | unattachedFilter.AddGeneralRule(OlderThanXDays(30)) 132 | unattachedFilter.AddGeneralRule(Negate(HasTag("Release"))) 133 | unattachedFilter.AddGeneralRule(Negate(TaggedForCleanup())) 134 | 135 | bucketFilter := New() 136 | bucketFilter.AddBucketRule(NotModifiedInXDays(182)) 137 | bucketFilter.AddGeneralRule(OlderThanXDays(7)) 138 | bucketFilter.AddGeneralRule(Negate(HasTag("Release"))) 139 | bucketFilter.AddGeneralRule(Negate(TaggedForCleanup())) 140 | 141 | // Create some helper tag maps 142 | someTags := map[string]string{"test-key": "test-value"} 143 | whitelistTags := map[string]string{"cloudsweeper-whitelisted": ""} 144 | 145 | // Test instances 146 | // No 147 | inst1 := &testInstance{} 148 | inst1.creationTime = time.Now().AddDate(0, -3, 0) 149 | inst1.tags = someTags 150 | 151 | // Yes 152 | inst2 := &testInstance{} 153 | inst2.creationTime = time.Now().AddDate(0, -4, 0) 154 | 155 | // No 156 | inst3 := &testInstance{} 157 | inst3.creationTime = time.Now().AddDate(-5, 0, 0) 158 | inst3.tags = whitelistTags 159 | 160 | // No 161 | inst4 := &testInstance{} 162 | inst4.creationTime = time.Now() 163 | 164 | filInst := Instances([]cloud.Instance{inst1, inst2, inst3, inst4}, untaggedFilter) 165 | if len(filInst) != 1 { 166 | t.Error("Failed to filter instances") 167 | } 168 | 169 | // Test images 170 | // No 171 | img1 := &testImg{} 172 | img1.creationTime = time.Now() 173 | 174 | // No 175 | img2 := &testImg{} 176 | img2.creationTime = time.Now().AddDate(-3, 0, 0) 177 | img2.tags = whitelistTags 178 | 179 | // Yes 180 | img3 := &testImg{} 181 | img3.creationTime = time.Now().AddDate(0, -6, -3) 182 | 183 | // No 184 | img4 := &testImg{} 185 | img4.creationTime = time.Now().AddDate(0, 0, -3) 186 | img4.tags = someTags 187 | 188 | // No 189 | img5 := &testImg{} 190 | img5.creationTime = time.Now().AddDate(-3, 0, -5) 191 | img5.tags = map[string]string{"release": ""} 192 | 193 | // Yes 194 | img6 := &testImg{} 195 | img6.creationTime = time.Now().AddDate(0, 0, -32) 196 | 197 | filImg := Images([]cloud.Image{img1, img2, img3, img4, img5, img6}, oldFilter, untaggedFilter) 198 | if len(filImg) != 2 { 199 | t.Error("Failed to filter images") 200 | } 201 | 202 | // Test volumes 203 | // No 204 | vol1 := &testVolume{} 205 | vol1.creationTime = time.Now() 206 | vol1.attached = false 207 | 208 | // Yes 209 | vol2 := &testVolume{} 210 | vol2.attached = true 211 | vol2.creationTime = time.Now().AddDate(0, -6, -1) 212 | 213 | // No 214 | vol3 := &testVolume{} 215 | vol3.attached = true 216 | vol3.creationTime = time.Now().AddDate(0, 0, -32) 217 | vol3.tags = someTags 218 | 219 | // No 220 | vol4 := &testVolume{} 221 | vol4.creationTime = time.Now().AddDate(0, 0, -2) 222 | vol4.attached = false 223 | 224 | // No 225 | vol5 := &testVolume{} 226 | vol5.creationTime = time.Now().AddDate(0, -9, 0) 227 | vol5.attached = false 228 | vol5.tags = whitelistTags 229 | 230 | filVol := Volumes([]cloud.Volume{vol1, vol2, vol3, vol4, vol5}, oldFilter, unattachedFilter, untaggedFilter) 231 | if len(filVol) != 1 { 232 | t.Error("Failed to filter volumes") 233 | } 234 | 235 | // Test snapshots 236 | // No 237 | snap1 := &testSnap{} 238 | snap1.creationTime = time.Now().AddDate(0, 0, -3) 239 | 240 | // Yes 241 | snap2 := &testSnap{} 242 | snap2.creationTime = time.Now().AddDate(-4, 0, 0) 243 | snap2.tags = someTags 244 | 245 | // Yes 246 | snap3 := &testSnap{} 247 | snap3.creationTime = time.Now().AddDate(0, 0, -40) 248 | 249 | // No 250 | snap4 := &testSnap{} 251 | snap4.creationTime = time.Now().AddDate(0, -8, 4) 252 | snap4.tags = whitelistTags 253 | 254 | filSnaps := Snapshots([]cloud.Snapshot{snap1, snap2, snap3, snap4}, oldFilter, untaggedFilter) 255 | if len(filSnaps) != 2 { 256 | t.Error("Failed to filter snapshots") 257 | } 258 | 259 | // Test buckets 260 | // No 261 | buck1 := &testBucket{} 262 | buck1.creationTime = time.Now().AddDate(0, -8, 0) 263 | buck1.lastModified = time.Now().AddDate(0, 0, -2) 264 | 265 | // Yes 266 | buck2 := &testBucket{} 267 | buck2.creationTime = time.Now().AddDate(-7, 0, 0) 268 | buck2.lastModified = time.Now().AddDate(-2, 0, 0) 269 | buck2.tags = someTags 270 | 271 | // No 272 | buck3 := &testBucket{} 273 | buck3.creationTime = time.Now().AddDate(0, 0, -45) 274 | buck3.lastModified = time.Now() 275 | 276 | filBucks := Buckets([]cloud.Bucket{buck1, buck2, buck3}, bucketFilter) 277 | if len(filBucks) != 1 { 278 | t.Error("Failed to filter buckets") 279 | } 280 | } 281 | -------------------------------------------------------------------------------- /cloud/filter/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package filter 5 | 6 | import ( 7 | "strings" 8 | "time" 9 | 10 | "github.com/cloudtools/cloudsweeper/cloud" 11 | ) 12 | 13 | // IsWhitelisted checks if the given resource has a whitelisting tag 14 | func IsWhitelisted(resource cloud.Resource) bool { 15 | for key := range resource.Tags() { 16 | if strings.Replace(strings.ToLower(key), "_", "-", -1) == WhitelistTagKey { 17 | return true 18 | } 19 | } 20 | return false 21 | } 22 | 23 | func ParseFormat(image cloud.Image) (name string, creationTime time.Time) { 24 | nameParts := strings.Split(image.Name(), "-") 25 | if len(nameParts) < 2 { 26 | return "", time.Time{} 27 | } 28 | rawDate := nameParts[len(nameParts)-1] 29 | componentName := strings.Join(nameParts[:len(nameParts)-1], "-") 30 | const format = "20060102150405" 31 | if parsedDate, err := time.Parse(format, rawDate); err == nil { 32 | return componentName, parsedDate 33 | } 34 | return "", time.Time{} 35 | } 36 | 37 | func (f *ResourceFilter) includeResource(resource cloud.Resource) bool { 38 | for i := range f.generalRules { 39 | if !f.generalRules[i](resource) { 40 | return false 41 | } 42 | } 43 | return true 44 | } 45 | 46 | func (f *ResourceFilter) includeInstance(instance cloud.Instance) bool { 47 | if !f.includeResource(instance) { 48 | return false 49 | } 50 | for i := range f.instanceRules { 51 | if !f.instanceRules[i](instance) { 52 | return false 53 | } 54 | } 55 | return !IsWhitelisted(instance) || f.OverrideWhitelist 56 | } 57 | 58 | func (f *ResourceFilter) includeVolume(volume cloud.Volume) bool { 59 | if !f.includeResource(volume) { 60 | return false 61 | } 62 | for i := range f.volumeRules { 63 | if !f.volumeRules[i](volume) { 64 | return false 65 | } 66 | } 67 | return !IsWhitelisted(volume) || f.OverrideWhitelist 68 | } 69 | 70 | func (f *ResourceFilter) includeImage(image cloud.Image) bool { 71 | if !f.includeResource(image) { 72 | return false 73 | } 74 | for i := range f.imageRules { 75 | if !f.imageRules[i](image) { 76 | return false 77 | } 78 | } 79 | return !IsWhitelisted(image) || f.OverrideWhitelist 80 | } 81 | 82 | func (f *ResourceFilter) includeSnapshot(snapshot cloud.Snapshot) bool { 83 | if !f.includeResource(snapshot) { 84 | return false 85 | } 86 | for i := range f.snapshotRules { 87 | if !f.snapshotRules[i](snapshot) { 88 | return false 89 | } 90 | } 91 | return !IsWhitelisted(snapshot) || f.OverrideWhitelist 92 | } 93 | 94 | func (f *ResourceFilter) includeBucket(bucket cloud.Bucket) bool { 95 | if !f.includeResource(bucket) { 96 | return false 97 | } 98 | for i := range f.bucketRules { 99 | if !f.bucketRules[i](bucket) { 100 | return false 101 | } 102 | } 103 | return !IsWhitelisted(bucket) || f.OverrideWhitelist 104 | } 105 | 106 | func or(resource cloud.Resource, filters []*ResourceFilter) bool { 107 | if inst, ok := resource.(cloud.Instance); ok { 108 | for _, filter := range filters { 109 | if filter.includeInstance(inst) { 110 | return true 111 | } 112 | } 113 | return false 114 | } 115 | 116 | if img, ok := resource.(cloud.Image); ok { 117 | for _, filter := range filters { 118 | if filter.includeImage(img) { 119 | return true 120 | } 121 | } 122 | return false 123 | } 124 | 125 | if vol, ok := resource.(cloud.Volume); ok { 126 | for _, filter := range filters { 127 | if filter.includeVolume(vol) { 128 | return true 129 | } 130 | } 131 | return false 132 | } 133 | 134 | if snap, ok := resource.(cloud.Snapshot); ok { 135 | for _, filter := range filters { 136 | if filter.includeSnapshot(snap) { 137 | return true 138 | } 139 | } 140 | return false 141 | } 142 | 143 | if buck, ok := resource.(cloud.Bucket); ok { 144 | for _, filter := range filters { 145 | if filter.includeBucket(buck) { 146 | return true 147 | } 148 | } 149 | return false 150 | } 151 | 152 | return false 153 | } 154 | -------------------------------------------------------------------------------- /cloud/filter/rules.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package filter 5 | 6 | import ( 7 | "log" 8 | "strconv" 9 | "strings" 10 | "time" 11 | 12 | "github.com/cloudtools/cloudsweeper/cloud" 13 | ) 14 | 15 | const ( 16 | // WhitelistTagKey marks a resource to not matched by filter 17 | WhitelistTagKey = "cloudsweeper-whitelisted" 18 | // LifetimeTagKey marks a resource to be cleaned up after X days 19 | LifetimeTagKey = "cloudsweeper-lifetime" 20 | // ExpiryTagKey marks a resource to be cleaned up at the specified date (YYYY-MM-DD) 21 | ExpiryTagKey = "cloudsweeper-expiry" 22 | // DeleteTagKey marks a resource for deletion. This is used internally by houskeeper 23 | // to keep track of resources that should be cleaned up, but was not explicitly tagged 24 | // by the resource owner. 25 | DeleteTagKey = "cloudsweeper-delete-at" 26 | // ExpiryTagValueFormat is the format to use when setting expiry date 27 | ExpiryTagValueFormat = "2006-01-02" // Used to parse string 28 | ) 29 | 30 | // Below are general rules 31 | 32 | // Negate will simply negate another rule 33 | func Negate(funcToNegate func(r cloud.Resource) bool) func(cloud.Resource) bool { 34 | return func(r cloud.Resource) bool { 35 | return !funcToNegate(r) 36 | } 37 | } 38 | 39 | // TaggedForCleanup checks if resource is already tagged for cleanup 40 | func TaggedForCleanup() func(cloud.Resource) bool { 41 | return func(r cloud.Resource) bool { 42 | return HasTag(DeleteTagKey)(r) 43 | } 44 | } 45 | 46 | // OlderThanXHours returns a resource that is older than the 47 | // specified amount of hours. 48 | func OlderThanXHours(hours int) func(cloud.Resource) bool { 49 | return func(r cloud.Resource) bool { 50 | return time.Now().After(r.CreationTime().Add(time.Duration(hours) * time.Hour)) 51 | } 52 | } 53 | 54 | // OlderThanXDays return a resource that is older than the 55 | // specified amount of days 56 | func OlderThanXDays(days int) func(cloud.Resource) bool { 57 | return func(r cloud.Resource) bool { 58 | return time.Now().After(r.CreationTime().AddDate(0, 0, days)) 59 | } 60 | } 61 | 62 | // OlderThanXMonths return a resource that is older than the 63 | // specified amount of months 64 | func OlderThanXMonths(months int) func(cloud.Resource) bool { 65 | return func(r cloud.Resource) bool { 66 | return time.Now().After(r.CreationTime().AddDate(0, months, 0)) 67 | } 68 | } 69 | 70 | // OlderThanXYears return a resource that is older than the 71 | // specified amount of years 72 | func OlderThanXYears(years int) func(cloud.Resource) bool { 73 | return func(r cloud.Resource) bool { 74 | return time.Now().After(r.CreationTime().AddDate(years, 0, 0)) 75 | } 76 | } 77 | 78 | // NameContains checks if a resource's name contains a 79 | // specified substring 80 | func NameContains(contains string) func(cloud.Resource) bool { 81 | return func(r cloud.Resource) bool { 82 | name := "" 83 | if n, ok := r.Tags()["Name"]; ok { 84 | name = n 85 | } 86 | return strings.Contains(strings.ToLower(name), strings.ToLower(contains)) 87 | } 88 | } 89 | 90 | // IDMatches checks if a resource's ID matches any of the 91 | // specified IDs. 92 | func IDMatches(ids ...string) func(cloud.Resource) bool { 93 | return func(r cloud.Resource) bool { 94 | for i := range ids { 95 | if ids[i] == r.ID() { 96 | return true 97 | } 98 | } 99 | return false 100 | } 101 | } 102 | 103 | // HasTag checks if a resource have a specified tag or not 104 | func HasTag(tagKey string) func(cloud.Resource) bool { 105 | return func(r cloud.Resource) bool { 106 | for key := range r.Tags() { 107 | if strings.ToLower(key) == strings.ToLower(tagKey) { 108 | return true 109 | } 110 | } 111 | return false 112 | } 113 | } 114 | 115 | // IsUntaggedWithException checks if a resource is untagged with the exception of a specific tag 116 | func IsUntaggedWithException(exceptionTag string) func(cloud.Resource) bool { 117 | return func(r cloud.Resource) bool { 118 | if len(r.Tags()) == 0 { 119 | return true 120 | } else if len(r.Tags()) == 1 { 121 | return HasTag(exceptionTag)(r) 122 | } 123 | return false 124 | } 125 | } 126 | 127 | // IsPublic checks if a resource is public 128 | func IsPublic() func(cloud.Resource) bool { 129 | return func(r cloud.Resource) bool { 130 | return r.Public() 131 | } 132 | } 133 | 134 | // LifetimeExceeded check if a resource have the lifetime tag, 135 | // with the format "cloudsweeper-lifetime: days-X" (where X is the amount of 136 | // days to keep the resource). If the lifetime is passed, then 137 | // this resource should be included in the filter. 138 | func LifetimeExceeded() func(cloud.Resource) bool { 139 | return func(r cloud.Resource) bool { 140 | lifetime, hasLifetime := r.Tags()[LifetimeTagKey] 141 | if !hasLifetime { 142 | // If resource doesn't have the lifetime tag then don't include it 143 | return false 144 | } 145 | days := strings.Split(lifetime, "-") 146 | if len(days) != 2 { 147 | // Lifetime tag is not on the correct format 148 | log.Printf("%s have an incorrect lifetime tag: %s", r.ID(), lifetime) 149 | return false 150 | } 151 | numberOfDays, err := strconv.Atoi(days[1]) 152 | if err != nil { 153 | // Lifetime tag is not on the correct format 154 | log.Printf("%s have an incorrect lifetime tag: %s", r.ID(), lifetime) 155 | return false 156 | } 157 | expiery := r.CreationTime().Add(time.Hour * 24 * time.Duration(numberOfDays)) 158 | return time.Now().After(expiery) 159 | } 160 | } 161 | 162 | // ExpiryDatePassed checks is the expiry date for a resource has passed. The 163 | // expiry tag has the format "cloudsweeper-expiry: 2018-06-17". 164 | func ExpiryDatePassed() func(cloud.Resource) bool { 165 | return func(r cloud.Resource) bool { 166 | expiryVal, hasExpiry := r.Tags()[ExpiryTagKey] 167 | if !hasExpiry { 168 | // Don't include resource that doesn't have expiry tag 169 | return false 170 | } 171 | expiryDate, err := time.Parse(ExpiryTagValueFormat, expiryVal) 172 | if err != nil { 173 | log.Printf("%s has incorrect expiry tag:%s", r.ID(), expiryVal) 174 | return false 175 | } 176 | return time.Now().After(expiryDate) 177 | } 178 | } 179 | 180 | // DeleteWithinXHours checks if a resources is marked for deletion and if 181 | // it's about to be deleted within the specified amount of hours. This also 182 | // includes resources which deletion time is passed. 183 | func DeleteWithinXHours(hours int) func(cloud.Resource) bool { 184 | return func(r cloud.Resource) bool { 185 | deleteTimeString, hasDeletion := r.Tags()[DeleteTagKey] 186 | if !hasDeletion { 187 | return false 188 | } 189 | deleteTime, err := time.Parse(time.RFC3339, deleteTimeString) 190 | if err != nil { 191 | log.Printf("%s has malformed deletion tag: %s\n", r.ID(), deleteTimeString) 192 | return false 193 | } 194 | within := deleteTime.Add(-(time.Duration(hours) * time.Hour)) 195 | return time.Now().After(within) 196 | } 197 | } 198 | 199 | // DeleteAtPassed checks is the delete-at time for a resource has passed. The 200 | // delete tag has the format "cloudsweeper-delete-at: 2018-01-25T16:51:39-08:00". 201 | func DeleteAtPassed() func(cloud.Resource) bool { 202 | return func(r cloud.Resource) bool { 203 | deleteAt, exist := r.Tags()[DeleteTagKey] 204 | if !exist { 205 | return false 206 | } 207 | deleteAtTime, err := time.Parse(time.RFC3339, deleteAt) 208 | if err != nil { 209 | log.Printf("%s has malformed deletion tag: %s\n", r.ID(), deleteAt) 210 | return false 211 | } 212 | return time.Now().After(deleteAtTime) 213 | } 214 | } 215 | 216 | // Below are volume rules 217 | 218 | // IsUnattached checks if volume is not attached to an instance 219 | func IsUnattached() func(cloud.Volume) bool { 220 | return func(v cloud.Volume) bool { 221 | return !v.Attached() 222 | } 223 | } 224 | 225 | // Below are snapshot rules 226 | 227 | // IsInUse checks if the snapshot is currently being used by an AMI 228 | func IsInUse() func(cloud.Snapshot) bool { 229 | return func(s cloud.Snapshot) bool { 230 | return s.InUse() 231 | } 232 | } 233 | 234 | // IsNotInUse is the opposite of IsInUse 235 | func IsNotInUse() func(cloud.Snapshot) bool { 236 | return func(s cloud.Snapshot) bool { 237 | return !(IsInUse())(s) 238 | } 239 | } 240 | 241 | // Below are image rules 242 | 243 | // Checks whether or not an image follows the - format 244 | func FollowsFormat() func(cloud.Image) bool { 245 | return func(s cloud.Image) bool { 246 | name, creationTime := ParseFormat(s) 247 | if (name != "" && creationTime != time.Time{}) { 248 | return true 249 | } 250 | return false 251 | } 252 | } 253 | 254 | // DoesNotFollowFormat is the opposite of FollowsFormat 255 | func DoesNotFollowFormat() func(cloud.Image) bool { 256 | return func(s cloud.Image) bool { 257 | return !(FollowsFormat())(s) 258 | } 259 | } 260 | 261 | // Below are bucket rules 262 | 263 | // NotModifiedInXDays returns bucket which have not had any modification 264 | // to them within X days. 265 | func NotModifiedInXDays(days int) func(cloud.Bucket) bool { 266 | return func(b cloud.Bucket) bool { 267 | return time.Now().After(b.LastModified().AddDate(0, 0, days)) 268 | } 269 | } 270 | -------------------------------------------------------------------------------- /cloud/filter/rules_test.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package filter 5 | 6 | import ( 7 | "testing" 8 | "time" 9 | 10 | "github.com/cloudtools/cloudsweeper/cloud" 11 | ) 12 | 13 | const ( 14 | testOwner = "475063612724" 15 | testID = "some-resource-id" 16 | testLocation = "us-west-2" 17 | testCSP = cloud.AWS 18 | testPublic = false 19 | 20 | testSize = 10 21 | testEncrypted = false 22 | testVolumeType = "volume-type" 23 | ) 24 | 25 | type testResource struct { 26 | creationTime time.Time 27 | tags map[string]string 28 | } 29 | 30 | func (r *testResource) CSP() cloud.CSP { return testCSP } 31 | func (r *testResource) Owner() string { return testOwner } 32 | func (r *testResource) ID() string { return testID } 33 | func (r *testResource) Tags() map[string]string { return r.tags } 34 | func (r *testResource) Location() string { return testLocation } 35 | func (r *testResource) Public() bool { return testPublic } 36 | func (r *testResource) CreationTime() time.Time { return r.creationTime } 37 | func (r *testResource) SetTag(key, value string, overwrite bool) error { return nil } 38 | func (r *testResource) RemoveTag(key string) error { return nil } 39 | func (r *testResource) Cleanup() error { return nil } 40 | 41 | func TestNegate(t *testing.T) { 42 | foo := &testResource{time.Now(), map[string]string{}} 43 | fun := Negate(func(r cloud.Resource) bool { 44 | return true 45 | }) 46 | res := fun(foo) 47 | if res != false { 48 | t.Error("Failed to negate, got true when expected false") 49 | } 50 | } 51 | 52 | func TestAlreadyTaggedForDelete(t *testing.T) { 53 | foo := &testResource{time.Now(), map[string]string{}} 54 | foo.tags = map[string]string{DeleteTagKey: time.Now().Format(time.RFC3339)} 55 | fun := TaggedForCleanup() 56 | res := fun(foo) 57 | if !res { 58 | t.Error("The resource should be tagged for cleanup") 59 | } 60 | } 61 | 62 | func TestOlderHours(t *testing.T) { 63 | oldTime := time.Now().Add(-(10 * time.Hour)) 64 | foo := &testResource{oldTime, map[string]string{}} 65 | 66 | if !OlderThanXHours(5)(foo) { 67 | t.Error("Resource is older than 5 hours") 68 | } 69 | 70 | foo.creationTime = time.Now() 71 | if OlderThanXHours(5)(foo) { 72 | t.Error("Resource is not older than 5 hours") 73 | } 74 | } 75 | 76 | func TestOlderDays(t *testing.T) { 77 | oldTime := time.Now().Add(-(100 * time.Hour)) 78 | foo := &testResource{oldTime, map[string]string{}} 79 | 80 | if !OlderThanXDays(2)(foo) { 81 | t.Error("Resource is older than 2 days") 82 | } 83 | 84 | foo.creationTime = time.Now() 85 | if OlderThanXDays(2)(foo) { 86 | t.Error("Resource is not older than 2 days") 87 | } 88 | } 89 | 90 | func TestOlderMonths(t *testing.T) { 91 | oldTime := time.Now().AddDate(0, -5, 0) 92 | foo := &testResource{oldTime, map[string]string{}} 93 | 94 | if !OlderThanXMonths(2)(foo) { 95 | t.Error("Resource is older than 2 months") 96 | } 97 | 98 | foo.creationTime = time.Now() 99 | 100 | if OlderThanXMonths(2)(foo) { 101 | t.Error("Resource is not older than 2 months") 102 | } 103 | } 104 | 105 | func TestOlderYears(t *testing.T) { 106 | oldTime := time.Now().AddDate(-10, 0, 0) 107 | foo := &testResource{oldTime, map[string]string{}} 108 | 109 | if !OlderThanXYears(4)(foo) { 110 | t.Error("Resource is older than 4 years") 111 | } 112 | 113 | foo.creationTime = time.Now() 114 | if OlderThanXYears(4)(foo) { 115 | t.Error("Resource is not older than 4 years") 116 | } 117 | } 118 | 119 | func TestNames(t *testing.T) { 120 | tags := make(map[string]string) 121 | 122 | tags["Name"] = "SomeCoolName" 123 | 124 | foo := &testResource{time.Now(), tags} 125 | 126 | if !NameContains("SomeCoolName")(foo) { 127 | t.Error("Resource should contain name") 128 | } 129 | 130 | if !NameContains("Cool")(foo) { 131 | t.Error("Resource should contain subset of name") 132 | } 133 | 134 | foo.tags = map[string]string{} 135 | if NameContains("SomeCoolName")(foo) { 136 | t.Error("Resource does not have name") 137 | } 138 | 139 | } 140 | 141 | func TestIDMatch(t *testing.T) { 142 | foo := &testResource{time.Now(), map[string]string{}} 143 | 144 | if !IDMatches(testID)(foo) { 145 | t.Error("Resource ID should match") 146 | } 147 | 148 | if IDMatches("not-a-good-id")(foo) { 149 | t.Error("Resource ID should not match") 150 | } 151 | } 152 | 153 | func TestHasTag(t *testing.T) { 154 | tags := make(map[string]string) 155 | tags["some-tag-key"] = "some-tag-value" 156 | 157 | foo := &testResource{time.Now(), tags} 158 | 159 | if !HasTag("some-tag-key")(foo) { 160 | t.Error("Resource should have tag") 161 | } 162 | 163 | if HasTag("some-tag")(foo) { 164 | t.Error("Resource does not have tag") 165 | } 166 | } 167 | 168 | func TestPublic(t *testing.T) { 169 | foo := &testResource{time.Now(), map[string]string{}} 170 | 171 | if IsPublic()(foo) != testPublic { 172 | t.Error("Resource public value wrong") 173 | } 174 | } 175 | 176 | func TestLifetimeExceeded(t *testing.T) { 177 | tags := make(map[string]string) 178 | 179 | foo := &testResource{time.Now(), tags} 180 | 181 | if LifetimeExceeded()(foo) { 182 | t.Error("Resource doesn't have tag") 183 | } 184 | 185 | tags[LifetimeTagKey] = "days-5" 186 | 187 | oldTime := time.Now().AddDate(0, 0, -6) 188 | 189 | foo.creationTime = oldTime 190 | foo.tags = tags 191 | 192 | if !LifetimeExceeded()(foo) { 193 | t.Error("Lifetime should be exceeded") 194 | } 195 | 196 | foo.tags[LifetimeTagKey] = "invalidtag" 197 | 198 | if LifetimeExceeded()(foo) { 199 | t.Error("Tag value is malformed") 200 | } 201 | 202 | foo.tags[LifetimeTagKey] = "days-five" 203 | 204 | if LifetimeExceeded()(foo) { 205 | t.Error("Tag value is malformed") 206 | } 207 | 208 | foo.tags[LifetimeTagKey] = "days-7" 209 | 210 | if LifetimeExceeded()(foo) { 211 | t.Error("Lifetime is not exceeded") 212 | } 213 | } 214 | 215 | func TestExpiryPassed(t *testing.T) { 216 | tags := make(map[string]string) 217 | 218 | foo := &testResource{time.Now(), tags} 219 | 220 | if ExpiryDatePassed()(foo) { 221 | t.Error("Resource have no expiry tag") 222 | } 223 | 224 | foo.tags[ExpiryTagKey] = time.Now().AddDate(0, 0, -5).Format("2006-01-02") 225 | 226 | if !ExpiryDatePassed()(foo) { 227 | t.Error("Expiry should have passed") 228 | } 229 | 230 | foo.tags[ExpiryTagKey] = "malformed-tag" 231 | 232 | if ExpiryDatePassed()(foo) { 233 | t.Error("Tag is malformed") 234 | } 235 | 236 | foo.tags[ExpiryTagKey] = time.Now().AddDate(0, 1, 0).Format("2006-01-02") 237 | if ExpiryDatePassed()(foo) { 238 | t.Error("Resource is not expired") 239 | } 240 | } 241 | 242 | func TestDeleteWithin(t *testing.T) { 243 | deleteTime := time.Now().AddDate(0, 0, 2).Format(time.RFC3339) 244 | tags := make(map[string]string) 245 | foo := &testResource{time.Now(), tags} 246 | 247 | if DeleteWithinXHours(72)(foo) { 248 | t.Error("Resource has no delete tag") 249 | } 250 | 251 | foo.tags[DeleteTagKey] = deleteTime 252 | 253 | if !DeleteWithinXHours(72)(foo) { 254 | t.Error("Should be deleted within 72 hours") 255 | } 256 | 257 | if DeleteWithinXHours(5)(foo) { 258 | t.Error("Should not be deleted within 5 hours") 259 | } 260 | 261 | foo.tags[DeleteTagKey] = "malformed" 262 | 263 | if DeleteWithinXHours(72)(foo) { 264 | t.Error("Tag is malformed") 265 | } 266 | } 267 | 268 | func TestDeletePassed(t *testing.T) { 269 | deleteTime := time.Now().AddDate(0, 0, -2).Format(time.RFC3339) 270 | tags := make(map[string]string) 271 | foo := &testResource{time.Now(), tags} 272 | 273 | if DeleteAtPassed()(foo) { 274 | t.Error("Resource has no delete tag") 275 | } 276 | 277 | foo.tags[DeleteTagKey] = deleteTime 278 | 279 | if !DeleteAtPassed()(foo) { 280 | t.Error("Delete time should be passed") 281 | } 282 | 283 | foo.tags[DeleteTagKey] = time.Now().AddDate(0, 0, 2).Format(time.RFC3339) 284 | 285 | if DeleteAtPassed()(foo) { 286 | t.Error("Delete time is not passed") 287 | } 288 | 289 | foo.tags[DeleteTagKey] = "malformed" 290 | 291 | if DeleteAtPassed()(foo) { 292 | t.Error("Malformed tag value") 293 | } 294 | } 295 | 296 | type testVolume struct { 297 | testResource 298 | attached bool 299 | } 300 | 301 | func (v *testVolume) SizeGB() int64 { return testSize } 302 | func (v *testVolume) Attached() bool { return v.attached } 303 | func (v *testVolume) Encrypted() bool { return testEncrypted } 304 | func (v *testVolume) VolumeType() string { return testVolumeType } 305 | 306 | func TestAttached(t *testing.T) { 307 | foo := &testVolume{ 308 | testResource{time.Now(), map[string]string{}}, 309 | false, 310 | } 311 | 312 | foo.attached = true 313 | 314 | if IsUnattached()(foo) { 315 | t.Error("Should be attached") 316 | } 317 | 318 | foo.attached = false 319 | 320 | if !IsUnattached()(foo) { 321 | t.Error("Should not be attached") 322 | } 323 | } 324 | 325 | type testBucket struct { 326 | testResource 327 | lastModified time.Time 328 | } 329 | 330 | func (b *testBucket) LastModified() time.Time { return b.lastModified } 331 | func (b *testBucket) ObjectCount() int64 { return 10 } 332 | func (b *testBucket) TotalSizeGB() float64 { return 5.13 } 333 | func (b *testBucket) StorageTypeSizesGB() map[string]float64 { return make(map[string]float64) } 334 | 335 | func TestNotModified(t *testing.T) { 336 | foo := &testBucket{ 337 | testResource{time.Now(), map[string]string{}}, 338 | time.Now(), 339 | } 340 | 341 | if NotModifiedInXDays(5)(foo) { 342 | t.Error("Has been modified within 5 days") 343 | } 344 | 345 | foo.lastModified = time.Now().AddDate(0, -5, 0) 346 | 347 | if !NotModifiedInXDays(5)(foo) { 348 | t.Error("Not modified within 5 days") 349 | } 350 | } 351 | 352 | type testSnap struct { 353 | testResource 354 | inUse bool 355 | } 356 | 357 | func (s *testSnap) Encrypted() bool { return false } 358 | func (s *testSnap) SizeGB() int64 { return 5 } 359 | func (s *testSnap) InUse() bool { return s.inUse } 360 | 361 | func TestInUse(t *testing.T) { 362 | foo := &testSnap{ 363 | testResource{time.Now(), map[string]string{}}, 364 | false, 365 | } 366 | 367 | if IsInUse()(foo) { 368 | t.Error("Snapshot is not in use") 369 | } 370 | 371 | foo.inUse = true 372 | 373 | if IsNotInUse()(foo) { 374 | t.Error("Snapshot is in use") 375 | } 376 | } 377 | -------------------------------------------------------------------------------- /cloud/gcp.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "log" 9 | "strings" 10 | "sync" 11 | "time" 12 | 13 | compute "google.golang.org/api/compute/v1" 14 | storage "google.golang.org/api/storage/v1" 15 | ) 16 | 17 | // Google Cloud API error codes can be found here: 18 | // https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto 19 | 20 | var ( 21 | // ErrPermissionDenied is returned if not enough permissions to perform action 22 | ErrPermissionDenied = errors.New("permission denied") 23 | ) 24 | 25 | // gcpResourceManager uses the Go API client for Google Cloud 26 | // https://github.com/google/google-api-go-client 27 | type gcpResourceManager struct { 28 | projects []string 29 | compute *compute.Service 30 | storage *storage.Service 31 | } 32 | 33 | func (m *gcpResourceManager) Owners() []string { 34 | return m.projects 35 | } 36 | 37 | func (m *gcpResourceManager) InstancesPerAccount() map[string][]Instance { 38 | log.Println("Getting instances in all projects") 39 | result := make(map[string][]Instance) 40 | var resultMutex sync.Mutex // Projects are processed in parallel 41 | m.forEachProject(func(project string) { 42 | instList := []Instance{} 43 | var listMutex sync.Mutex // Zones are proccessed in parallel 44 | m.forEachZone(project, func(zone string) { 45 | inst, err := m.getInstances(project, zone) 46 | if err != nil { 47 | log.Printf("Could not list instances in (%s, %s): %s", project, zone, err) 48 | if err == ErrPermissionDenied { 49 | log.Println(err) 50 | } else { 51 | // If it was an unknown error, abort 52 | log.Fatalln(err) 53 | } 54 | } else if len(inst) > 0 { 55 | listMutex.Lock() 56 | instList = append(instList, inst...) 57 | listMutex.Unlock() 58 | } 59 | }) 60 | resultMutex.Lock() 61 | result[project] = instList 62 | resultMutex.Unlock() 63 | }) 64 | return result 65 | } 66 | 67 | func (m *gcpResourceManager) ImagesPerAccount() map[string][]Image { 68 | log.Println("Getting images in all projects") 69 | result := make(map[string][]Image) 70 | var resultMutex sync.Mutex // Projects are processed in parallel 71 | m.forEachProject(func(project string) { 72 | images, err := m.getImages(project) 73 | if err != nil { 74 | log.Printf("Could not list images in %s: %s", project, err) 75 | if err == ErrPermissionDenied { 76 | log.Println(err) 77 | } else { 78 | // If it was an unknown error, abort 79 | log.Fatalln(err) 80 | } 81 | } else if len(images) > 0 { 82 | resultMutex.Lock() 83 | result[project] = images 84 | resultMutex.Unlock() 85 | } 86 | }) 87 | return result 88 | } 89 | 90 | func (m *gcpResourceManager) VolumesPerAccount() map[string][]Volume { 91 | log.Println("Getting volumes in all projects") 92 | result := make(map[string][]Volume) 93 | var resultMutex sync.Mutex // Projects are processed in parallel 94 | m.forEachProject(func(project string) { 95 | diskList := []Volume{} 96 | var listMutex sync.Mutex // Zones are proccessed in parallel 97 | m.forEachZone(project, func(zone string) { 98 | volumes, err := m.getVolumes(project, zone) 99 | if err != nil { 100 | log.Printf("Could not list disks in (%s, %s): %s", project, zone, err) 101 | if err == ErrPermissionDenied { 102 | log.Println(err) 103 | } else { 104 | // If it was an unknown error, abort 105 | log.Fatalln(err) 106 | } 107 | } else if len(volumes) > 0 { 108 | listMutex.Lock() 109 | diskList = append(diskList, volumes...) 110 | listMutex.Unlock() 111 | } 112 | }) 113 | resultMutex.Lock() 114 | result[project] = diskList 115 | resultMutex.Unlock() 116 | }) 117 | return result 118 | } 119 | 120 | func (m *gcpResourceManager) SnapshotsPerAccount() map[string][]Snapshot { 121 | log.Println("Getting snapshots in all projects") 122 | result := make(map[string][]Snapshot) 123 | var resultMutex sync.Mutex 124 | m.forEachProject(func(project string) { 125 | snapshots, err := m.getSnapshots(project) 126 | if err != nil { 127 | log.Printf("Could not list snapshots in %s: %s", project, err) 128 | if err == ErrPermissionDenied { 129 | log.Println(err) 130 | } else { 131 | // If it was an unknown error, abort 132 | log.Fatalln(err) 133 | } 134 | } else if len(snapshots) > 0 { 135 | resultMutex.Lock() 136 | result[project] = snapshots 137 | resultMutex.Unlock() 138 | } 139 | }) 140 | return result 141 | } 142 | 143 | func (m *gcpResourceManager) BucketsPerAccount() map[string][]Bucket { 144 | log.Println("Getting buckets in all projects") 145 | result := make(map[string][]Bucket) 146 | var resultMutex sync.Mutex 147 | m.forEachProject(func(project string) { 148 | buckets, err := m.getBuckets(project) 149 | if err != nil { 150 | log.Printf("Could not list buckets in %s: %s", project, err) 151 | if err == ErrPermissionDenied { 152 | log.Println(err) 153 | } else { 154 | // If it was an unknown error, abort 155 | log.Fatalln(err) 156 | } 157 | } else if len(buckets) > 0 { 158 | resultMutex.Lock() 159 | result[project] = buckets 160 | resultMutex.Unlock() 161 | } 162 | }) 163 | return result 164 | } 165 | 166 | func (m *gcpResourceManager) AllResourcesPerAccount() map[string]*ResourceCollection { 167 | log.Println("Getting all compute resources in all accounts") 168 | result := make(map[string]*ResourceCollection) 169 | var resultMutex sync.Mutex 170 | var wg sync.WaitGroup 171 | var instanceMap map[string][]Instance 172 | var imageMap map[string][]Image 173 | var volumeMap map[string][]Volume 174 | var snapMap map[string][]Snapshot 175 | wg.Add(4) 176 | go func() { 177 | instanceMap = m.InstancesPerAccount() 178 | wg.Done() 179 | }() 180 | go func() { 181 | imageMap = m.ImagesPerAccount() 182 | wg.Done() 183 | }() 184 | go func() { 185 | volumeMap = m.VolumesPerAccount() 186 | wg.Done() 187 | }() 188 | go func() { 189 | snapMap = m.SnapshotsPerAccount() 190 | wg.Done() 191 | }() 192 | wg.Wait() 193 | for _, project := range m.projects { 194 | collection := &ResourceCollection{ 195 | Owner: project, 196 | Instances: instanceMap[project], 197 | Images: imageMap[project], 198 | Volumes: volumeMap[project], 199 | Snapshots: snapMap[project], 200 | } 201 | resultMutex.Lock() 202 | result[project] = collection 203 | resultMutex.Unlock() 204 | } 205 | return result 206 | } 207 | 208 | func (m *gcpResourceManager) CleanupInstances(instances []Instance) error { 209 | return cleanupInstances(instances) 210 | } 211 | 212 | func (m *gcpResourceManager) CleanupImages(images []Image) error { 213 | return cleanupImages(images) 214 | } 215 | 216 | func (m *gcpResourceManager) CleanupVolumes(volumes []Volume) error { 217 | return cleanupVolumes(volumes) 218 | } 219 | 220 | func (m *gcpResourceManager) CleanupSnapshots(snapshots []Snapshot) error { 221 | return cleanupSnapshots(snapshots) 222 | } 223 | 224 | func (m *gcpResourceManager) CleanupBuckets(buckets []Bucket) error { 225 | return cleanupBuckets(buckets) 226 | } 227 | 228 | func (m *gcpResourceManager) forEachProject(f func(project string)) { 229 | var wg sync.WaitGroup 230 | wg.Add(len(m.projects)) 231 | for i := range m.projects { 232 | go func(i int) { 233 | log.Printf("Accessing project %s", m.projects[i]) 234 | f(m.projects[i]) 235 | wg.Done() 236 | }(i) 237 | } 238 | wg.Wait() 239 | } 240 | 241 | func (m *gcpResourceManager) forEachZone(project string, f func(zone string)) { 242 | zones, err := m.compute.Zones.List(project).Do() 243 | if err != nil { 244 | log.Printf("Could not list zones in %s. Err: %v", project, err) 245 | return 246 | } 247 | var wg sync.WaitGroup 248 | for _, z := range zones.Items { 249 | wg.Add(1) 250 | go func(z string) { 251 | f(z) 252 | wg.Done() 253 | }(z.Name) 254 | } 255 | wg.Wait() 256 | } 257 | 258 | func (m *gcpResourceManager) getInstances(project, zone string) ([]Instance, error) { 259 | instances, err := m.compute.Instances.List(project, zone).Do() 260 | if err != nil { 261 | if instances != nil && isGCPAccessDeniedError(instances.HTTPStatusCode) { 262 | return nil, ErrPermissionDenied 263 | } 264 | return nil, err 265 | } 266 | res := []Instance{} 267 | for _, i := range instances.Items { 268 | creationTime, err := time.Parse(time.RFC3339, i.CreationTimestamp) 269 | if err != nil { 270 | log.Printf("Could not parse timestamp of %s (in %s): %s", i.Name, project, err) 271 | // Set to Now so it doesn't incorrecntly get tagged for deletion 272 | creationTime = time.Now() 273 | } 274 | labels := i.Labels 275 | if labels == nil { 276 | labels = make(map[string]string) 277 | } 278 | res = append(res, &gcpInstance{baseInstance{ 279 | baseResource: baseResource{ 280 | csp: GCP, 281 | owner: project, 282 | id: i.Name, 283 | location: zone, 284 | public: true, 285 | tags: i.Labels, 286 | creationTime: creationTime, 287 | }, 288 | instanceType: parseGCPResourceURL(i.MachineType), 289 | }, 290 | m.compute, 291 | }) 292 | } 293 | return res, nil 294 | } 295 | 296 | func (m *gcpResourceManager) getImages(project string) ([]Image, error) { 297 | images, err := m.compute.Images.List(project).Do() 298 | if err != nil { 299 | if images != nil && isGCPAccessDeniedError(images.HTTPStatusCode) { 300 | return nil, ErrPermissionDenied 301 | } 302 | return nil, err 303 | } 304 | imgList := []Image{} 305 | for _, img := range images.Items { 306 | creationTime, err := time.Parse(time.RFC3339, img.CreationTimestamp) 307 | if err != nil { 308 | log.Printf("Could not parse timestamp of %s (in %s): %s", img.Name, project, err) 309 | // Set to Now so it doesn't incorrecntly get tagged for deletion 310 | creationTime = time.Now() 311 | } 312 | labels := img.Labels 313 | if labels == nil { 314 | labels = make(map[string]string) 315 | } 316 | imgList = append(imgList, &gcpImage{ 317 | baseImage: baseImage{ 318 | baseResource: baseResource{ 319 | csp: GCP, 320 | id: img.Name, 321 | owner: project, 322 | location: "", 323 | creationTime: creationTime, 324 | tags: labels, 325 | public: true, 326 | }, 327 | name: img.Name, 328 | sizeGB: img.DiskSizeGb, 329 | }, 330 | compute: m.compute, 331 | }) 332 | } 333 | return imgList, nil 334 | } 335 | 336 | func (m *gcpResourceManager) getVolumes(project, zone string) ([]Volume, error) { 337 | volumes, err := m.compute.Disks.List(project, zone).Do() 338 | if err != nil { 339 | if volumes != nil && isGCPAccessDeniedError(volumes.HTTPStatusCode) { 340 | return nil, ErrPermissionDenied 341 | } 342 | return nil, err 343 | } 344 | diskList := []Volume{} 345 | for _, disk := range volumes.Items { 346 | creationTime, err := time.Parse(time.RFC3339, disk.CreationTimestamp) 347 | if err != nil { 348 | log.Printf("Could not parse timestamp of %s (in %s): %s", disk.Name, project, err) 349 | // Set to Now so it doesn't incorrecntly get tagged for deletion 350 | creationTime = time.Now() 351 | } 352 | labels := disk.Labels 353 | if labels == nil { 354 | labels = make(map[string]string) 355 | } 356 | diskList = append(diskList, &gcpVolume{ 357 | baseVolume: baseVolume{ 358 | baseResource: baseResource{ 359 | csp: GCP, 360 | owner: project, 361 | id: disk.Name, 362 | location: zone, 363 | creationTime: creationTime, 364 | public: true, 365 | tags: labels, 366 | }, 367 | sizeGB: disk.SizeGb, 368 | encrypted: false, 369 | attached: disk.Users != nil && len(disk.Users) > 0, 370 | volumeType: parseGCPResourceURL(disk.Type), 371 | }, 372 | compute: m.compute, 373 | }) 374 | } 375 | return diskList, nil 376 | } 377 | 378 | func (m *gcpResourceManager) getSnapshots(project string) ([]Snapshot, error) { 379 | snapshots, err := m.compute.Snapshots.List(project).Do() 380 | if err != nil { 381 | if snapshots != nil && isGCPAccessDeniedError(snapshots.HTTPStatusCode) { 382 | return nil, ErrPermissionDenied 383 | } 384 | return nil, err 385 | } 386 | snapList := []Snapshot{} 387 | for _, snap := range snapshots.Items { 388 | creationTime, err := time.Parse(time.RFC3339, snap.CreationTimestamp) 389 | if err != nil { 390 | log.Printf("Could not parse timestamp of %s (in %s): %s", snap.Name, project, err) 391 | // Set to Now so it doesn't incorrecntly get tagged for deletion 392 | creationTime = time.Now() 393 | } 394 | labels := snap.Labels 395 | if labels == nil { 396 | labels = make(map[string]string) 397 | } 398 | snapList = append(snapList, &gcpSnapshot{ 399 | baseSnapshot: baseSnapshot{ 400 | baseResource: baseResource{ 401 | csp: GCP, 402 | id: snap.Name, 403 | owner: project, 404 | location: "", 405 | public: true, 406 | creationTime: creationTime, 407 | tags: labels, 408 | }, 409 | encrypted: false, 410 | inUse: false, 411 | sizeGB: snap.DiskSizeGb, 412 | }, 413 | compute: m.compute, 414 | }) 415 | } 416 | return snapList, nil 417 | } 418 | 419 | func (m *gcpResourceManager) getBuckets(project string) ([]Bucket, error) { 420 | buckets, err := m.storage.Buckets.List(project).Do() 421 | if err != nil { 422 | if buckets != nil && isGCPAccessDeniedError(buckets.HTTPStatusCode) { 423 | return nil, ErrPermissionDenied 424 | } 425 | return nil, err 426 | } 427 | buckList := []Bucket{} 428 | for _, buck := range buckets.Items { 429 | creationTime, err := time.Parse(time.RFC3339, buck.TimeCreated) 430 | if err != nil { 431 | // Set to Now so it doesn't incorrecntly get tagged for deletion 432 | creationTime = time.Now() 433 | } 434 | lastModified, err := time.Parse(time.RFC3339, buck.Updated) 435 | if err != nil { 436 | lastModified = time.Time{} 437 | } 438 | labels := buck.Labels 439 | if labels == nil { 440 | labels = make(map[string]string) 441 | } 442 | count, size, err := m.bucketDetails(buck.Name) 443 | if err != nil { 444 | log.Printf("Could not get object details for %s: %s", buck.Name, err) 445 | } 446 | buckList = append(buckList, &gcpBucket{ 447 | baseBucket: baseBucket{ 448 | baseResource: baseResource{ 449 | csp: GCP, 450 | owner: project, 451 | id: buck.Name, 452 | tags: labels, 453 | creationTime: creationTime, 454 | public: false, 455 | location: buck.Location, 456 | }, 457 | lastModified: lastModified, 458 | objectCount: count, 459 | totalSizeGB: size, 460 | storageTypeSizesGB: make(map[string]float64), 461 | }, 462 | storage: m.storage, 463 | }) 464 | } 465 | return buckList, nil 466 | } 467 | 468 | // bucketDetails will determine how many objects there are in a bucket and what 469 | // the total bucket size is. 470 | func (m *gcpResourceManager) bucketDetails(bucketID string) (int64, float64, error) { 471 | var count int64 472 | var sizeGB float64 473 | var nextPageToken string 474 | for ok := true; ok; ok = nextPageToken != "" { 475 | objs, err := m.storage.Objects.List(bucketID).Do() 476 | if err != nil { 477 | if objs != nil && isGCPAccessDeniedError(objs.HTTPStatusCode) { 478 | return 0, 0.0, ErrPermissionDenied 479 | } 480 | return 0, 0.0, err 481 | } 482 | nextPageToken = objs.NextPageToken 483 | for _, obj := range objs.Items { 484 | sizeGB += (float64(obj.Size) / gbDivider) 485 | count++ 486 | } 487 | } 488 | return count, sizeGB, nil 489 | } 490 | 491 | // Figure out if http response code is permission denied 492 | func isGCPAccessDeniedError(code int) bool { 493 | switch code { 494 | case 403: 495 | return true 496 | case 401: 497 | return true 498 | default: 499 | return false 500 | } 501 | } 502 | 503 | func parseGCPResourceURL(in string) string { 504 | parts := strings.Split(in, "/") 505 | n := len(parts) 506 | if n > 0 { 507 | return parts[n-1] 508 | } 509 | return in 510 | } 511 | -------------------------------------------------------------------------------- /cloud/image.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "log" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/service/ec2" 14 | compute "google.golang.org/api/compute/v1" 15 | ) 16 | 17 | type baseImage struct { 18 | baseResource 19 | name string 20 | sizeGB int64 21 | } 22 | 23 | func (i *baseImage) Name() string { 24 | return i.name 25 | } 26 | 27 | func (i *baseImage) SizeGB() int64 { 28 | return i.sizeGB 29 | } 30 | 31 | func cleanupImages(images []Image) error { 32 | resList := []Resource{} 33 | for i := range images { 34 | v, ok := images[i].(Resource) 35 | if !ok { 36 | return errors.New("Could not convert Image to Resource") 37 | } 38 | resList = append(resList, v) 39 | } 40 | return cleanupResources(resList) 41 | } 42 | 43 | // AWS 44 | 45 | type awsImage struct { 46 | baseImage 47 | } 48 | 49 | func (i *awsImage) Cleanup() error { 50 | log.Printf("Cleaning up image %s in %s", i.ID(), i.Owner()) 51 | return awsTryWithBackoff(i.cleanup) 52 | } 53 | 54 | func (i *awsImage) cleanup() error { 55 | client := clientForAWSResource(i) 56 | input := &ec2.DeregisterImageInput{ 57 | ImageId: aws.String(i.ID()), 58 | } 59 | _, err := client.DeregisterImage(input) 60 | if err != nil { 61 | aerr, ok := err.(awserr.Error) 62 | if ok && aerr.Code() == requestLimitErrorCode { 63 | return errAWSRequestLimit 64 | } 65 | } 66 | return err 67 | } 68 | 69 | func (i *awsImage) SetTag(key, value string, overwrite bool) error { 70 | return addAWSTag(i, key, value, overwrite) 71 | } 72 | 73 | func (i *awsImage) RemoveTag(key string) error { 74 | return removeAWSTag(i, key) 75 | } 76 | 77 | func (i *awsImage) MakePrivate() error { 78 | log.Printf("Making image %s private in %s", i.ID(), i.Owner()) 79 | if !i.Public() { 80 | // Image is already private 81 | return nil 82 | } 83 | client := clientForAWSResource(i) 84 | input := &ec2.ModifyImageAttributeInput{ 85 | ImageId: aws.String(i.ID()), 86 | LaunchPermission: &ec2.LaunchPermissionModifications{ 87 | Remove: []*ec2.LaunchPermission{&ec2.LaunchPermission{ 88 | Group: aws.String("all"), 89 | }}, 90 | }, 91 | } 92 | _, err := client.ModifyImageAttribute(input) 93 | if err != nil { 94 | return err 95 | } 96 | i.public = false 97 | return nil 98 | } 99 | 100 | // GCP 101 | 102 | type gcpImage struct { 103 | baseImage 104 | compute *compute.Service 105 | } 106 | 107 | func (i *gcpImage) Cleanup() error { 108 | log.Printf("Cleaning up image %s in %s", i.ID(), i.Owner()) 109 | _, err := i.compute.Images.Delete(i.Owner(), i.ID()).Do() 110 | return err 111 | } 112 | 113 | func (i *gcpImage) SetTag(key, value string, overwrite bool) error { 114 | img, err := i.compute.Images.Get(i.Owner(), i.ID()).Do() 115 | if err != nil { 116 | return nil 117 | } 118 | newLabels := img.Labels 119 | if newLabels == nil { 120 | newLabels = make(map[string]string) 121 | } 122 | if _, exist := newLabels[key]; exist && !overwrite { 123 | return fmt.Errorf("Key %s already exist on %s", key, i.ID()) 124 | } 125 | newLabels[key] = value 126 | req := &compute.GlobalSetLabelsRequest{ 127 | Labels: newLabels, 128 | LabelFingerprint: img.LabelFingerprint, 129 | } 130 | _, err = i.compute.Images.SetLabels(i.Owner(), i.ID(), req).Do() 131 | if err != nil { 132 | return err 133 | } 134 | i.tags = newLabels 135 | return nil 136 | } 137 | 138 | func (i *gcpImage) RemoveTag(key string) error { 139 | newLabels := make(map[string]string) 140 | for k, val := range i.tags { 141 | if k != key { 142 | newLabels[k] = val 143 | } 144 | } 145 | img, err := i.compute.Images.Get(i.Owner(), i.ID()).Do() 146 | if err != nil { 147 | return err 148 | } 149 | req := &compute.GlobalSetLabelsRequest{ 150 | Labels: newLabels, 151 | LabelFingerprint: img.LabelFingerprint, 152 | } 153 | _, err = i.compute.Images.SetLabels(i.Owner(), i.ID(), req).Do() 154 | if err != nil { 155 | return err 156 | } 157 | i.tags = newLabels 158 | return nil 159 | } 160 | 161 | func (i *gcpImage) MakePrivate() error { 162 | log.Println("Attempted to make GCP image private, NO-OP") 163 | return nil 164 | } 165 | -------------------------------------------------------------------------------- /cloud/instance.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "log" 10 | 11 | "github.com/aws/aws-sdk-go/aws/awserr" 12 | 13 | "github.com/aws/aws-sdk-go/aws" 14 | "github.com/aws/aws-sdk-go/service/ec2" 15 | compute "google.golang.org/api/compute/v1" 16 | ) 17 | 18 | type baseInstance struct { 19 | baseResource 20 | instanceType string 21 | } 22 | 23 | func (i *baseInstance) InstanceType() string { 24 | return i.instanceType 25 | } 26 | 27 | func cleanupInstances(instances []Instance) error { 28 | resList := []Resource{} 29 | for i := range instances { 30 | v, ok := instances[i].(Resource) 31 | if !ok { 32 | return errors.New("Could not convert Instance to Resource") 33 | } 34 | resList = append(resList, v) 35 | } 36 | return cleanupResources(resList) 37 | } 38 | 39 | // AWS 40 | 41 | type awsInstance struct { 42 | baseInstance 43 | } 44 | 45 | // Cleanup will termiante this instance 46 | func (i *awsInstance) Cleanup() error { 47 | log.Printf("Cleaning up instance %s in %s", i.ID(), i.Owner()) 48 | return awsTryWithBackoff(i.cleanup) 49 | } 50 | 51 | func (i *awsInstance) cleanup() error { 52 | client := clientForAWSResource(i) 53 | input := &ec2.TerminateInstancesInput{ 54 | InstanceIds: aws.StringSlice([]string{i.id}), 55 | } 56 | _, err := client.TerminateInstances(input) 57 | if err != nil { 58 | aerr, ok := err.(awserr.Error) 59 | if ok && aerr.Code() == requestLimitErrorCode { 60 | return errAWSRequestLimit 61 | } 62 | } 63 | return err 64 | } 65 | 66 | func (i *awsInstance) SetTag(key, value string, overwrite bool) error { 67 | return addAWSTag(i, key, value, overwrite) 68 | } 69 | 70 | func (i *awsInstance) RemoveTag(key string) error { 71 | return removeAWSTag(i, key) 72 | } 73 | 74 | // GCP 75 | 76 | type gcpInstance struct { 77 | baseInstance 78 | compute *compute.Service 79 | } 80 | 81 | func (i *gcpInstance) Cleanup() error { 82 | log.Printf("Cleaning up instance %s in %s", i.ID(), i.Owner()) 83 | _, err := i.compute.Instances.Delete(i.Owner(), i.Location(), i.ID()).Do() 84 | return err 85 | } 86 | 87 | func (i *gcpInstance) SetTag(key, value string, overwrite bool) error { 88 | inst, err := i.compute.Instances.Get(i.Owner(), i.Location(), i.ID()).Do() 89 | if err != nil { 90 | return err 91 | } 92 | newLabels := inst.Labels 93 | if newLabels == nil { 94 | newLabels = make(map[string]string) 95 | } 96 | if _, exist := newLabels[key]; exist && !overwrite { 97 | return fmt.Errorf("Key %s already exist on %s", key, i.ID()) 98 | } 99 | newLabels[key] = value 100 | req := &compute.InstancesSetLabelsRequest{ 101 | Labels: newLabels, 102 | LabelFingerprint: inst.LabelFingerprint, 103 | } 104 | _, err = i.compute.Instances.SetLabels(i.Owner(), i.Location(), i.ID(), req).Do() 105 | if err != nil { 106 | return err 107 | } 108 | i.tags = newLabels 109 | return nil 110 | } 111 | 112 | func (i *gcpInstance) RemoveTag(key string) error { 113 | newLabels := make(map[string]string) 114 | for k, val := range i.tags { 115 | if k != key { 116 | newLabels[k] = val 117 | } 118 | } 119 | inst, err := i.compute.Instances.Get(i.Owner(), i.Location(), i.ID()).Do() 120 | if err != nil { 121 | return err 122 | } 123 | req := &compute.InstancesSetLabelsRequest{ 124 | Labels: newLabels, 125 | LabelFingerprint: inst.LabelFingerprint, 126 | } 127 | _, err = i.compute.Instances.SetLabels(i.Owner(), i.Location(), i.ID(), req).Do() 128 | if err != nil { 129 | return err 130 | } 131 | i.tags = newLabels 132 | return nil 133 | } 134 | -------------------------------------------------------------------------------- /cloud/resource.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "log" 9 | "sync" 10 | "time" 11 | ) 12 | 13 | type baseResource struct { 14 | csp CSP 15 | owner string 16 | id string 17 | tags map[string]string 18 | location string 19 | public bool 20 | creationTime time.Time 21 | } 22 | 23 | func (r *baseResource) CSP() CSP { 24 | return r.csp 25 | } 26 | 27 | func (r *baseResource) Owner() string { 28 | return r.owner 29 | } 30 | 31 | func (r *baseResource) ID() string { 32 | return r.id 33 | } 34 | 35 | func (r *baseResource) Tags() map[string]string { 36 | return r.tags 37 | } 38 | 39 | func (r *baseResource) Location() string { 40 | return r.location 41 | } 42 | 43 | func (r *baseResource) Public() bool { 44 | return r.public 45 | } 46 | 47 | func (r *baseResource) CreationTime() time.Time { 48 | return r.creationTime 49 | } 50 | 51 | func cleanupResources(resources []Resource) error { 52 | failed := false 53 | var wg sync.WaitGroup 54 | wg.Add(len(resources)) 55 | for i := range resources { 56 | go func(index int) { 57 | err := resources[index].Cleanup() 58 | if err != nil { 59 | log.Printf("Cleaning up %s for owner %s failed\n%s\n", resources[index].ID(), resources[index].Owner(), err) 60 | failed = true 61 | } 62 | wg.Done() 63 | }(i) 64 | } 65 | wg.Wait() 66 | if failed { 67 | return errors.New("One or more resource cleanups failed") 68 | } 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /cloud/snapshot.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "log" 10 | 11 | "github.com/aws/aws-sdk-go/aws/awserr" 12 | 13 | "github.com/aws/aws-sdk-go/aws" 14 | "github.com/aws/aws-sdk-go/service/ec2" 15 | compute "google.golang.org/api/compute/v1" 16 | ) 17 | 18 | type baseSnapshot struct { 19 | baseResource 20 | encrypted bool 21 | inUse bool 22 | sizeGB int64 23 | } 24 | 25 | func (s *baseSnapshot) Encrypted() bool { 26 | return s.encrypted 27 | } 28 | 29 | func (s *baseSnapshot) InUse() bool { 30 | return s.inUse 31 | } 32 | 33 | func (s *baseSnapshot) SizeGB() int64 { 34 | return s.sizeGB 35 | } 36 | 37 | func cleanupSnapshots(snapshots []Snapshot) error { 38 | resList := []Resource{} 39 | for i := range snapshots { 40 | v, ok := snapshots[i].(Resource) 41 | if !ok { 42 | return errors.New("Could not convert Snapshot to Resource") 43 | } 44 | resList = append(resList, v) 45 | } 46 | return cleanupResources(resList) 47 | } 48 | 49 | // AWS 50 | 51 | type awsSnapshot struct { 52 | baseSnapshot 53 | } 54 | 55 | func (s *awsSnapshot) Cleanup() error { 56 | log.Printf("Cleaning up snapshot %s in %s", s.ID(), s.Owner()) 57 | return awsTryWithBackoff(s.cleanup) 58 | } 59 | 60 | func (s *awsSnapshot) cleanup() error { 61 | client := clientForAWSResource(s) 62 | input := &ec2.DeleteSnapshotInput{ 63 | SnapshotId: aws.String(s.ID()), 64 | } 65 | _, err := client.DeleteSnapshot(input) 66 | if err != nil { 67 | aerr, ok := err.(awserr.Error) 68 | if ok && aerr.Code() == requestLimitErrorCode { 69 | return errAWSRequestLimit 70 | } 71 | } 72 | return err 73 | } 74 | 75 | func (s *awsSnapshot) SetTag(key, value string, overwrite bool) error { 76 | return addAWSTag(s, key, value, overwrite) 77 | } 78 | 79 | func (s *awsSnapshot) RemoveTag(key string) error { 80 | return removeAWSTag(s, key) 81 | } 82 | 83 | // GCP 84 | 85 | type gcpSnapshot struct { 86 | baseSnapshot 87 | compute *compute.Service 88 | } 89 | 90 | func (s *gcpSnapshot) Cleanup() error { 91 | log.Printf("Cleaning up snapshot %s in %s", s.ID(), s.Owner()) 92 | _, err := s.compute.Snapshots.Delete(s.Owner(), s.ID()).Do() 93 | return err 94 | } 95 | 96 | func (s *gcpSnapshot) SetTag(key, value string, overwrite bool) error { 97 | snap, err := s.compute.Snapshots.Get(s.Owner(), s.ID()).Do() 98 | if err != nil { 99 | return err 100 | } 101 | newLabels := snap.Labels 102 | if newLabels == nil { 103 | newLabels = make(map[string]string) 104 | } 105 | if _, exist := newLabels[key]; exist && !overwrite { 106 | return fmt.Errorf("Key %s already exist on %s", key, s.ID()) 107 | } 108 | newLabels[key] = value 109 | req := &compute.GlobalSetLabelsRequest{ 110 | Labels: newLabels, 111 | LabelFingerprint: snap.LabelFingerprint, 112 | } 113 | _, err = s.compute.Snapshots.SetLabels(s.Owner(), s.ID(), req).Do() 114 | if err != nil { 115 | return err 116 | } 117 | s.tags = newLabels 118 | return nil 119 | } 120 | 121 | func (s *gcpSnapshot) RemoveTag(key string) error { 122 | newLabels := make(map[string]string) 123 | for k, val := range s.tags { 124 | if k != key { 125 | newLabels[k] = val 126 | } 127 | } 128 | snap, err := s.compute.Snapshots.Get(s.Owner(), s.ID()).Do() 129 | if err != nil { 130 | return err 131 | } 132 | req := &compute.GlobalSetLabelsRequest{ 133 | Labels: newLabels, 134 | LabelFingerprint: snap.LabelFingerprint, 135 | } 136 | _, err = s.compute.Snapshots.SetLabels(s.Owner(), s.ID(), req).Do() 137 | if err != nil { 138 | return err 139 | } 140 | s.tags = newLabels 141 | return nil 142 | } 143 | -------------------------------------------------------------------------------- /cloud/volume.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cloud 5 | 6 | import ( 7 | "errors" 8 | "fmt" 9 | "log" 10 | 11 | "github.com/aws/aws-sdk-go/aws" 12 | "github.com/aws/aws-sdk-go/aws/awserr" 13 | "github.com/aws/aws-sdk-go/service/ec2" 14 | compute "google.golang.org/api/compute/v1" 15 | ) 16 | 17 | type baseVolume struct { 18 | baseResource 19 | sizeGB int64 20 | attached bool 21 | encrypted bool 22 | volumeType string 23 | } 24 | 25 | func (v *baseVolume) SizeGB() int64 { 26 | return v.sizeGB 27 | } 28 | 29 | func (v *baseVolume) Attached() bool { 30 | return v.attached 31 | } 32 | 33 | func (v *baseVolume) Encrypted() bool { 34 | return v.encrypted 35 | } 36 | 37 | func (v *baseVolume) VolumeType() string { 38 | return v.volumeType 39 | } 40 | 41 | func cleanupVolumes(volumes []Volume) error { 42 | resList := []Resource{} 43 | for i := range volumes { 44 | v, ok := volumes[i].(Resource) 45 | if !ok { 46 | return errors.New("Could not convert Volume to Resource") 47 | } 48 | resList = append(resList, v) 49 | } 50 | return cleanupResources(resList) 51 | } 52 | 53 | // AWS 54 | 55 | type awsVolume struct { 56 | baseVolume 57 | } 58 | 59 | func (v *awsVolume) Cleanup() error { 60 | log.Printf("Cleaning up volume %s in %s", v.ID(), v.Owner()) 61 | return awsTryWithBackoff(v.cleanup) 62 | } 63 | 64 | func (v *awsVolume) cleanup() error { 65 | client := clientForAWSResource(v) 66 | input := &ec2.DeleteVolumeInput{ 67 | VolumeId: aws.String(v.ID()), 68 | } 69 | _, err := client.DeleteVolume(input) 70 | if err != nil { 71 | aerr, ok := err.(awserr.Error) 72 | if ok && aerr.Code() == requestLimitErrorCode { 73 | return errAWSRequestLimit 74 | } 75 | } 76 | return err 77 | } 78 | 79 | func (v *awsVolume) SetTag(key, value string, overwrite bool) error { 80 | return addAWSTag(v, key, value, overwrite) 81 | } 82 | 83 | func (v *awsVolume) RemoveTag(key string) error { 84 | return removeAWSTag(v, key) 85 | } 86 | 87 | // GCP 88 | 89 | type gcpVolume struct { 90 | baseVolume 91 | compute *compute.Service 92 | } 93 | 94 | func (v *gcpVolume) Cleanup() error { 95 | log.Printf("Cleaning up volume %s in %s", v.ID(), v.Owner()) 96 | _, err := v.compute.Disks.Delete(v.Owner(), v.Location(), v.ID()).Do() 97 | return err 98 | } 99 | 100 | func (v *gcpVolume) SetTag(key, value string, overwrite bool) error { 101 | disk, err := v.compute.Disks.Get(v.Owner(), v.Location(), v.ID()).Do() 102 | if err != nil { 103 | return err 104 | } 105 | newLabels := disk.Labels 106 | if newLabels == nil { 107 | newLabels = make(map[string]string) 108 | } 109 | if _, exist := newLabels[key]; exist && !overwrite { 110 | return fmt.Errorf("Key %s already exist on %s", key, v.ID()) 111 | } 112 | newLabels[key] = value 113 | req := &compute.ZoneSetLabelsRequest{ 114 | LabelFingerprint: disk.LabelFingerprint, 115 | Labels: newLabels, 116 | } 117 | _, err = v.compute.Disks.SetLabels(v.Owner(), v.Location(), v.ID(), req).Do() 118 | if err != nil { 119 | return err 120 | } 121 | v.tags = newLabels 122 | return nil 123 | } 124 | 125 | func (v *gcpVolume) RemoveTag(key string) error { 126 | newLabels := make(map[string]string) 127 | for k, val := range v.tags { 128 | if k != key { 129 | newLabels[k] = val 130 | } 131 | } 132 | disk, err := v.compute.Disks.Get(v.Owner(), v.Location(), v.ID()).Do() 133 | if err != nil { 134 | return err 135 | } 136 | req := &compute.ZoneSetLabelsRequest{ 137 | Labels: newLabels, 138 | LabelFingerprint: disk.LabelFingerprint, 139 | } 140 | _, err = v.compute.Disks.SetLabels(v.Owner(), v.Location(), v.ID(), req).Do() 141 | if err != nil { 142 | return err 143 | } 144 | v.tags = newLabels 145 | return nil 146 | } 147 | -------------------------------------------------------------------------------- /cloudsweeper/cleanup/cleanup.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package cleanup 5 | 6 | import ( 7 | "log" 8 | "sort" 9 | "time" 10 | 11 | "github.com/cloudtools/cloudsweeper/cloud" 12 | "github.com/cloudtools/cloudsweeper/cloud/billing" 13 | "github.com/cloudtools/cloudsweeper/cloud/filter" 14 | ) 15 | 16 | const ( 17 | releaseTag = "Release" 18 | totalCostThreshold = 10.0 19 | ) 20 | 21 | // MarkForCleanup will look for resources that should be automatically 22 | // cleaned up. These resources are not deleted directly, but are given 23 | // a tag that will delete the resources 4 days from now. The rules 24 | // for marking a resource for cleanup are the following: 25 | // - unattached volumes > 30 days old 26 | // - unused/unaccessed buckets > 6 months (182 days) 27 | // - non-whitelisted AMIs > 6 months 28 | // - non-whitelisted snapshots > 6 months 29 | // - non-whitelisted volumes > 6 months 30 | // - untagged resources > 30 days (this should take care of instances) 31 | func MarkForCleanup(mngr cloud.ResourceManager, thresholds map[string]int, dryRun bool) map[string]*cloud.AllResourceCollection { 32 | allResources := mngr.AllResourcesPerAccount() 33 | allBuckets := mngr.BucketsPerAccount() 34 | allResourcesToTag := make(map[string]*cloud.AllResourceCollection) 35 | 36 | for owner, res := range allResources { 37 | log.Println("Marking resources for cleanup in", owner) 38 | 39 | getThreshold := func(key string, thresholds map[string]int) int { 40 | threshold, found := thresholds[key] 41 | if found { 42 | return threshold 43 | } else { 44 | log.Fatalf("Threshold '%s' not found", key) 45 | return 99999 46 | } 47 | } 48 | 49 | untaggedFilter := filter.New() 50 | untaggedFilter.AddGeneralRule(filter.IsUntaggedWithException("Name")) 51 | untaggedFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-untagged-older-than-days", thresholds))) 52 | untaggedFilter.AddSnapshotRule(filter.IsNotInUse()) 53 | untaggedFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 54 | untaggedFilter.AddVolumeRule(filter.IsUnattached()) 55 | 56 | instanceFilter := filter.New() 57 | instanceFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-instances-older-than-days", thresholds))) 58 | instanceFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 59 | instanceFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 60 | 61 | snapshotFilter := filter.New() 62 | snapshotFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-snapshots-older-than-days", thresholds))) 63 | snapshotFilter.AddSnapshotRule(filter.IsNotInUse()) 64 | snapshotFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 65 | snapshotFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 66 | 67 | imageFilter := filter.New() 68 | imageFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-images-older-than-days", thresholds))) 69 | imageFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 70 | imageFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 71 | imageFilter.AddImageRule(filter.DoesNotFollowFormat()) 72 | 73 | volumeFilter := filter.New() 74 | volumeFilter.AddVolumeRule(filter.IsUnattached()) 75 | volumeFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-unattatched-older-than-days", thresholds))) 76 | volumeFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 77 | volumeFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 78 | 79 | bucketFilter := filter.New() 80 | bucketFilter.AddBucketRule(filter.NotModifiedInXDays(getThreshold("clean-bucket-not-modified-days", thresholds))) 81 | bucketFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("clean-bucket-older-than-days", thresholds))) 82 | bucketFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 83 | bucketFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 84 | 85 | timeToDelete := time.Now().AddDate(0, 0, 4) 86 | 87 | resourcesToTag := cloud.AllResourceCollection{} 88 | resourcesToTag.Owner = owner 89 | // Store a separate list of all resources since I couldn't for the life of me figure out how to 90 | // pass a []Image to a function that takes []Resource without explicitly converting everything... 91 | tagList := []cloud.Resource{} 92 | totalCost := 0.0 93 | 94 | // Tag instances 95 | for _, res := range filter.Instances(res.Instances, instanceFilter, untaggedFilter) { 96 | resourcesToTag.Instances = append(resourcesToTag.Instances, res) 97 | tagList = append(tagList, res) 98 | days := time.Now().Sub(res.CreationTime()).Hours() / 24.0 99 | costPerDay := billing.ResourceCostPerDay(res) 100 | totalCost += days * costPerDay 101 | } 102 | 103 | // Tag volumes 104 | for _, res := range filter.Volumes(res.Volumes, volumeFilter, untaggedFilter) { 105 | resourcesToTag.Volumes = append(resourcesToTag.Volumes, res) 106 | tagList = append(tagList, res) 107 | days := time.Now().Sub(res.CreationTime()).Hours() / 24.0 108 | costPerDay := billing.ResourceCostPerDay(res) 109 | totalCost += days * costPerDay 110 | } 111 | 112 | // Tag snapshots 113 | for _, res := range filter.Snapshots(res.Snapshots, snapshotFilter, untaggedFilter) { 114 | resourcesToTag.Snapshots = append(resourcesToTag.Snapshots, res) 115 | tagList = append(tagList, res) 116 | days := time.Now().Sub(res.CreationTime()).Hours() / 24.0 117 | costPerDay := billing.ResourceCostPerDay(res) 118 | totalCost += days * costPerDay 119 | } 120 | 121 | // Tag untagged images 122 | for _, res := range filter.Images(res.Images, untaggedFilter) { 123 | resourcesToTag.Images = append(resourcesToTag.Images, res) 124 | tagList = append(tagList, res) 125 | days := time.Now().Sub(res.CreationTime()).Hours() / 24.0 126 | costPerDay := billing.ResourceCostPerDay(res) 127 | totalCost += days * costPerDay 128 | } 129 | 130 | // Tag buckets 131 | if buck, ok := allBuckets[owner]; ok { 132 | for _, res := range filter.Buckets(buck, bucketFilter, untaggedFilter) { 133 | resourcesToTag.Buckets = append(resourcesToTag.Buckets, res) 134 | tagList = append(tagList, res) 135 | totalCost += billing.BucketPricePerMonth(res) 136 | } 137 | } 138 | 139 | // Helper map to avoid duplicated images 140 | alreadySelectedImages := map[string]bool{} 141 | for _, image := range resourcesToTag.Images { 142 | alreadySelectedImages[image.ID()] = true 143 | } 144 | 145 | // Tag images that DO NOT follow the component-date pattern 146 | for _, image := range filter.Images(res.Images, imageFilter) { 147 | if _, found := alreadySelectedImages[image.ID()]; !found { 148 | resourcesToTag.Images = append(resourcesToTag.Images, image) 149 | tagList = append(tagList, image) 150 | } 151 | } 152 | 153 | // Tag images that DO follow the component-date pattern 154 | componentImageFilter := filter.New() 155 | componentImageFilter.AddGeneralRule(filter.Negate(filter.HasTag(releaseTag))) 156 | componentImageFilter.AddGeneralRule(filter.Negate(filter.TaggedForCleanup())) 157 | componentImageFilter.AddImageRule(filter.FollowsFormat()) 158 | 159 | componentImages := getAllButNLatestComponents(res.Images, getThreshold("clean-keep-n-component-images", thresholds)) 160 | for _, image := range filter.Images(componentImages, componentImageFilter) { 161 | if _, found := alreadySelectedImages[image.ID()]; !found { 162 | resourcesToTag.Images = append(resourcesToTag.Images, image) 163 | tagList = append(tagList, image) 164 | } 165 | } 166 | 167 | if dryRun { 168 | log.Printf("Not tagging resources since this is a dry run") 169 | } else if totalCost < totalCostThreshold { 170 | log.Printf("%s: Skipping the tagging of resources, total cost $%.2f is less than $%.2f", owner, totalCost, totalCostThreshold) 171 | } else { 172 | for _, res := range tagList { 173 | err := res.SetTag(filter.DeleteTagKey, timeToDelete.Format(time.RFC3339), true) 174 | if err != nil { 175 | log.Printf("%s: Failed to tag %s for deletion: %s\n", owner, res.ID(), err) 176 | } else { 177 | log.Printf("%s: Marked %s for deletion at %s\n", owner, res.ID(), timeToDelete) 178 | } 179 | } 180 | } 181 | allResourcesToTag[owner] = &resourcesToTag 182 | } 183 | return allResourcesToTag 184 | } 185 | 186 | // GetAllButNLatestComponents will look at AMIs, and return all but the two latest for each 187 | // component, where the naming of the AMIs is on the form: 188 | // "-" 189 | func getAllButNLatestComponents(images []cloud.Image, componentsToKeep int) []cloud.Image { 190 | resourcesToTag := []cloud.Image{} 191 | componentDatesMap := map[string][]time.Time{} 192 | 193 | for _, image := range images { 194 | componentName, creationDate := filter.ParseFormat(image) 195 | if _, found := componentDatesMap[componentName]; !found { 196 | componentDatesMap[componentName] = []time.Time{} 197 | } 198 | componentDatesMap[componentName] = append(componentDatesMap[componentName], creationDate) 199 | } 200 | 201 | findThreshold := func(componentName string) time.Time { 202 | times, found := componentDatesMap[componentName] 203 | if !found { 204 | log.Fatalln("Times not found for some reason") 205 | return time.Now().AddDate(-10, 0, 0) 206 | } 207 | 208 | sort.Slice(times, func(i, j int) bool { 209 | // Sort times so that newest are first 210 | return times[i].After(times[j]) 211 | }) 212 | 213 | minimumIndex := componentsToKeep 214 | if minimumIndex > len(times) { 215 | minimumIndex = len(times) 216 | } 217 | threshold := times[minimumIndex-1] 218 | return threshold 219 | } 220 | 221 | for _, image := range images { 222 | componentName, creationDate := filter.ParseFormat(image) 223 | threshold := findThreshold(componentName) 224 | if creationDate.Before(threshold) { 225 | // This AMI is too old, mark it 226 | resourcesToTag = append(resourcesToTag, image) 227 | } 228 | } 229 | return resourcesToTag 230 | } 231 | 232 | // PerformCleanup will run different cleanup functions which all 233 | // do some sort of rule based cleanup 234 | func PerformCleanup(mngr cloud.ResourceManager) { 235 | // Cleanup all resources with a lifetime tag that has passed. This 236 | // includes both the lifetime and the expiry tag 237 | cleanupLifetimePassed(mngr) 238 | } 239 | 240 | func cleanupLifetimePassed(mngr cloud.ResourceManager) { 241 | allResources := mngr.AllResourcesPerAccount() 242 | allBuckets := mngr.BucketsPerAccount() 243 | for owner, resources := range allResources { 244 | log.Println("Performing lifetime check in", owner) 245 | lifetimeFilter := filter.New() 246 | lifetimeFilter.AddGeneralRule(filter.LifetimeExceeded()) 247 | 248 | expiryFilter := filter.New() 249 | expiryFilter.AddGeneralRule(filter.ExpiryDatePassed()) 250 | 251 | deleteAtFilter := filter.New() 252 | deleteAtFilter.AddGeneralRule(filter.DeleteAtPassed()) 253 | 254 | err := mngr.CleanupInstances(filter.Instances(resources.Instances, lifetimeFilter, expiryFilter, deleteAtFilter)) 255 | if err != nil { 256 | log.Printf("Could not cleanup instances in %s, err:\n%s", owner, err) 257 | } 258 | err = mngr.CleanupImages(filter.Images(resources.Images, lifetimeFilter, expiryFilter, deleteAtFilter)) 259 | if err != nil { 260 | log.Printf("Could not cleanup images in %s, err:\n%s", owner, err) 261 | } 262 | err = mngr.CleanupVolumes(filter.Volumes(resources.Volumes, lifetimeFilter, expiryFilter, deleteAtFilter)) 263 | if err != nil { 264 | log.Printf("Could not cleanup volumes in %s, err:\n%s", owner, err) 265 | } 266 | err = mngr.CleanupSnapshots(filter.Snapshots(resources.Snapshots, lifetimeFilter, expiryFilter, deleteAtFilter)) 267 | if err != nil { 268 | log.Printf("Could not cleanup snapshots in %s, err:\n%s", owner, err) 269 | } 270 | if bucks, ok := allBuckets[owner]; ok { 271 | err = mngr.CleanupBuckets(filter.Buckets(bucks, lifetimeFilter, expiryFilter, deleteAtFilter)) 272 | if err != nil { 273 | log.Printf("Could not cleanup buckets in %s, err:\n%s", owner, err) 274 | } 275 | } 276 | } 277 | } 278 | 279 | // ResetCloudsweeper will remove any cleanup tags existing in the accounts 280 | // associated with the provided resource manager 281 | func ResetCloudsweeper(mngr cloud.ResourceManager) { 282 | allResources := mngr.AllResourcesPerAccount() 283 | allBuckets := mngr.BucketsPerAccount() 284 | 285 | for owner, res := range allResources { 286 | log.Println("Resetting Cloudsweeper tags in", owner) 287 | taggedFilter := filter.New() 288 | taggedFilter.AddGeneralRule(filter.HasTag(filter.DeleteTagKey)) 289 | 290 | handleError := func(res cloud.Resource, err error) { 291 | if err != nil { 292 | log.Printf("Failed to remove tag on %s: %s\n", res.ID(), err) 293 | } else { 294 | log.Printf("Removed cleanup tag on %s\n", res.ID()) 295 | } 296 | } 297 | 298 | // Un-Tag instances 299 | for _, res := range filter.Instances(res.Instances, taggedFilter) { 300 | handleError(res, res.RemoveTag(filter.DeleteTagKey)) 301 | } 302 | 303 | // Un-Tag volumes 304 | for _, res := range filter.Volumes(res.Volumes, taggedFilter) { 305 | handleError(res, res.RemoveTag(filter.DeleteTagKey)) 306 | } 307 | 308 | // Un-Tag snapshots 309 | for _, res := range filter.Snapshots(res.Snapshots, taggedFilter) { 310 | handleError(res, res.RemoveTag(filter.DeleteTagKey)) 311 | } 312 | 313 | // Un-Tag images 314 | for _, res := range filter.Images(res.Images, taggedFilter) { 315 | handleError(res, res.RemoveTag(filter.DeleteTagKey)) 316 | } 317 | 318 | // Un-Tag buckets 319 | if buck, ok := allBuckets[owner]; ok { 320 | for _, res := range filter.Buckets(buck, taggedFilter) { 321 | handleError(res, res.RemoveTag(filter.DeleteTagKey)) 322 | } 323 | } 324 | 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /cloudsweeper/find/aws.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package find 5 | 6 | import ( 7 | "fmt" 8 | "log" 9 | "strings" 10 | 11 | "github.com/cloudtools/cloudsweeper/cloudsweeper" 12 | 13 | "github.com/cloudtools/cloudsweeper/cloud" 14 | ) 15 | 16 | type awsResourceType int 17 | 18 | const ( 19 | awsTypeInstance awsResourceType = iota 20 | awsTypeVolume 21 | awsTypeSnapshop 22 | awsTypeImage 23 | ) 24 | 25 | type awsClient struct { 26 | cloudManager cloud.ResourceManager 27 | organization *cloudsweeper.Organization 28 | } 29 | 30 | func (c *awsClient) CSP() cloud.CSP { 31 | return cloud.AWS 32 | } 33 | 34 | func (c *awsClient) FindResource(id string) error { 35 | resourceType, err := c.determineResourceType(id) 36 | if err != nil { 37 | return err 38 | } 39 | 40 | for account, resources := range c.cloudManager.AllResourcesPerAccount() { 41 | log.Printf("Looking for %s in account %s\n", id, account) 42 | switch resourceType { 43 | case awsTypeInstance: 44 | for _, inst := range resources.Instances { 45 | if inst.ID() == id { 46 | // Found instance 47 | log.Printf("Found instance in account %s", account) 48 | employee, err := c.getEmployee(account) 49 | if err != nil { 50 | return err 51 | } 52 | foundInstance(inst, account, employee) 53 | return nil 54 | } 55 | } 56 | case awsTypeVolume: 57 | for _, vol := range resources.Volumes { 58 | if vol.ID() == id { 59 | // Found volume 60 | employee, err := c.getEmployee(account) 61 | if err != nil { 62 | return err 63 | } 64 | foundVolume(vol, account, employee) 65 | return nil 66 | } 67 | } 68 | case awsTypeImage: 69 | for _, ami := range resources.Images { 70 | if ami.ID() == id { 71 | // Found AMI 72 | employee, err := c.getEmployee(account) 73 | if err != nil { 74 | return err 75 | } 76 | foundImage(ami, account, employee) 77 | return nil 78 | } 79 | } 80 | case awsTypeSnapshop: 81 | for _, snap := range resources.Snapshots { 82 | if snap.ID() == id { 83 | // Found snapshot 84 | employee, err := c.getEmployee(account) 85 | if err != nil { 86 | return err 87 | } 88 | foundSnapshot(snap, account, employee) 89 | return nil 90 | } 91 | } 92 | } 93 | } 94 | return fmt.Errorf("Resource %s not found in any account", id) 95 | } 96 | 97 | func (c *awsClient) determineResourceType(id string) (awsResourceType, error) { 98 | idParts := strings.Split(id, "-") 99 | if len(idParts) != 2 { 100 | return -1, fmt.Errorf("Looks like ID %s is not a valid AWS resource id", id) 101 | } 102 | prefix := idParts[0] 103 | switch prefix { 104 | case "i": 105 | log.Println("Resource is an instance") 106 | return awsTypeInstance, nil 107 | case "vol": 108 | log.Println("Resource is a volume") 109 | return awsTypeVolume, nil 110 | case "ami": 111 | log.Println("Resource is an image/AMI") 112 | return awsTypeImage, nil 113 | case "snap": 114 | log.Println("Resource is a snapshot") 115 | return awsTypeSnapshop, nil 116 | default: 117 | return -1, fmt.Errorf("Unsupported resource type, must be one of either instance, volume, AMI, or snapshot") 118 | } 119 | } 120 | 121 | func (c *awsClient) getEmployee(accountID string) (*cloudsweeper.Employee, error) { 122 | users := c.organization.AccountToUserMapping(cloud.AWS) 123 | employees := c.organization.UsernameToEmployeeMapping() 124 | if user, ok := users[accountID]; ok { 125 | if employee, ok := employees[user]; ok { 126 | return employee, nil 127 | } 128 | } 129 | return nil, fmt.Errorf("Could not find information about account %s", accountID) 130 | } 131 | -------------------------------------------------------------------------------- /cloudsweeper/find/find.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | // Package find is containing functionality to find more information 5 | // about a cloud resource given its ID. 6 | package find 7 | 8 | import ( 9 | "fmt" 10 | "time" 11 | 12 | "github.com/cloudtools/cloudsweeper/cloud" 13 | "github.com/cloudtools/cloudsweeper/cloudsweeper" 14 | ) 15 | 16 | const foundBannerTemplate = ` 17 | 18 | ############################################# 19 | Found %s 20 | ############################################# 21 | 22 | ` 23 | 24 | // Client is a client for finding a resource in a specific cloud 25 | type Client interface { 26 | FindResource(id string) error 27 | CSP() cloud.CSP 28 | } 29 | 30 | // Init will initialize a finding Client for the given CSP 31 | func Init(mngr cloud.ResourceManager, org *cloudsweeper.Organization, csp cloud.CSP) (Client, error) { 32 | if csp == cloud.AWS { 33 | return &awsClient{ 34 | cloudManager: mngr, 35 | organization: org, 36 | }, nil 37 | } 38 | return nil, fmt.Errorf("Unsupported CSP: %s", csp) 39 | } 40 | 41 | func foundInstance(inst cloud.Instance, account string, owner *cloudsweeper.Employee) { 42 | fmt.Printf(foundBannerTemplate, "Instance") 43 | foundResource(inst, account, owner) 44 | fmt.Printf("Instance Type: %s\n", inst.InstanceType()) 45 | } 46 | 47 | func foundVolume(vol cloud.Volume, account string, owner *cloudsweeper.Employee) { 48 | fmt.Printf(foundBannerTemplate, "Volume") 49 | foundResource(vol, account, owner) 50 | fmt.Printf("Volume Type: %s\n", vol.VolumeType()) 51 | fmt.Printf("Size: %d GB\n", vol.SizeGB()) 52 | } 53 | 54 | func foundImage(image cloud.Image, account string, owner *cloudsweeper.Employee) { 55 | fmt.Printf(foundBannerTemplate, "Image") 56 | foundResource(image, account, owner) 57 | var isPublic string 58 | if image.Public() { 59 | isPublic = "Yes" 60 | } else { 61 | isPublic = "No" 62 | } 63 | fmt.Printf("Is public: %s\n", isPublic) 64 | fmt.Printf("Size: %d GB\n", image.SizeGB()) 65 | } 66 | 67 | func foundSnapshot(snap cloud.Snapshot, account string, owner *cloudsweeper.Employee) { 68 | fmt.Printf(foundBannerTemplate, "Snapshot") 69 | foundResource(snap, account, owner) 70 | fmt.Printf("Size: %d GB\n", snap.SizeGB()) 71 | } 72 | 73 | func foundResource(res cloud.Resource, account string, owner *cloudsweeper.Employee) { 74 | var resourceName = "" 75 | if name, ok := res.Tags()["Name"]; ok { 76 | resourceName = name 77 | } 78 | 79 | fmt.Printf("Account: %s (%s)\n", owner.Username, account) 80 | fmt.Printf("Resource ID: %s\n", res.ID()) 81 | fmt.Printf("Resource name: %s\n", resourceName) 82 | fmt.Printf("Region: %s\n", res.Location()) 83 | fmt.Printf("Creation Time: %s\n", res.CreationTime().Format(time.RFC3339)) 84 | fmt.Printf("Tags:\n") 85 | for key, val := range res.Tags() { 86 | if val != "" { 87 | fmt.Printf("\t\t%s: %s\n", key, val) 88 | } else { 89 | fmt.Printf("\t\t%s\n", key) 90 | } 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /cloudsweeper/notify/helpers.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package notify 5 | 6 | import ( 7 | "bytes" 8 | "fmt" 9 | "html/template" 10 | "time" 11 | 12 | "github.com/cloudtools/cloudsweeper/cloud" 13 | "github.com/cloudtools/cloudsweeper/cloud/billing" 14 | "github.com/cloudtools/cloudsweeper/cloud/filter" 15 | "github.com/cloudtools/cloudsweeper/mailer" 16 | ) 17 | 18 | var emailEdgeCases = map[string]string{} // Use this map to fix bad mappings between usernames and email aliases 19 | 20 | func generateMail(data interface{}, templateString string) (string, error) { 21 | t := template.New("emailTemplate").Funcs(extraTemplateFunctions()) 22 | t, err := t.Parse(templateString) 23 | if err != nil { 24 | return "", err 25 | } 26 | var result bytes.Buffer 27 | err = t.Execute(&result, data) 28 | if err != nil { 29 | return "", err 30 | } 31 | return result.String(), nil 32 | } 33 | 34 | // This function will convert some edge case emails to their proper 35 | // email. This is useful if some user doesn't share the common org domain 36 | func convertEmailExceptions(oldMail string) string { 37 | name, hasEdgeCase := emailEdgeCases[oldMail] 38 | if hasEdgeCase { 39 | return name 40 | } 41 | return oldMail 42 | } 43 | 44 | func getMailClient(notifyClient *Client) mailer.Client { 45 | username := notifyClient.config.SMTPUsername 46 | password := notifyClient.config.SMTPPassword 47 | server := notifyClient.config.SMTPServer 48 | port := notifyClient.config.SMTPPort 49 | from := notifyClient.config.MailFrom 50 | displayName := notifyClient.config.DisplayName 51 | return mailer.NewClient(username, password, displayName, from, server, port) 52 | } 53 | 54 | func accumulatedCost(res cloud.Resource) float64 { 55 | days := time.Now().Sub(res.CreationTime()).Hours() / 24.0 56 | costPerDay := billing.ResourceCostPerDay(res) 57 | return days * costPerDay 58 | } 59 | 60 | func extraTemplateFunctions() template.FuncMap { 61 | return template.FuncMap{ 62 | "fdate": func(t time.Time, format string) string { return t.Format(format) }, 63 | "daysrunning": func(t time.Time) string { 64 | if (t == time.Time{}) { 65 | return "never" 66 | } 67 | days := int(time.Now().Sub(t).Hours() / 24.0) 68 | switch days { 69 | case 0: 70 | return "today" 71 | case 1: 72 | return "yesterday" 73 | default: 74 | return fmt.Sprintf("%d days ago", days) 75 | } 76 | }, 77 | // TODO: this should be configurable 78 | "modifiedInTheLast6Months": func(t time.Time) string { 79 | if time.Now().Before(t.AddDate(0, 6, 0)) { 80 | return "true" 81 | } 82 | return "false" 83 | }, 84 | 85 | "even": func(num int) bool { return num%2 == 0 }, 86 | "yesno": func(b bool) string { 87 | if b { 88 | return "Yes" 89 | } 90 | return "No" 91 | }, 92 | "whitelisted": func(res cloud.Resource) bool { 93 | return filter.IsWhitelisted(res) 94 | }, 95 | "accucost": func(res cloud.Resource) string { 96 | totalCost := accumulatedCost(res) 97 | return fmt.Sprintf("$%.2f", totalCost) 98 | }, 99 | "bucketcost": func(res cloud.Bucket) float64 { 100 | return billing.BucketPricePerMonth(res) 101 | }, 102 | "instname": func(inst cloud.Instance) string { 103 | if inst.CSP() == cloud.AWS { 104 | name, exist := inst.Tags()["Name"] 105 | if exist { 106 | return name 107 | } 108 | return "" 109 | 110 | } else if inst.CSP() == cloud.GCP { 111 | return inst.ID() 112 | } else { 113 | return "" 114 | } 115 | }, 116 | "productname": func(res cloud.Resource) string { 117 | product, exist := res.Tags()["product"] 118 | if exist { 119 | return product 120 | } 121 | return "" 122 | }, 123 | "rolename": func(res cloud.Resource) string { 124 | role, exist := res.Tags()["role"] 125 | if exist { 126 | return role 127 | } 128 | return "" 129 | }, 130 | "maybeRealName": func(account string, accountToUser map[string]string) string { 131 | if name, ok := accountToUser[account]; ok { 132 | return name 133 | } 134 | return account 135 | }, 136 | "prettyTag": func(key, val string) string { 137 | if val == "" { 138 | return key 139 | } 140 | return fmt.Sprintf("%s: %s", key, val) 141 | }, 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /cloudsweeper/notify/notify.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | // Package notify is responsible for all actions related 5 | // to notifying employees and managers about their resources. 6 | // 7 | // Email credentials must be set using os environment variables 8 | // in order to be able to send mail. Note that monthToDateAddressee 9 | // is intended to be sent weekly to your entire org. The 10 | // totalSumAddressee is meant to send a total report to the person 11 | // in your org monitoring costs. 12 | // 13 | // The templates.go file contains all email templates used for 14 | // notifications. This uses the native Go template engine. 15 | package notify 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "sort" 21 | "time" 22 | 23 | "github.com/cloudtools/cloudsweeper/mailer" 24 | 25 | "github.com/cloudtools/cloudsweeper/cloud" 26 | "github.com/cloudtools/cloudsweeper/cloud/billing" 27 | "github.com/cloudtools/cloudsweeper/cloud/filter" 28 | cs "github.com/cloudtools/cloudsweeper/cloudsweeper" 29 | ) 30 | 31 | // Client is used to perform the notify actions. It must be 32 | // initalized with correct values to work properly. 33 | type Client struct { 34 | config *Config 35 | } 36 | 37 | // Config is a configuration for the notify Client 38 | type Config struct { 39 | SMTPUsername string 40 | SMTPPassword string 41 | SMTPServer string 42 | SMTPPort int 43 | DisplayName string 44 | MailFrom string 45 | EmailDomain string 46 | BillingReportAddressee string 47 | TotalSumAddresse string 48 | } 49 | 50 | // Init will initialize a notify Client with a given Config 51 | func Init(config *Config) *Client { 52 | return &Client{config: config} 53 | } 54 | 55 | type resourceMailData struct { 56 | Owner string 57 | OwnerID string 58 | Instances []cloud.Instance 59 | Images []cloud.Image 60 | Snapshots []cloud.Snapshot 61 | Volumes []cloud.Volume 62 | Buckets []cloud.Bucket 63 | HoursInAdvance int 64 | } 65 | 66 | func (d *resourceMailData) ResourceCount() int { 67 | return len(d.Images) + len(d.Instances) + len(d.Snapshots) + len(d.Volumes) + len(d.Buckets) 68 | } 69 | 70 | func (d *resourceMailData) SortByCost() { 71 | sort.Slice(d.Instances, func(i, j int) bool { 72 | return accumulatedCost(d.Instances[i]) > accumulatedCost(d.Instances[j]) 73 | }) 74 | sort.Slice(d.Images, func(i, j int) bool { 75 | return accumulatedCost(d.Images[i]) > accumulatedCost(d.Images[j]) 76 | }) 77 | sort.Slice(d.Snapshots, func(i, j int) bool { 78 | return accumulatedCost(d.Snapshots[i]) > accumulatedCost(d.Snapshots[j]) 79 | }) 80 | sort.Slice(d.Volumes, func(i, j int) bool { 81 | return accumulatedCost(d.Volumes[i]) > accumulatedCost(d.Volumes[j]) 82 | }) 83 | sort.Slice(d.Buckets, func(i, j int) bool { 84 | return billing.BucketPricePerMonth(d.Buckets[i]) > billing.BucketPricePerMonth(d.Buckets[j]) 85 | }) 86 | } 87 | 88 | func (d *resourceMailData) SendEmail(client mailer.Client, domain, mailTemplate, title string, debugAddressees ...string) { 89 | // Always sort by cost 90 | d.SortByCost() 91 | 92 | mailContent, err := generateMail(d, mailTemplate) 93 | if err != nil { 94 | log.Fatalln("Could not generate email:", err) 95 | } 96 | 97 | ownerMail := fmt.Sprintf("%s@%s", d.Owner, domain) 98 | recieverMail := convertEmailExceptions(ownerMail) 99 | log.Printf("Sending out email to %s\n", recieverMail) 100 | addressees := append(debugAddressees, recieverMail) 101 | err = client.SendEmail(title, mailContent, addressees...) 102 | if err != nil { 103 | log.Fatalf("Failed to email %s: %s\n", recieverMail, err) 104 | } 105 | } 106 | 107 | type monthToDateData struct { 108 | CSP cloud.CSP 109 | TotalCost float64 110 | SortedUsers billing.UserList 111 | MinimumTotalCost float64 112 | MinimumCost float64 113 | AccountToUser map[string]string 114 | } 115 | 116 | func initTotalSummaryMailData(totalSumAddressee string) *resourceMailData { 117 | return &resourceMailData{ 118 | Owner: totalSumAddressee, 119 | Instances: []cloud.Instance{}, 120 | Images: []cloud.Image{}, 121 | Snapshots: []cloud.Snapshot{}, 122 | Volumes: []cloud.Volume{}, 123 | Buckets: []cloud.Bucket{}, 124 | } 125 | } 126 | 127 | func initManagerToMailDataMapping(managers cs.Employees) map[string]*resourceMailData { 128 | result := make(map[string]*resourceMailData) 129 | for _, manager := range managers { 130 | result[manager.Username] = &resourceMailData{ 131 | Owner: manager.Username, 132 | Instances: []cloud.Instance{}, 133 | Images: []cloud.Image{}, 134 | Snapshots: []cloud.Snapshot{}, 135 | Volumes: []cloud.Volume{}, 136 | Buckets: []cloud.Bucket{}, 137 | } 138 | } 139 | return result 140 | } 141 | 142 | // OldResourceReview will review (but not do any cleanup action) old resources 143 | // that an owner might want to consider doing something about. The owner is then 144 | // sent an email with a list of these resources. Resources are sent for review 145 | // if they fulfil any of the following rules: 146 | // - Resource is older than 30 days 147 | // - A whitelisted resource is older than 6 months 148 | // - An instance marked with do-not-delete is older than a week 149 | func (c *Client) OldResourceReview(mngr cloud.ResourceManager, org *cs.Organization, csp cloud.CSP, thresholds map[string]int) { 150 | allCompute := mngr.AllResourcesPerAccount() 151 | allBuckets := mngr.BucketsPerAccount() 152 | accountUserMapping := org.AccountToUserMapping(csp) 153 | userEmployeeMapping := org.UsernameToEmployeeMapping() 154 | totalSummaryMailData := initTotalSummaryMailData(c.config.TotalSumAddresse) 155 | managerToMailDataMapping := initManagerToMailDataMapping(org.Managers) 156 | 157 | getThreshold := func(key string, thresholds map[string]int) int { 158 | threshold, found := thresholds[key] 159 | if found { 160 | return threshold 161 | } else { 162 | errorText := fmt.Sprintf("Threshold '%s' not found", key) 163 | log.Fatalln(errorText) 164 | return 99999 165 | } 166 | } 167 | 168 | // Create filters 169 | instanceFilter := filter.New() 170 | instanceFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-instances-older-than-days", thresholds))) 171 | 172 | imageFilter := filter.New() 173 | imageFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-images-older-than-days", thresholds))) 174 | 175 | volumeFilter := filter.New() 176 | volumeFilter.AddVolumeRule(filter.IsUnattached()) 177 | volumeFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-unattached-older-than-days", thresholds))) 178 | 179 | snapshotFilter := filter.New() 180 | snapshotFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-snapshots-older-than-days", thresholds))) 181 | 182 | bucketFilter := filter.New() 183 | bucketFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-buckets-older-than-days", thresholds))) 184 | 185 | whitelistFilter := filter.New() 186 | whitelistFilter.OverrideWhitelist = true 187 | whitelistFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-whitelist-older-than-days", thresholds))) 188 | 189 | untaggedFilter := filter.New() 190 | untaggedFilter.AddGeneralRule(filter.IsUntaggedWithException("Name")) 191 | untaggedFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-untagged-older-than-days", thresholds))) 192 | untaggedFilter.AddSnapshotRule(filter.IsNotInUse()) 193 | untaggedFilter.AddVolumeRule(filter.IsUnattached()) 194 | 195 | // These only apply to instances 196 | dndFilter := filter.New() 197 | dndFilter.AddGeneralRule(filter.HasTag("no-not-delete")) 198 | dndFilter.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-dnd-older-than-days", thresholds))) 199 | 200 | dndFilter2 := filter.New() 201 | dndFilter2.AddGeneralRule(filter.NameContains("do-not-delete")) 202 | dndFilter2.AddGeneralRule(filter.OlderThanXDays(getThreshold("notify-dnd-older-than-days", thresholds))) 203 | 204 | for account, resources := range allCompute { 205 | log.Println("Performing old resource review in", account) 206 | username := accountUserMapping[account] 207 | employee := userEmployeeMapping[username] 208 | 209 | // Apply filters 210 | userMailData := resourceMailData{ 211 | Owner: username, 212 | Instances: filter.Instances(resources.Instances, instanceFilter, whitelistFilter, dndFilter, dndFilter2, untaggedFilter), 213 | Images: filter.Images(resources.Images, imageFilter, whitelistFilter, untaggedFilter), 214 | Volumes: filter.Volumes(resources.Volumes, volumeFilter, whitelistFilter, untaggedFilter), 215 | Snapshots: filter.Snapshots(resources.Snapshots, snapshotFilter, whitelistFilter, untaggedFilter), 216 | Buckets: []cloud.Bucket{}, 217 | } 218 | if buckets, ok := allBuckets[account]; ok { 219 | userMailData.Buckets = filter.Buckets(buckets, bucketFilter, whitelistFilter, untaggedFilter) 220 | } 221 | 222 | // Add to the manager summary 223 | if managerSummaryMailData, ok := managerToMailDataMapping[employee.Manager.Username]; ok { // safe or org _should_ have thrown an error 224 | managerSummaryMailData.Instances = append(managerSummaryMailData.Instances, userMailData.Instances...) 225 | managerSummaryMailData.Images = append(managerSummaryMailData.Images, userMailData.Images...) 226 | managerSummaryMailData.Snapshots = append(managerSummaryMailData.Snapshots, userMailData.Snapshots...) 227 | managerSummaryMailData.Volumes = append(managerSummaryMailData.Volumes, userMailData.Volumes...) 228 | managerSummaryMailData.Buckets = append(managerSummaryMailData.Buckets, userMailData.Buckets...) 229 | } else { 230 | log.Fatalf("%s is not a manager??? Verify `organization.go` and the org repo itself for issues", employee.Manager.Username) 231 | } 232 | 233 | // Add to the total summary 234 | totalSummaryMailData.Instances = append(totalSummaryMailData.Instances, userMailData.Instances...) 235 | totalSummaryMailData.Images = append(totalSummaryMailData.Images, userMailData.Images...) 236 | totalSummaryMailData.Snapshots = append(totalSummaryMailData.Snapshots, userMailData.Snapshots...) 237 | totalSummaryMailData.Volumes = append(totalSummaryMailData.Volumes, userMailData.Volumes...) 238 | totalSummaryMailData.Buckets = append(totalSummaryMailData.Buckets, userMailData.Buckets...) 239 | 240 | if userMailData.ResourceCount() > 0 { 241 | title := fmt.Sprintf("You have %d old resources to review (%s)", userMailData.ResourceCount(), time.Now().Format("2006-01-02")) 242 | userMailData.SendEmail(getMailClient(c), c.config.EmailDomain, reviewMailTemplate, title) 243 | } 244 | } 245 | 246 | // Send out manager emails 247 | for username, managerSummaryMailData := range managerToMailDataMapping { 248 | log.Printf("Collecting old resources to review for %s's team\n", username) 249 | if managerSummaryMailData.ResourceCount() > 0 { 250 | title := fmt.Sprintf("Your team has %d old resources to review (%s)", managerSummaryMailData.ResourceCount(), time.Now().Format("2006-01-02")) 251 | managerSummaryMailData.SendEmail(getMailClient(c), c.config.EmailDomain, managerReviewMailTemplate, title) 252 | } 253 | } 254 | 255 | // Send out a total summary 256 | log.Println("Collecting old resource review for the org") 257 | title := fmt.Sprintf("Your org has %d old resources to review (%s)", totalSummaryMailData.ResourceCount(), time.Now().Format("2006-01-02")) 258 | totalSummaryMailData.SendEmail(getMailClient(c), c.config.EmailDomain, totalReviewMailTemplate, title) 259 | } 260 | 261 | // UntaggedResourcesReview will look for resources without any tags, and 262 | // send out a mail encouraging to tag tag them 263 | func (c *Client) UntaggedResourcesReview(mngr cloud.ResourceManager, accountUserMapping map[string]string) { 264 | // We only care about untagged resources in EC2 265 | allCompute := mngr.AllResourcesPerAccount() 266 | for account, resources := range allCompute { 267 | log.Printf("Performing untagged resources review in %s", account) 268 | untaggedFilter := filter.New() 269 | untaggedFilter.AddGeneralRule(filter.IsUntaggedWithException("Name")) 270 | 271 | // We care about un-tagged whitelisted resources too 272 | untaggedFilter.OverrideWhitelist = true 273 | 274 | username := accountUserMapping[account] 275 | mailData := resourceMailData{ 276 | Owner: username, 277 | OwnerID: account, 278 | Instances: filter.Instances(resources.Instances, untaggedFilter), 279 | // Only report on instances for now 280 | //Images: filter.Images(resources.Images, untaggedFilter), 281 | //Snapshots: filter.Snapshots(resources.Snapshots, untaggedFilter), 282 | //Volumes: filter.Volumes(resources.Volumes, untaggedFilter), 283 | Buckets: []cloud.Bucket{}, 284 | } 285 | 286 | if mailData.ResourceCount() > 0 { 287 | // Send mail 288 | title := fmt.Sprintf("You have %d un-tagged resources to review (%s)", mailData.ResourceCount(), time.Now().Format("2006-01-02")) 289 | // You can add some debug email address to ensure it works 290 | // debugAddressees := []string{"ben@example.com"} 291 | // mailData.SendEmail(getMailClient(c), c.config.EmailDomain, untaggedMailTemplate, title, debugAddressees...) 292 | mailData.SendEmail(getMailClient(c), c.config.EmailDomain, untaggedMailTemplate, title) 293 | } 294 | } 295 | } 296 | 297 | // DeletionWarning will find resources which are about to be deleted within 298 | // `hoursInAdvance` hours, and send an email to the owner of those resources 299 | // with a warning. Resources explicitly tagged to be deleted are not included 300 | // in this warning. 301 | func (c *Client) DeletionWarning(hoursInAdvance int, mngr cloud.ResourceManager, accountUserMapping map[string]string) { 302 | allCompute := mngr.AllResourcesPerAccount() 303 | allBuckets := mngr.BucketsPerAccount() 304 | for account, resources := range allCompute { 305 | ownerName := convertEmailExceptions(accountUserMapping[account]) 306 | fil := filter.New() 307 | fil.AddGeneralRule(filter.DeleteWithinXHours(hoursInAdvance)) 308 | mailData := resourceMailData{ 309 | ownerName, 310 | account, 311 | filter.Instances(resources.Instances, fil), 312 | filter.Images(resources.Images, fil), 313 | filter.Snapshots(resources.Snapshots, fil), 314 | filter.Volumes(resources.Volumes, fil), 315 | []cloud.Bucket{}, 316 | hoursInAdvance, 317 | } 318 | if buckets, ok := allBuckets[account]; ok { 319 | mailData.Buckets = filter.Buckets(buckets, fil) 320 | } 321 | 322 | if mailData.ResourceCount() > 0 { 323 | // Send email 324 | title := fmt.Sprintf("Deletion warning, %d resources are cleaned up within %d hours", mailData.ResourceCount(), hoursInAdvance) 325 | mailData.SendEmail(getMailClient(c), c.config.EmailDomain, deletionWarningTemplate, title) 326 | } 327 | } 328 | } 329 | 330 | // MonthToDateReport sends an email to engineering with the 331 | // Month-to-Date billing report 332 | func (c *Client) MonthToDateReport(report billing.Report, accountUserMapping map[string]string, sortedByTags bool) { 333 | mailClient := getMailClient(c) 334 | var sorted billing.UserList 335 | if sortedByTags { 336 | sorted = report.SortedTagsByTotalCost() 337 | } else { 338 | sorted = report.SortedUsersByTotalCost() 339 | } 340 | reportData := monthToDateData{report.CSP, report.TotalCost(), sorted, billing.MinimumTotalCost, billing.MinimumCost, accountUserMapping} 341 | mailContent, err := generateMail(reportData, monthToDateTemplate) 342 | if err != nil { 343 | log.Fatalln("Could not generate email:", err) 344 | } 345 | billingReportMail := fmt.Sprintf("%s@%s", c.config.BillingReportAddressee, c.config.EmailDomain) 346 | recipientMail := convertEmailExceptions(billingReportMail) 347 | log.Printf("Sending the Month-to-date report to %s\n", recipientMail) 348 | title := fmt.Sprintf("Month-to-date %s billing report", report.CSP) 349 | err = mailClient.SendEmail(title, mailContent, recipientMail) 350 | if err != nil { 351 | log.Printf("Failed to email %s: %s\n", recipientMail, err) 352 | } 353 | } 354 | 355 | // MarkingDryRunReport will send an email with all the resources that would have been marked for deletion 356 | func (c *Client) MarkingDryRunReport(taggedResources map[string]*cloud.AllResourceCollection, accountUserMapping map[string]string) { 357 | for account, resources := range taggedResources { 358 | // Use a debug user here 359 | mailData := resourceMailData{ 360 | Owner: "cloudsweeper-test", 361 | OwnerID: account, 362 | Instances: resources.Instances, 363 | Images: resources.Images, 364 | Snapshots: resources.Snapshots, 365 | Volumes: resources.Volumes, 366 | Buckets: resources.Buckets, 367 | } 368 | 369 | if mailData.ResourceCount() > 0 { 370 | // Send email 371 | title := fmt.Sprintf("Marking Dry Run Warning. The following resources would have been marked for deletion:") 372 | mailData.SendEmail(getMailClient(c), c.config.EmailDomain, markingDryRunTemplate, title) 373 | } 374 | } 375 | } 376 | -------------------------------------------------------------------------------- /cloudsweeper/organization.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | // Package cloudsweeper is where all the 'sweeping logic is defined. Such 5 | // as what to notify about and what to clean up. For using Cloudsweeper in 6 | // larger organizations, the Organization structure was implemeted. 7 | package cloudsweeper 8 | 9 | import ( 10 | "encoding/json" 11 | "fmt" 12 | 13 | "github.com/cloudtools/cloudsweeper/cloud" 14 | ) 15 | 16 | // Organization represents the employees, their departments, and their managers 17 | // within an organization. This structure was set up for an org wherein all 18 | // employees have their own cloud accounts, and are aggregated under a single 19 | // payer account. In the case you have only a single account, this will be 20 | // superfluous. 21 | type Organization struct { 22 | Managers Employees `json:"-"` 23 | ManagerIDs []managerID `json:"managers"` 24 | Departments Departments `json:"departments"` 25 | Employees Employees `json:"employees"` 26 | 27 | managerMapping map[string]*Employee 28 | departmentMapping map[string]*Department 29 | employeeMapping map[string]*Employee 30 | managerEmployees map[string]Employees 31 | } 32 | 33 | type managerID struct { 34 | ID string `json:"username"` 35 | } 36 | 37 | // Department represents a department in your org 38 | type Department struct { 39 | Number int `json:"number"` 40 | ID string `json:"id"` 41 | Name string `json:"name"` 42 | } 43 | 44 | // Departments is a list of Department 45 | type Departments []*Department 46 | 47 | // Employee represents an employee, which 48 | // belong to a department and has a manager. An employee can 49 | // also have multiple accounts and projects associated with 50 | // them in AWS and GCP. "Disabled" employees are employees 51 | // who should no longer be regarded as active in the company 52 | type Employee struct { 53 | Username string `json:"username"` 54 | RealName string `json:"real_name"` 55 | ManagerID string `json:"manager"` 56 | Manager *Employee `json:"-"` 57 | DepartmentID string `json:"department"` 58 | Department *Department `json:"-"` 59 | Disabled bool `json:"disabled,omitempty"` 60 | AWSAccounts AWSAccounts `json:"aws_accounts"` 61 | GCPProjects GCPProjects `json:"gcp_projects"` 62 | } 63 | 64 | // Employees is a list of Employee 65 | type Employees []*Employee 66 | 67 | // AWSAccount represents an account in AWS. An account 68 | // can have automatic cleanup enabled, indiacated by 69 | // the CloudsweeperEnabled attribute. 70 | type AWSAccount struct { 71 | ID string `json:"id"` 72 | CloudsweeperEnabled bool `json:"cloudsweeper_enabled,omitempty"` 73 | } 74 | 75 | // AWSAccounts is a list of AWSAccount 76 | type AWSAccounts []*AWSAccount 77 | 78 | // GCPProject represents a project in GPC. A project 79 | // can have automatic cleanup enabled, indiacated by 80 | // the CloudsweeperEnabled attribute. 81 | type GCPProject struct { 82 | ID string `json:"id"` 83 | CloudsweeperEnabled bool `json:"cloudsweeper_enabled,omitempty"` 84 | } 85 | 86 | // GCPProjects is a list of GCPProject 87 | type GCPProjects []*GCPProject 88 | 89 | // InitOrganization initializes an organisation from raw data, 90 | // e.g. the contents of a JSON file. 91 | func InitOrganization(orgData []byte) (*Organization, error) { 92 | org := new(Organization) 93 | err := json.Unmarshal(orgData, org) 94 | if err != nil { 95 | return nil, err 96 | } 97 | org.departmentMapping = make(map[string]*Department, len(org.Departments)) 98 | for i := range org.Departments { 99 | org.departmentMapping[org.Departments[i].ID] = org.Departments[i] 100 | } 101 | // First initalize all employees 102 | org.employeeMapping = make(map[string]*Employee, len(org.Employees)) 103 | for i := range org.Employees { 104 | org.employeeMapping[org.Employees[i].Username] = org.Employees[i] 105 | if department, exist := org.departmentMapping[org.Employees[i].DepartmentID]; exist { 106 | org.Employees[i].Department = department 107 | } else { 108 | // TODO: Fail if employee's department doesn't exist 109 | } 110 | } 111 | // Then map the employees' managers 112 | org.managerMapping = make(map[string]*Employee, len(org.Managers)) 113 | org.Managers = Employees{} 114 | for i := range org.ManagerIDs { 115 | if manager, exist := org.employeeMapping[org.ManagerIDs[i].ID]; exist { 116 | org.managerMapping[org.ManagerIDs[i].ID] = manager 117 | } else { 118 | // A manager doesn't have an record in the employee list 119 | return nil, fmt.Errorf("Manager %s is not in the list of employees", org.ManagerIDs[i]) 120 | } 121 | org.Managers = append(org.Managers, org.employeeMapping[org.ManagerIDs[i].ID]) 122 | } 123 | org.managerEmployees = make(map[string]Employees, len(org.Managers)) 124 | for i := range org.Employees { 125 | if manager, exist := org.managerMapping[org.Employees[i].ManagerID]; exist { 126 | org.Employees[i].Manager = manager 127 | org.managerEmployees[manager.Username] = append(org.managerEmployees[manager.Username], org.Employees[i]) 128 | } else { 129 | // TODO: Fail if employee's manager doesn't exist 130 | } 131 | } 132 | return org, nil 133 | } 134 | 135 | // EmployeesForManager gets all the employees who has the 136 | // specifed manager as their manager. 137 | func (org *Organization) EmployeesForManager(manager *Employee) (Employees, error) { 138 | if _, isManager := org.managerMapping[manager.Username]; !isManager { 139 | return nil, fmt.Errorf("%s is not a manager", manager.Username) 140 | } 141 | if employees, exist := org.managerEmployees[manager.Username]; exist { 142 | return employees, nil 143 | } 144 | // Manager has no employees 145 | return Employees{}, nil 146 | } 147 | 148 | // EnabledAccounts will return a list of all cloudsweeper enabled accounts 149 | // in the specified CSP 150 | func (org *Organization) EnabledAccounts(csp cloud.CSP) []string { 151 | accounts := []string{} 152 | for _, employee := range org.Employees { 153 | switch csp { 154 | case cloud.AWS: 155 | for _, account := range employee.AWSAccounts { 156 | if account.CloudsweeperEnabled { 157 | accounts = append(accounts, account.ID) 158 | } 159 | } 160 | case cloud.GCP: 161 | for _, project := range employee.GCPProjects { 162 | if project.CloudsweeperEnabled { 163 | accounts = append(accounts, project.ID) 164 | } 165 | } 166 | } 167 | } 168 | return accounts 169 | } 170 | 171 | // AccountToUserMapping is a helper method that maps accounts to their owners 172 | // username. This is useful for sending out emails to the owner of an account. 173 | func (org *Organization) AccountToUserMapping(csp cloud.CSP) map[string]string { 174 | result := make(map[string]string) 175 | for _, employee := range org.Employees { 176 | switch csp { 177 | case cloud.AWS: 178 | for _, account := range employee.AWSAccounts { 179 | result[account.ID] = employee.Username 180 | } 181 | case cloud.GCP: 182 | for _, project := range employee.GCPProjects { 183 | result[project.ID] = employee.Username 184 | } 185 | } 186 | } 187 | return result 188 | } 189 | 190 | // UsernameToEmployeeMapping is a helper method that returns a map of username to Employee struct. 191 | func (org *Organization) UsernameToEmployeeMapping() map[string]*Employee { 192 | return org.employeeMapping 193 | } 194 | -------------------------------------------------------------------------------- /cloudsweeper/setup/aws.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package setup 5 | 6 | import ( 7 | "encoding/json" 8 | "errors" 9 | "fmt" 10 | "log" 11 | "math/rand" 12 | "os" 13 | "time" 14 | 15 | "github.com/aws/aws-sdk-go/aws" 16 | "github.com/aws/aws-sdk-go/aws/awserr" 17 | "github.com/aws/aws-sdk-go/aws/session" 18 | "github.com/aws/aws-sdk-go/service/iam" 19 | ) 20 | 21 | const awsInfo = ` 22 | Cloudsweeper can be used to monitor your AWS account 23 | in order to keep resource usage and cost down for you. 24 | 25 | Running this setup will give Cloudsweeper access to 26 | EC2 and S3 in order to perform monitoring and cleanup. 27 | ` 28 | 29 | // This allows a role to be assumed by the houskeeper user in the shared QA AWS account 30 | const awsAssumeRoleDoc = `{ 31 | "Version": "2012-10-17", 32 | "Statement": [ 33 | { 34 | "Effect": "Allow", 35 | "Principal": { 36 | "AWS": "%s" 37 | }, 38 | "Action": "sts:AssumeRole" 39 | } 40 | ] 41 | }` 42 | 43 | const ( 44 | roleName = "Cloudsweeper" 45 | policyName = "CloudsweeperPolicy" 46 | policyDesc = "Allow Cloudsweeper to access your resources" 47 | 48 | awsPolicyOrRoleExist = "EntityAlreadyExists" 49 | 50 | awsPolicyARNTemplate = "arn:aws:iam::%s:policy/%s" 51 | 52 | awsIDKey = "AWS_ACCESS_KEY_ID" 53 | awsSecretKey = "AWS_SECRET_ACCESS_KEY" 54 | ) 55 | 56 | var ( 57 | monitorEC2 = []string{"ec2:DescribeInstances", "ec2:DescribeInstanceAttribute", "ec2:DescribeSnapshots", "ec2:DescribeVolumeStatus", "ec2:DescribeVolumes", "ec2:DescribeInstanceStatus", "ec2:DescribeTags", "ec2:DescribeVolumeAttribute", "ec2:DescribeImages", "ec2:DescribeSnapshotAttribute"} 58 | monitorS3 = []string{"s3:GetBucketTagging", "s3:ListBucket", "s3:GetObject", "s3:ListAllMyBuckets", "s3:GetBucketLocation", "cloudwatch:GetMetricStatistics"} 59 | 60 | cleanupEC2 = []string{"ec2:DeregisterImage", "ec2:DeleteSnapshot", "ec2:DeleteTags", "ec2:ModifyImageAttribute", "ec2:DeleteVolume", "ec2:TerminateInstances", "ec2:CreateTags", "ec2:StopInstances"} 61 | cleanupS3 = []string{"s3:PutBucketTagging", "s3:DeleteObject", "s3:DeleteBucket"} 62 | 63 | errPolicyExist = errors.New("A policy with the same name already exist") 64 | errRoleExist = errors.New("A role with the same name already exist") 65 | errSkipAWS = errors.New("Don't override AWS settings") 66 | ) 67 | 68 | func awsSetup(masterARN string) error { 69 | fmt.Println("Performing AWS setup...") 70 | 71 | _, idExist := os.LookupEnv(awsIDKey) 72 | _, secretExist := os.LookupEnv(awsSecretKey) 73 | if !idExist || !secretExist { 74 | return errors.New("No AWS credentials exist") 75 | } 76 | fmt.Println(awsInfo) 77 | 78 | // Get user preferences 79 | conf := getAWSConf() 80 | if !conf.cleanup && !conf.monitor { 81 | fmt.Println("Skipping AWS setup...") 82 | return nil 83 | } 84 | 85 | sess := session.Must(session.NewSession()) 86 | iamClient := iam.New(sess, &aws.Config{}) 87 | 88 | // First create a policy based on what the user configured 89 | policy, err := createAWSPolicy(policyName, conf, iamClient) 90 | if err == errPolicyExist { 91 | // A policy already exist create a new policy with random suffix 92 | rand.Seed(time.Now().UnixNano()) 93 | newPolicyName := fmt.Sprintf("%s-%d", policyName, rand.Int63()) 94 | policy, err = createAWSPolicy(newPolicyName, conf, iamClient) 95 | if err != nil { 96 | return fmt.Errorf("Failed to create policy: %s", err) 97 | } 98 | } else if err != nil { 99 | return err 100 | } 101 | fmt.Printf("Created new policy:\n\t%s\n", *policy.Arn) 102 | 103 | // Now create role 104 | role, err := createAWSRole(masterARN, roleName, iamClient) 105 | if err == errRoleExist { 106 | // A role already exist, replace it 107 | err = deleteAWSRole(roleName, iamClient) 108 | if err != nil { 109 | return fmt.Errorf("Failed to delete old role: %s", err) 110 | } 111 | role, err = createAWSRole(masterARN, roleName, iamClient) 112 | if err != nil { 113 | return fmt.Errorf("Could not create Cloudsweeper role: %s", err) 114 | } 115 | } else if err != nil { 116 | return err 117 | } 118 | fmt.Printf("Created new role:\n\t%s\n", *role.Arn) 119 | 120 | // Finally connect the policy to the role 121 | _, err = iamClient.AttachRolePolicy(&iam.AttachRolePolicyInput{ 122 | RoleName: (*role).RoleName, 123 | PolicyArn: (*policy).Arn, 124 | }) 125 | if err != nil { 126 | return fmt.Errorf("Could not attach Cloudsweeper policy to Cloudsweeper role: %s", err) 127 | } 128 | return nil 129 | } 130 | 131 | func getAWSConf() *config { 132 | conf := new(config) 133 | if !getYes("Allow Cloudsweeper to monitor and cleanup?", true) { 134 | return conf 135 | } 136 | // Don't let the user choose what to allow, per request 137 | conf.monitorEC2 = true 138 | conf.monitorS3 = true 139 | conf.cleanupEC2 = true 140 | conf.cleanupS3 = true 141 | conf.monitor = true 142 | conf.cleanup = true 143 | /* 144 | conf.monitor = getYes("Setup monitoring (read) of your resources?", true) 145 | if conf.monitor { 146 | conf.monitorEC2 = getYes("\tMonitor EC2:", true) 147 | conf.monitorS3 = getYes("\tMonitor S3", true) 148 | } 149 | if !conf.monitor { 150 | return conf 151 | } 152 | conf.cleanup = getYes("Setup cleanup (read & write) of your resources?", true) 153 | if conf.cleanup { 154 | conf.cleanupEC2 = getYes("\tCleanup EC2:", true) 155 | conf.cleanupS3 = getYes("\tCleanup S3", true) 156 | } 157 | */ 158 | return conf 159 | } 160 | 161 | func deleteAWSRole(name string, iamClient *iam.IAM) error { 162 | // First detach all attached policies 163 | out, err := iamClient.ListAttachedRolePolicies(&iam.ListAttachedRolePoliciesInput{ 164 | RoleName: aws.String(name), 165 | }) 166 | if err != nil { 167 | return err 168 | } 169 | for _, pol := range out.AttachedPolicies { 170 | _, err := iamClient.DetachRolePolicy(&iam.DetachRolePolicyInput{ 171 | RoleName: aws.String(name), 172 | PolicyArn: pol.PolicyArn, 173 | }) 174 | if err != nil { 175 | return err 176 | } 177 | } 178 | 179 | // Now actually delete the role 180 | _, err = iamClient.DeleteRole(&iam.DeleteRoleInput{ 181 | RoleName: aws.String(name), 182 | }) 183 | return err 184 | } 185 | 186 | func createAWSRole(masterARN, name string, iamClient *iam.IAM) (*iam.Role, error) { 187 | // Create a new role that can be assumed by Cloudsweeper 188 | policyDoc := fmt.Sprintf(awsAssumeRoleDoc, masterARN) 189 | input := &iam.CreateRoleInput{ 190 | AssumeRolePolicyDocument: aws.String(policyDoc), 191 | Description: aws.String(policyDesc), 192 | RoleName: aws.String(roleName), 193 | } 194 | out, err := iamClient.CreateRole(input) 195 | if err != nil { 196 | // Role might already exist 197 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == awsPolicyOrRoleExist { 198 | return nil, errRoleExist 199 | } 200 | // Other error 201 | return nil, err 202 | } 203 | return out.Role, nil 204 | } 205 | 206 | func createAWSPolicy(name string, conf *config, iamClient *iam.IAM) (*iam.Policy, error) { 207 | input := &iam.CreatePolicyInput{ 208 | Description: aws.String(policyDesc), 209 | PolicyName: aws.String(name), 210 | PolicyDocument: aws.String(conf.PolicyJSON()), 211 | } 212 | out, err := iamClient.CreatePolicy(input) 213 | if err != nil { 214 | // Could be that the policy already exist 215 | if aerr, ok := err.(awserr.Error); ok && aerr.Code() == awsPolicyOrRoleExist { 216 | return nil, errPolicyExist 217 | } 218 | // Other error 219 | return nil, err 220 | } 221 | // Can't return (out.Policy, err) directly as out might == nil if err != nil 222 | return out.Policy, nil 223 | } 224 | 225 | type config struct { 226 | monitor, monitorEC2, monitorS3 bool 227 | cleanup, cleanupEC2, cleanupS3 bool 228 | } 229 | 230 | func (c config) String() string { 231 | template := ` 232 | 233 | ### Cloudsweeper configuration ### 234 | 235 | Allow monitoring of resources: %t 236 | EC2: %t 237 | S3: %t 238 | 239 | Allow cleanup of resource: %t 240 | EC2: %t 241 | S3: %t 242 | 243 | ` 244 | return fmt.Sprintf(template, c.monitor, c.monitorEC2, c.monitorS3, c.cleanup, c.cleanupEC2, c.cleanupS3) 245 | } 246 | 247 | type policyStatement struct { 248 | Sid string 249 | Effect string 250 | Action []string 251 | Resource string 252 | } 253 | 254 | type policyDocument struct { 255 | Version string 256 | Statement []policyStatement 257 | } 258 | 259 | func (c config) Policy() policyDocument { 260 | actionSet := make(map[string]struct{}) 261 | if c.monitorEC2 || c.cleanupEC2 { 262 | for i := range monitorEC2 { 263 | actionSet[monitorEC2[i]] = struct{}{} 264 | } 265 | } 266 | if c.monitorS3 || c.cleanupS3 { 267 | for i := range monitorS3 { 268 | actionSet[monitorS3[i]] = struct{}{} 269 | } 270 | } 271 | if c.cleanupEC2 { 272 | for i := range cleanupEC2 { 273 | actionSet[cleanupEC2[i]] = struct{}{} 274 | } 275 | } 276 | if c.cleanupS3 { 277 | for i := range cleanupS3 { 278 | actionSet[cleanupS3[i]] = struct{}{} 279 | } 280 | } 281 | 282 | doc := policyDocument{} 283 | statement := policyStatement{} 284 | statement.Action = []string{} 285 | 286 | for action := range actionSet { 287 | statement.Action = append(statement.Action, action) 288 | } 289 | 290 | statement.Effect = "Allow" 291 | statement.Resource = "*" 292 | statement.Sid = "VisualEditor0" 293 | 294 | doc.Version = "2012-10-17" 295 | doc.Statement = []policyStatement{statement} 296 | return doc 297 | } 298 | 299 | func (c config) PolicyJSON() string { 300 | doc := c.Policy() 301 | b, err := json.Marshal(doc) 302 | if err != nil { 303 | log.Fatalln("Failed to encode AWS policy") 304 | } 305 | return string(b) 306 | } 307 | -------------------------------------------------------------------------------- /cloudsweeper/setup/setup.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package setup 5 | 6 | import ( 7 | "bufio" 8 | "fmt" 9 | "log" 10 | "os" 11 | "strings" 12 | ) 13 | 14 | // PerformSetup will start setting up Cloudsweeper for the user. 15 | func PerformSetup(awsMasterARN string) { 16 | fmt.Println("Welcome to Cloudsweeper, performing account setup...") 17 | 18 | err := awsSetup(awsMasterARN) 19 | if err != nil { 20 | fmt.Printf("AWS setup failed: %s\n", err) 21 | os.Exit(1) 22 | } 23 | fmt.Println(` 24 | SUCCESS 25 | 26 | Nothing else to setup, all done! :)`) 27 | } 28 | 29 | func getYes(prompt string, yesDefault bool) bool { 30 | reader := bufio.NewReader(os.Stdin) 31 | if yesDefault { 32 | prompt = fmt.Sprintf("%s (Y/n): ", prompt) 33 | } else { 34 | prompt = fmt.Sprintf("%s (y/N): ", prompt) 35 | } 36 | fmt.Print(prompt) 37 | input, err := reader.ReadString('\n') 38 | if err != nil { 39 | log.Fatalln(err) 40 | } 41 | input = strings.TrimSpace(strings.ToLower(input)) 42 | if input == "" { 43 | return yesDefault 44 | } 45 | return strings.Contains(input, "y") 46 | } 47 | -------------------------------------------------------------------------------- /cmd/cloudsweeper/config.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package main 5 | 6 | import ( 7 | "flag" 8 | "log" 9 | "strconv" 10 | 11 | "github.com/joho/godotenv" 12 | ) 13 | 14 | const optionalDefault = "" 15 | 16 | type lookup struct { 17 | confKey string 18 | defaultValue string 19 | } 20 | 21 | var configMapping = map[string]lookup{ 22 | // General variables 23 | "csp": lookup{"CS_CSP", "aws"}, 24 | "org-file": lookup{"CS_ORG_FILE", "organization.json"}, 25 | 26 | // Billing related 27 | "billing-account": lookup{"CS_BILLING_ACCOUNT", ""}, 28 | "billing-bucket-region": lookup{"CS_BILLING_BUCKET_REGION", ""}, 29 | "billing-csv-prefix": lookup{"CS_BILLING_CSV_PREFIX", ""}, 30 | "billing-bucket": lookup{"CS_BILLING_BUCKET_NAME", ""}, 31 | "billing-sort-tag": lookup{"CS_BILLING_SORT_TAG", optionalDefault}, 32 | 33 | // Email variables 34 | "smtp-username": lookup{"CS_SMTP_USER", ""}, 35 | "smtp-password": lookup{"CS_SMTP_PASSWORD", ""}, 36 | "smtp-server": lookup{"CS_SMTP_SERVER", ""}, 37 | "smtp-port": lookup{"CS_SMTP_PORT", "587"}, 38 | 39 | // Notifying specific variables 40 | "warning-hours": lookup{"CS_WARNING_HOURS", "48"}, 41 | "display-name": lookup{"CS_DISPLAY_NAME", "Cloudsweeper"}, 42 | "mail-from": lookup{"CS_MAIL_FROM", ""}, 43 | "billing-report-addressee": lookup{"CS_BILLING_REPORT_ADDRESSEE", ""}, 44 | "total-sum-addressee": lookup{"CS_TOTAL_SUM_ADDRESSEE", ""}, 45 | "mail-domain": lookup{"CS_EMAIL_DOMAIN", ""}, 46 | 47 | // Setup variables 48 | "aws-master-arn": lookup{"CS_MASTER_ARN", ""}, 49 | 50 | // Clean thresholds 51 | "clean-untagged-older-than-days": lookup{"CLEAN_UNTAGGED_OLDER_THAN_DAYS", "30"}, 52 | "clean-instances-older-than-days": lookup{"CLEAN_INSTANCES_OLDER_THAN_DAYS", "182"}, 53 | "clean-images-older-than-days": lookup{"CLEAN_IMAGES_OLDER_THAN_DAYS", "182"}, 54 | "clean-snapshots-older-than-days": lookup{"CLEAN_SNAPSHOTS_OLDER_THAN_DAYS", "182"}, 55 | "clean-unattatched-older-than-days": lookup{"CLEAN_UNATTATCHED_OLDER_THAN_DAYS", "30"}, 56 | "clean-bucket-not-modified-days": lookup{"CLEAN_BUCKET_NOT_MODIFIED_DAYS", "182"}, 57 | "clean-bucket-older-than-days": lookup{"CLEAN_BUCKET_OLDER_THAN_DAYS", "7"}, 58 | "clean-keep-n-component-images": lookup{"CLEAN_KEEP_N_COMPONENT_IMAGES", "2"}, 59 | 60 | // Notify thresholds 61 | "notify-untagged-older-than-days": lookup{"NOTIFY_UNTAGGED_OLDER_THAN_DAYS", "14"}, 62 | "notify-instances-older-than-days": lookup{"NOTIFY_INSTANCES_OLDER_THAN_DAYS", "30"}, 63 | "notify-images-older-than-days": lookup{"NOTIFY_IMAGES_OLDER_THAN_DAYS", "30"}, 64 | "notify-unattached-older-than-days": lookup{"NOTIFY_UNATTATCHED_OLDER_THAN_DAYS", "30"}, 65 | "notify-snapshots-older-than-days": lookup{"NOTIFY_SNAPSHOTS_OLDER_THAN_DAYS", "30"}, 66 | "notify-buckets-older-than-days": lookup{"NOTIFY_BUCKETS_OLDER_THAN_DAYS", "30"}, 67 | "notify-whitelist-older-than-days": lookup{"NOTIFY_WHITELIST_OLDER_THAN_DAYS", "182"}, 68 | "notify-dnd-older-than-days": lookup{"NOTIFY_DND_OLDER_THAN_DAYS", "7"}, 69 | } 70 | 71 | func loadConfig() { 72 | var err error 73 | config, err = godotenv.Read(configFileName) 74 | if err != nil { 75 | log.Fatalf("Could not load config file '%s': %s", configFileName, err) 76 | } 77 | } 78 | 79 | func loadThresholds() { 80 | for _, v := range thnames { 81 | thresholds[v] = findConfigInt(v) 82 | } 83 | } 84 | 85 | func findConfig(name string) string { 86 | if _, exist := configMapping[name]; !exist { 87 | log.Fatalf("Unknown config option: %s", name) 88 | } 89 | flagVal := flag.Lookup(name).Value.String() 90 | if flagVal != "" { 91 | return flagVal 92 | } else if confVal, ok := config[configMapping[name].confKey]; ok && confVal != "" { 93 | maybeNoValExit(confVal, name) 94 | return confVal 95 | } else { 96 | defaultVal := configMapping[name].defaultValue 97 | if defaultVal == optionalDefault { 98 | return "" 99 | } 100 | maybeNoValExit(defaultVal, name) 101 | return defaultVal 102 | } 103 | } 104 | 105 | func maybeNoValExit(val, name string) { 106 | if val == "" { 107 | log.Fatalf("No value specified for --%s", name) 108 | } 109 | } 110 | 111 | func findConfigInt(name string) int { 112 | val := findConfig(name) 113 | i, err := strconv.Atoi(val) 114 | if err != nil { 115 | log.Fatalf("Value specified for %s is not an integer", name) 116 | } 117 | return i 118 | } 119 | -------------------------------------------------------------------------------- /cmd/cloudsweeper/main.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | package main 5 | 6 | import ( 7 | "flag" 8 | "fmt" 9 | "io/ioutil" 10 | "log" 11 | "os" 12 | "strings" 13 | 14 | "github.com/cloudtools/cloudsweeper/cloud" 15 | "github.com/cloudtools/cloudsweeper/cloud/billing" 16 | cs "github.com/cloudtools/cloudsweeper/cloudsweeper" 17 | "github.com/cloudtools/cloudsweeper/cloudsweeper/cleanup" 18 | "github.com/cloudtools/cloudsweeper/cloudsweeper/find" 19 | "github.com/cloudtools/cloudsweeper/cloudsweeper/notify" 20 | "github.com/cloudtools/cloudsweeper/cloudsweeper/setup" 21 | ) 22 | 23 | const ( 24 | configFileName = "config.conf" 25 | cspFlagAWS = "aws" 26 | cspFlagGCP = "gcp" 27 | ) 28 | 29 | var ( 30 | config map[string]string 31 | 32 | cspToUse = flag.String("csp", "", "Which CSP to run against") 33 | orgFile = flag.String("org-file", "", "Specify where to find the JSON with organization information") 34 | 35 | awsBillingAccount = flag.String("billing-account", "", "Specify AWS billing account id (e.g. 1234661312)") 36 | awsBillingBucketRegion = flag.String("billing-bucket-region", "", "Specify AWS region where --billing-bucket is location") 37 | gcpBillingCSVPrefix = flag.String("billing-csv-prefix", "", "Specify name prefix of GCP billing CSV files") 38 | billingBucket = flag.String("billing-bucket", "", "Specify bucket with billing CSVs") 39 | awsBillingSortTag = flag.String("billing-sort-tag", "", "Specify a tag to sort on when creating report") 40 | 41 | mailUser = flag.String("smtp-username", "", "SMTP username used to send email") 42 | mailPassword = flag.String("smtp-password", "", "SMTP password used to send email") 43 | mailServer = flag.String("smtp-server", "", "SMTP server used to send mail") 44 | mailPort = flag.String("smtp-port", "", "SMTP port used to send mail") 45 | 46 | warningHours = flag.String("warning-hours", "", "The number of hours in advance to warn about resource deletion") 47 | displayName = flag.String("display-name", "", "Name displayed on emails sent by Cloudsweeper") 48 | mailFrom = flag.String("mail-from", "", "'From Email' displayed on emails sent by Cloudsweeper") 49 | billingReportReceiver = flag.String("billing-report-addressee", "", "Receiver of month to date billing report") 50 | summaryManager = flag.String("total-sum-addressee", "", "Receiver of total cost sums") 51 | mailDomain = flag.String("mail-domain", "", "The mail domain appended to usernames specified in the organization") 52 | 53 | setupARN = flag.String("aws-master-arn", "", "AWS ARN of role in account used by Cloudsweeper to assume roles") 54 | 55 | findResourceID = flag.String("resource-id", "", "ID of resource to find with find-resource command") 56 | 57 | dryRun = flag.Bool("marking-dry-run", false, "Whether to perform a dry run for mark and delete (nothing will actually be marked)") 58 | 59 | // Thresholds 60 | thresholds = make(map[string]int) 61 | thnames = []string{ 62 | "clean-untagged-older-than-days", 63 | "clean-instances-older-than-days", 64 | "clean-images-older-than-days", 65 | "clean-snapshots-older-than-days", 66 | "clean-unattatched-older-than-days", 67 | "clean-bucket-not-modified-days", 68 | "clean-bucket-older-than-days", 69 | "clean-keep-n-component-images", 70 | "notify-untagged-older-than-days", 71 | "notify-instances-older-than-days", 72 | "notify-images-older-than-days", 73 | "notify-unattached-older-than-days", 74 | "notify-snapshots-older-than-days", 75 | "notify-buckets-older-than-days", 76 | "notify-whitelist-older-than-days", 77 | "notify-dnd-older-than-days", 78 | } 79 | 80 | // Clean thresholds 81 | cleanUntaggedOlderThanDays = flag.String("clean-untagged-older-than-days", "", "Clean untagged resources if older than X days (default: 30)") 82 | cleanInstancesOlderThanDays = flag.String("clean-instances-older-than-days", "", "Clean if instance is older than X days (default: 182)") 83 | cleanImagesOlderThanDays = flag.String("clean-images-older-than-days", "", "Clean if image is older than X days (default: 182)") 84 | cleanSnapshotsOlderThanDays = flag.String("clean-snapshots-older-than-days", "", "Clean if snapshot is older than X days (default: 182)") 85 | cleanUnattatchedOlderThanDays = flag.String("clean-unattatched-older-than-days", "", "Clean unattached volumes older than X days (default: 30)") 86 | cleanBucketNotModifiedDays = flag.String("clean-bucket-not-modified-days", "", "Clean s3 bucket if not modified for more than X days (default: 182)") 87 | cleanBucketOlderThanDays = flag.String("clean-bucket-older-than-days", "", "Clean s3 bucket if older than X days (default: 7)") 88 | cleanKeepNComponentImages = flag.String("clean-keep-n-component-images", "", "Clean images with component-date naming that are older than the N most recent ones (default: 2)") 89 | 90 | // Notify thresholds 91 | notifyUntaggedOlderThanDays = flag.String("notify-untagged-older-than-days", "", "Notify if untagged resource is older than X days (default: 14)") 92 | notifyInstancesOlderThanDays = flag.String("notify-instances-older-than-days", "", "Notify if instances is older than X days (default: 30)") 93 | notifyImagesOlderThanDays = flag.String("notify-images-older-than-days", "", "Notify if image is older than X days (default: 30)") 94 | notifyVolumesOlderThanDays = flag.String("notify-unattached-older-than-days", "", "Notify if volume is older than X days (default: 30)") 95 | notifySnapshotsOlderThanDays = flag.String("notify-snapshots-older-than-days", "", "Notify if snapshot is older than X days (default: 30)") 96 | notifyBucketsOlderThanDays = flag.String("notify-buckets-older-than-days", "", "Notify if bucket is older than X days (default: 30)") 97 | notifyWhitelistOlderThanDays = flag.String("notify-whitelist-older-than-days", "", "Notify if whitelisted is older than X days (default: 182)") 98 | notifyDndOlderThanDays = flag.String("notify-dnd-older-than-days", "", "Do not delete older than X days (default: 7)") 99 | ) 100 | 101 | const banner = ` 102 | ___ _ _ 103 | / __\ | ___ _ _ __| |_____ _____ ___ _ __ ___ _ __ 104 | / / | |/ _ \| | | |/ _` + "`" + ` / __\ \ /\ / / _ \/ _ \ '_ \ / _ \ '__| 105 | / /___| | (_) | |_| | (_| \__ \\ V V / __/ __/ |_) | __/ | 106 | \____/|_|\___/ \__,_|\__,_|___/ \_/\_/ \___|\___| .__/ \___|_| 107 | |_| 108 | ` 109 | 110 | func main() { 111 | fmt.Println(banner) 112 | loadConfig() 113 | flag.Parse() 114 | loadThresholds() 115 | csp := cspFromConfig(findConfig("csp")) 116 | log.Printf("Running against %s...\n", csp) 117 | switch getPositionalCmd() { 118 | case "cleanup": 119 | log.Println("Cleaning up old resources") 120 | org := parseOrganization(findConfig("org-file")) 121 | mngr := initManager(csp, org) 122 | cleanup.PerformCleanup(mngr) 123 | case "reset": 124 | log.Println("Resetting all tags") 125 | org := parseOrganization(findConfig("org-file")) 126 | mngr := initManager(csp, org) 127 | cleanup.ResetCloudsweeper(mngr) 128 | case "mark-for-cleanup": 129 | log.Println("Marking old resources for cleanup") 130 | org := parseOrganization(findConfig("org-file")) 131 | mngr := initManager(csp, org) 132 | taggedResources := cleanup.MarkForCleanup(mngr, thresholds, *dryRun) 133 | if *dryRun { 134 | client := initNotifyClient() 135 | client.MarkingDryRunReport(taggedResources, org.AccountToUserMapping(csp)) 136 | } else { 137 | log.Println("Not sending marking report since this was not a dry run") 138 | } 139 | case "review": 140 | log.Println("Sending out old resource review") 141 | org := parseOrganization(findConfig("org-file")) 142 | mngr := initManager(csp, org) 143 | client := initNotifyClient() 144 | client.OldResourceReview(mngr, org, csp, thresholds) 145 | case "warn": 146 | log.Println("Sending out cleanup warning") 147 | org := parseOrganization(findConfig("org-file")) 148 | mngr := initManager(csp, org) 149 | client := initNotifyClient() 150 | client.DeletionWarning(findConfigInt("warning-hours"), mngr, org.AccountToUserMapping(csp)) 151 | case "billing-report": 152 | log.Println("Generating month-to-date billing report for", csp) 153 | var reporter billing.Reporter 154 | if csp == cloud.AWS { 155 | billingAccount := findConfig("billing-account") 156 | bucket := findConfig("billing-bucket") 157 | region := findConfig("billing-bucket-region") 158 | sortTag := findConfig("billing-sort-tag") 159 | reporter = billing.NewReporterAWS(billingAccount, bucket, region, sortTag) 160 | } else if csp == cloud.GCP { 161 | bucket := findConfig("billing-bucket") 162 | prefix := findConfig("billing-csv-prefix") 163 | reporter = billing.NewReporterGCP(bucket, prefix) 164 | } else { 165 | log.Fatalf("Invalid CSP specified") 166 | return 167 | } 168 | report := billing.GenerateReport(reporter) 169 | org := parseOrganization(findConfig("org-file")) 170 | mapping := org.AccountToUserMapping(csp) 171 | sortTagKey := findConfig("billing-sort-tag") 172 | log.Println(report.FormatReport(mapping, sortTagKey != "")) 173 | client := initNotifyClient() 174 | client.MonthToDateReport(report, mapping, sortTagKey != "") 175 | case "find-untagged": 176 | log.Println("Finding untagged resources") 177 | org := parseOrganization(findConfig("org-file")) 178 | mngr := initManager(csp, org) 179 | mapping := org.AccountToUserMapping(csp) 180 | client := initNotifyClient() 181 | client.UntaggedResourcesReview(mngr, mapping) 182 | case "find-resource": 183 | id := *findResourceID 184 | if id == "" { 185 | log.Fatalln("Must specify a resource ID to find, using --resource-id=") 186 | } 187 | log.Printf("Finding resource with ID %s", id) 188 | org := parseOrganization(findConfig("org-file")) 189 | mngr := initManager(csp, org) 190 | client, err := find.Init(mngr, org, csp) 191 | if err != nil { 192 | log.Fatalf("Could not initalize find client: %s", err) 193 | } 194 | err = client.FindResource(id) 195 | if err != nil { 196 | log.Fatal(err) 197 | } 198 | case "setup": 199 | log.Println("Running cloudsweeper setup") 200 | setup.PerformSetup(findConfig("aws-master-arn")) 201 | default: 202 | log.Fatalln("Please supply a command") 203 | } 204 | } 205 | 206 | func initManager(csp cloud.CSP, org *cs.Organization) cloud.ResourceManager { 207 | manager, err := cloud.NewManager(csp, org.EnabledAccounts(csp)...) 208 | if err != nil { 209 | log.Fatal(err) 210 | return nil 211 | } 212 | return manager 213 | } 214 | 215 | func initNotifyClient() *notify.Client { 216 | config := ¬ify.Config{ 217 | SMTPUsername: findConfig("smtp-username"), 218 | SMTPPassword: findConfig("smtp-password"), 219 | SMTPServer: findConfig("smtp-server"), 220 | SMTPPort: findConfigInt("smtp-port"), 221 | DisplayName: findConfig("display-name"), 222 | MailFrom: findConfig("mail-from"), 223 | EmailDomain: findConfig("mail-domain"), 224 | BillingReportAddressee: findConfig("billing-report-addressee"), 225 | TotalSumAddresse: findConfig("total-sum-addressee"), 226 | } 227 | return notify.Init(config) 228 | } 229 | 230 | func parseOrganization(inputFile string) *cs.Organization { 231 | raw, err := ioutil.ReadFile(inputFile) 232 | if err != nil { 233 | log.Fatalf("Could not read organization file: %s\n", err) 234 | } 235 | org, err := cs.InitOrganization(raw) 236 | if err != nil { 237 | log.Fatalf("Failed to initalize organization: %s\n", err) 238 | } 239 | return org 240 | } 241 | 242 | func getPositionalCmd() string { 243 | n := len(os.Args) 244 | if n <= 1 { 245 | return "" 246 | } 247 | return os.Args[n-1] 248 | } 249 | 250 | func cspFromConfig(rawFlag string) cloud.CSP { 251 | flagVal := strings.ToLower(rawFlag) 252 | switch flagVal { 253 | case cspFlagAWS: 254 | return cloud.AWS 255 | case cspFlagGCP: 256 | return cloud.GCP 257 | default: 258 | fmt.Fprintf(os.Stderr, "Invalid CSP flag \"%s\" specified\n", rawFlag) 259 | os.Exit(1) 260 | return cloud.AWS 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /config.conf: -------------------------------------------------------------------------------- 1 | ####################################################################### 2 | # Configuration of Cloudsweeper # 3 | ####################################################################### 4 | 5 | ######################### Generic configs ############################# 6 | # CS_CSP defines which CSP to run against. Can be either 7 | # 'aws' or 'gcp. Can be overridden using the '--csp' flag. 8 | CS_CSP: aws 9 | # CS_ORG_FILE defines the location of the organization 10 | # definition file. This can be any local path on the machine. 11 | CS_ORG_FILE: organization.json 12 | # CS_WARNING_HOURS defines when Cloudsweeper will start warning 13 | # about resource cleanup. If there is less than the specified amount 14 | # of hours left before a resource will be cleaned up, then an 15 | # email will be sent. Because of how this works, it is important to 16 | # run Cloudsweeper often enough so that a warning can be sent out. 17 | # Preferably once every day. 18 | CS_WARNING_HOURS: 48 19 | 20 | ########################## Billing configs ############################ 21 | # CS_BILLING_ACCOUNT defines the AWS account ID where the 22 | # billing report CSV is located. 23 | CS_BILLING_ACCOUNT: foo 24 | # CS_BILLING_BUCKET_NAME defines the name/id of the bucket where the 25 | # billing report file will be located. 26 | CS_BILLING_BUCKET_NAME: foo 27 | # CS_BILLING_BUCKET_REGION defines the AWS region where the bucket 28 | # specified by CS_BILLING_BUCKET_NAME is located. 29 | CS_BILLING_BUCKET_REGION: us-west-2 30 | # CS_BILLING_CSV_PREFIX defines the prefix of the billing report 31 | # file in GCP. This prefix will be appended with the date and 32 | # .csv file endding (e.g. -2018-10-09.csv). 33 | CS_BILLING_CSV_PREFIX: foo 34 | # CS_BILLING_SORT_TAG defines a tag in the AWS billing report CSV to 35 | # sort on. If this is left empty, sorting is done based on users. 36 | CS_BILLING_SORT_TAG: 37 | 38 | ########################### SMTP configs ############################## 39 | # CS_SMTP_USER defines the username used when authenticating with 40 | # the SMTP server to send mail. If using Gmail, this would be 41 | # the full email, e.g. example@gmail.com. 42 | CS_SMTP_USER: example@gmail.com 43 | # CS_SMTP_PASSWORD defines the password used when authenticating with 44 | # the SMTP server to send mail. 45 | CS_SMTP_PASSWORD: password 46 | # CS_SMTP_SERVER defines the server that will be used for sending 47 | # email. 48 | CS_SMTP_SERVER: smtp.gmail.com 49 | # CS_SMTP_PORT defines the port that will be used when connecting 50 | # to the SMTP server. 51 | CS_SMTP_PORT: 587 52 | 53 | ####################### Notification configs ########################## 54 | # CS_DISPLAY_NAME defines the name that will be shown as sender in the 55 | # mails sent by Cloudsweeper. 56 | CS_DISPLAY_NAME: Cloudsweeper 57 | # CS_MAIL_FROM defines the email address that will be shown as sender in the 58 | # mails sent by Cloudsweeper. This is often the same as CS_SMTP_USER 59 | CS_MAIL_FROM: example@gmail.com 60 | # CS_EMAIL_DOMAIN defines the domain used for email in your company. It 61 | # will be appended to the employee username like "@". 62 | # If you have an employee which has a different domain name than the 63 | # rest, you add it as an exception in the emailEdgeCases map within 64 | # "cloudsweeper/notify/helpers.go". 65 | CS_EMAIL_DOMAIN: example.com 66 | # CS_BILLING_REPORT_ADDRESSEE defines an employee/alias where the billing report 67 | # should be sent. This could perhaps be a common engineering email that 68 | # all engineers recieve. 69 | # e.g 'engineering' - then the full email address will be engineering@ 70 | CS_BILLING_REPORT_ADDRESSEE: engineering 71 | # CS_TOTAL_SUM_ADDRESSEE defines an employee/alias that should 72 | # get a total summary of all resources. This person is probably 73 | # the one responsible for cost management within your company. 74 | # e.g 'cogs' - then the full email address will be cogs@ 75 | CS_TOTAL_SUM_ADDRESSEE: cogs 76 | 77 | ########################## Setup configs ############################## 78 | # CS_MASTER_ARN defines the ARN of the AWS IAM user within an account 79 | # that is used by the master machine, as descibed in Instructions.md. 80 | CS_MASTER_ARN: arn:aws:iam::123456789123:user/cloudsweeper-master 81 | 82 | 83 | ########################## Thresholds ############################## 84 | # CLEAN_UNTAGGED_OLDER_THAN_DAYS defines the number of days before an untagged instance is cleaned up 85 | # CLEAN_UNTAGGED_OLDER_THAN_DAYS: 30 86 | # CLEAN_INSTANCES_OLDER_THAN_DAYS defines the number of days before an instance is cleaned up 87 | # CLEAN_INSTANCES_OLDER_THAN_DAYS: 180 88 | # CLEAN_IMAGES_OLDER_THAN_DAYS defines the number of days before an instance is cleaned up 89 | # CLEAN_IMAGES_OLDER_THAN_DAYS: 180 90 | # CLEAN_SNAPSHOTS_OLDER_THAN_DAYS defines the number of days before an instance is cleaned up 91 | # CLEAN_SNAPSHOTS_OLDER_THAN_DAYS: 180 92 | # CLEAN_UNATTATCHED_OLDER_THAN_DAYS defines the number of days before an unattached volume is cleaned up 93 | # CLEAN_UNATTATCHED_OLDER_THAN_DAYS: 30 94 | # CLEAN_BUCKET_NOT_MODIFIED_DAYS defines the number of days that an S3 bucket must be idle for before cleanup occours 95 | # CLEAN_BUCKET_NOT_MODIFIED_DAYS: 182 96 | # CLEAN_BUCKET_OLDER_THAN_DAYS defines the number of days than an S3 bucket must exist for before being cleaned up 97 | # CLEAN_BUCKET_OLDER_THAN_DAYS: 7 98 | # CLEAN_KEEP_N_COMPONENT_IMAGES defines the number of latest component images to clean. All but the N most recent will be cleanup up 99 | # CLEAN_KEEP_N_COMPONENT_IMAGES: 2 100 | 101 | # NOTIFY_INSTANCES_OLDER_THAN_DAYS defines the number of days before notifications are sent out for instances 102 | # NOTIFY_INSTANCES_OLDER_THAN_DAYS: 30 103 | # NOTIFY_IMAGES_OLDER_THAN_DAYS defines the number of days before notifications are sent out for images 104 | # NOTIFY_IMAGES_OLDER_THAN_DAYS: 30 105 | # NOTIFY_VOLUMES_OLDER_THAN_DAYS defines the number of days before notifications are sent out for volumes 106 | # NOTIFY_VOLUMES_OLDER_THAN_DAYS: 30 107 | # NOTIFY_SNAPSHOTS_OLDER_THAN_DAYS defines the number of days before notifications are sent out for snapshots 108 | # NOTIFY_SNAPSHOTS_OLDER_THAN_DAYS: 30 109 | # NOTIFY_BUCKETS_OLDER_THAN_DAYS defines the number of days before notifications are sent out for buckets 110 | # NOTIFY_BUCKETS_OLDER_THAN_DAYS: 30 111 | # NOTIFY_WHITELIST_OLDER_THAN_DAYS defines the number of days before notifications are sent out for whitelisted items 112 | # NOTIFY_WHITELIST_OLDER_THAN_DAYS: 180 113 | # NOTIFY_DND_OLDER_THAN_DAYS defines the number of days that a Do Not Destroy tag must exist for before sending out a notification 114 | # NOTIFY_DND_OLDER_THAN_DAYS: 7 115 | -------------------------------------------------------------------------------- /mailer/mailer.go: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2018 VMware, Inc. All Rights Reserved. 2 | // SPDX-License-Identifier: BSD-2-Clause 3 | 4 | // Package mailer is a utility to send email. Configuration is not within 5 | // the scope of this package, it simply takes an SMTP server, port, 6 | // username and password as an argument to the NewClient function. 7 | // 8 | // This has been tested with Gmail using smtp.gmail.com and port 587 9 | package mailer 10 | 11 | import ( 12 | "bytes" 13 | "fmt" 14 | "net/smtp" 15 | "strings" 16 | "text/template" 17 | ) 18 | 19 | const ( 20 | emailTemplate = `From: {{ .DisplayName }} <{{- .From -}}> 21 | To: {{ .To }} 22 | Subject: {{ .Subject }} 23 | MIME-version: 1.0; 24 | Content-Type: text/html; charset="UTF-8"; 25 | 26 | {{ .Body }}` 27 | ) 28 | 29 | // Client is used to send emails using standard settings 30 | type Client interface { 31 | // SendEmail will send a mail to the specified email address 32 | SendEmail(subject, content string, recipients ...string) error 33 | } 34 | 35 | type mailer struct { 36 | user string 37 | auth smtp.Auth 38 | from string 39 | displayName string 40 | smtpServer string 41 | smtpPort int 42 | } 43 | 44 | // NewClient will create a new email client for sending mails 45 | func NewClient(username, password, displayName, from, smtpServer string, smtpPort int) Client { 46 | auth := smtp.PlainAuth("", username, password, smtpServer) 47 | m := new(mailer) 48 | m.auth = auth 49 | m.from = from 50 | m.displayName = displayName 51 | m.smtpServer = smtpServer 52 | m.smtpPort = smtpPort 53 | 54 | return m 55 | } 56 | 57 | // SendEmail will send a mail to the specified address. Please note that 58 | // the content is not HTML escaped. That would be up to whoever uses the method 59 | func (m *mailer) SendEmail(subject, content string, recipients ...string) error { 60 | server := fmt.Sprintf("%s:%d", m.smtpServer, m.smtpPort) 61 | var msg bytes.Buffer 62 | 63 | context := &mailContext{ 64 | From: m.from, 65 | To: strings.Join(recipients, ", "), 66 | Subject: subject, 67 | Body: content, 68 | DisplayName: m.displayName, 69 | } 70 | 71 | t := template.New("mailTemplate") 72 | t, err := t.Parse(emailTemplate) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | err = t.Execute(&msg, context) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | err = smtp.SendMail(server, m.auth, m.from, recipients, msg.Bytes()) 83 | return err 84 | } 85 | 86 | type mailContext struct { 87 | From string 88 | To string 89 | Subject string 90 | Body string 91 | DisplayName string 92 | } 93 | -------------------------------------------------------------------------------- /organization.json: -------------------------------------------------------------------------------- 1 | { 2 | "managers": [ 3 | { 4 | "username": "somemanager" 5 | } 6 | ], 7 | "departments": [ 8 | { 9 | "number": 1, 10 | "id": "dev", 11 | "name": "Developers" 12 | } 13 | ], 14 | "employees": [ 15 | { 16 | "username": "someuser", 17 | "real_name": "Some User", 18 | "manager": "somemanager", 19 | "department": "dev", 20 | "disabled": false, 21 | "aws_accounts": [ 22 | { 23 | "id": "111111111111", 24 | "cloudsweeper_enabled": true 25 | } 26 | ], 27 | "gcp_projects": [ 28 | { 29 | "id": "some-gcp-project" 30 | } 31 | ] 32 | }, 33 | { 34 | "username": "somemanager", 35 | "real_name": "Some Manager", 36 | "manager": "", 37 | "department": "dev", 38 | "aws_accounts": [ 39 | { 40 | "id": "999999999999", 41 | "cloudsweeper_enabled": false 42 | } 43 | ], 44 | "gcp_projects": [] 45 | } 46 | ] 47 | } --------------------------------------------------------------------------------