├── .github └── workflows │ ├── blackbox-workflow.yml │ └── main.yaml ├── .gitignore ├── Dockerfile ├── Dockerfile.build ├── LICENSE ├── Makefile ├── README.md ├── deploy ├── deployment.yaml ├── redis-deploy.yaml ├── redis-svc.yaml └── secret.yaml ├── glide.lock ├── glide.yaml ├── main.go ├── pkg ├── alertSlack.go ├── ec2.go ├── init.go └── web.go └── views └── urls.html /.github/workflows/blackbox-workflow.yml: -------------------------------------------------------------------------------- 1 | name: Blackbox 2 | 3 | on: 4 | repository_dispatch: 5 | types: [event-trigger] 6 | workflow_dispatch: ~ 7 | 8 | # .github/workflows/blackbox-workflow.yml 9 | jobs: 10 | run-blackbox-action: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - uses: actions/checkout@v3 14 | with: 15 | ref: '${{ github.event.repository.default_branch }}' 16 | fetch-depth: 0 17 | - id: blackbox 18 | uses: olxbr/blackbox-action@v1 19 | with: 20 | config: ${{ secrets.BLACK_BOX_CONFIG }} 21 | env: 22 | DEFAULT_BRANCH: ${{ github.event.repository.default_branch }} 23 | REPO_NAME: ${{ github.event.repository.name }} 24 | DOCKER_ECR_PASSWORD: ${{ secrets.DOCKER_ECR_PASSWORD }} 25 | DOCKER_REGISTRY: ${{ secrets.CONTAINER_REGISTRY_HOST }} 26 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_CROSS_ACCESS_KEY_ID }} 27 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_CROSS_SECRET_ACCESS_KEY }} 28 | LOGLEVEL: 'INFO' 29 | -------------------------------------------------------------------------------- /.github/workflows/main.yaml: -------------------------------------------------------------------------------- 1 | name: build_and_push 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: [master] 7 | tags: 8 | - 'v[0-9]+.[0-9]+.[0-9]+*' 9 | 10 | jobs: 11 | 12 | build_docker: 13 | name: Build Docker image 14 | runs-on: ubuntu-20.04 15 | steps: 16 | - name: Checkout code 17 | uses: actions/checkout@v2 18 | - name: Build Docker image and save it to a file 19 | run: | 20 | DOCKER_REPO=vivareal make docker_image 21 | mkdir -p /tmp/docker-cache 22 | docker save -o /tmp/docker-cache/x9.tar vivareal/x9:latest 23 | - name: Cache Docker image for further jobs in this workflow run 24 | uses: actions/upload-artifact@v2 25 | with: 26 | name: docker-image-cache 27 | path: /tmp/docker-cache/ 28 | retention-days: 1 29 | 30 | push_master_docker: 31 | if: github.ref == 'refs/heads/master' 32 | name: Push Docker image with tag master 33 | needs: build_docker 34 | runs-on: ubuntu-20.04 35 | env: 36 | DOCKER_REPO: vivareal 37 | DOCKER_IMAGE_VERSION: master 38 | steps: 39 | - name: Checkout code 40 | uses: actions/checkout@v2 41 | - name: Download cached Docker image 42 | uses: actions/download-artifact@v2 43 | with: 44 | name: docker-image-cache 45 | path: /tmp/docker-cache/ 46 | - name: Load Docker image and tag it as master 47 | run: | 48 | docker load < /tmp/docker-cache/x9.tar 49 | make docker_tag 50 | - uses: docker/login-action@v1 51 | with: 52 | username: ${{ secrets.DOCKERHUB_USERNAME }} 53 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 54 | - name: Push Docker image with tag master 55 | run: make docker_push 56 | 57 | push_release_docker: 58 | if: "startsWith(github.ref, 'refs/tags/v')" 59 | name: Push Docker image according to release 60 | needs: build_docker 61 | runs-on: ubuntu-20.04 62 | env: 63 | DOCKER_REPO: vivareal 64 | steps: 65 | - name: Checkout code 66 | uses: actions/checkout@v2 67 | - name: Download cached Docker image 68 | uses: actions/download-artifact@v2 69 | with: 70 | name: docker-image-cache 71 | path: /tmp/docker-cache/ 72 | - name: Load Docker image and tag it according to current release 73 | run: | 74 | docker load < /tmp/docker-cache/x9.tar 75 | make docker_tag DOCKER_IMAGE_VERSION=${GITHUB_REF#refs/tags/} 76 | - uses: docker/login-action@v1 77 | with: 78 | username: ${{ secrets.DOCKERHUB_USERNAME }} 79 | password: ${{ secrets.DOCKERHUB_PASSWORD }} 80 | - name: Push Docker image with release tag 81 | run: make docker_push DOCKER_IMAGE_VERSION=${GITHUB_REF#refs/tags/} 82 | - name: Push Docker image with latest tag if this version is stable 83 | run: | 84 | if [[ "${{ github.ref }}" =~ "^refs/tags/v[0-9]+\.[0-9]+\.[0-9]+$" ]]; then 85 | make docker_push DOCKER_IMAGE_VERSION=latest 86 | fi 87 | 88 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | x9 2 | .DS_Store 3 | vendor 4 | *.orig 5 | .*.sw[po] 6 | .cover 7 | dump.rdb 8 | main 9 | !.github/* 10 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM alpine 2 | RUN apk add --no-cache ca-certificates 3 | ADD views /app/views 4 | ADD x9 /usr/bin/x9 5 | WORKDIR /app 6 | ENTRYPOINT ["/usr/bin/x9"] 7 | -------------------------------------------------------------------------------- /Dockerfile.build: -------------------------------------------------------------------------------- 1 | FROM golang:1.8-alpine 2 | RUN apk add --no-cache git tar wget make 3 | RUN wget -qO- https://github.com/Masterminds/glide/releases/download/v0.12.1/glide-v0.12.1-linux-amd64.tar.gz | tar xvz --strip-components=1 -C /go/bin/ linux-amd64/glide 4 | ADD glide.yaml /go/src/github.com/grupozapvivareal/x9/glide.yaml 5 | ADD glide.lock /go/src/github.com/grupozapvivareal/x9/glide.lock 6 | WORKDIR /go/src/github.com/grupozapvivareal/x9 7 | RUN glide install 8 | ADD . /go/src/github.com/grupozapvivareal/x9 9 | RUN make install 10 | ENTRYPOINT ["/go/bin/x9"] 11 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 VivaReal 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | HOME?=$$(HOME) 2 | DOCKER_IMAGE_VERSION?=latest 3 | DOCKER_REPO?=myrepo 4 | 5 | 6 | build: 7 | CGO_ENABLED=0 go build -v -a --installsuffix cgo --ldflags="-s" -o x9 8 | 9 | run: 10 | go run main.go 11 | 12 | install: 13 | CGO_ENABLED=0 go install -v -a --installsuffix cgo --ldflags="-s" 14 | 15 | docker_build: 16 | docker build -t ${DOCKER_REPO}/x9:build -f Dockerfile.build . 17 | 18 | docker_image: docker_build 19 | docker run --rm --entrypoint /bin/sh -v ${PWD}:/out:rw ${DOCKER_REPO}/x9:build -c "cp /go/bin/x9 /out/x9" 20 | docker build -t ${DOCKER_REPO}/x9 . 21 | 22 | docker_tag: 23 | docker tag ${DOCKER_REPO}/x9 ${DOCKER_REPO}/x9:${DOCKER_IMAGE_VERSION} 24 | 25 | docker_run: 26 | docker run -d --rm ${DOCKER_REPO}/x9:${DOCKER_IMAGE_VERSION} 27 | 28 | docker_push: 29 | docker push ${DOCKER_REPO}/x9:${DOCKER_IMAGE_VERSION} 30 | 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # x9 2 | X-9 is a brazilian slang for "informer". 3 | 4 | A X-9 person is motivated by envy. 5 | 6 | This X-9 was motivated by keep learning golang. 7 | 8 | It is a tool that alerts in Slack missbehavior of AWS instances. 9 | 10 | A missbehavior is when an instance runs for short time (check the option TOLERANCE). 11 | 12 | It is most usefull to monitor auto scaling groups (ASG). When an instance is member of a ASG the ASG name is used as its key. 13 | 14 | It also provides an easy API that anwsers easy questions we are tired to give everyday, like: 15 | 16 | 17 | * What are the instances running right now, by tags, regions ? 18 | * How many instances we created in the last 24 hours ? 19 | * How many spot instances we created today ? 20 | * How many instances run for less than an hour ? 21 | * How many spot instances run for less than an hour today ? 22 | * How many instances we run by region ? 23 | * How many instaces are run with a certain tag (for example by environment)? 24 | * How many instances by type ? 25 | * What are the missbehaving auto scaling groups right now ? 26 | 27 | (check /api for a complete list) 28 | 29 | # How does it work ? 30 | 31 | Simply by getting the result of "describe instances" and sumarizes it in redis. 32 | The data expires in 24 hours. 33 | 34 | # Tags 35 | 36 | A rudimentary tag support is provided in this first release. You must use the tags bellow. 37 | If don't use tags or different tags, don't worry, they will be show as "none". 38 | 39 | |Tag|Description| 40 | |---|---| 41 | |Env|Environment, ie: QA, DEV, STG, PROD| 42 | |Product|System| 43 | |App|Component of System| 44 | 45 | # Running it locally 46 | 47 | ``` 48 | $ # make sure your aws cli is working 49 | $ brew install go 50 | $ brew install redis 51 | $ brew install glide 52 | $ mkdir ~/go/src/github.com/grupozapvivareal 53 | $ cd ~/go/src/github.com/grupozapvivareal 54 | $ git clone git@github.com:VivaReal/x9.git 55 | $ export GOPATH=~/go/ 56 | $ glide install 57 | $ redis-server & 58 | $ export SLACK_BOT_URL="https://myslack...." 59 | $ go run main.go 60 | $ curl localhost:6969 61 | ``` 62 | 63 | 64 | # Using AWS key pairs 65 | 66 | ``` 67 | $ export AWS_ACCESS_KEY_ID=XXXXXXXXX 68 | $ export AWS_SECRET_ACCESS_KEY=XXXXXXX 69 | $ go run main.go 70 | ``` 71 | 72 | # How to build it 73 | ``` 74 | $ make build 75 | (will create the x9 executable file) 76 | ``` 77 | 78 | # How to build it and run in Docker 79 | ``` 80 | $ make docker_image DOCKER_REPO="myrepo" 81 | $ docker run --name redis -d redis 82 | $ docker run -p 6969:6969 \ 83 | -e REDIS_SERVER=redis:6379 \ 84 | -e AWS_ACCESS_KEY_ID=XXX \ 85 | -e AWS_SECRET_ACCESS_KEY="XXX" \ 86 | -e SLACK_BOT_URL="https://myslack...." \ 87 | myrepo/x9 88 | $ curl localhost:6969/api 89 | ``` 90 | 91 | # All options and their defaults 92 | 93 | |*Environment variable*|*Default value*|*Description*| 94 | |---|---|---| 95 | |AWS_ACCESS_KEY_ID|-|optional, default provided by the aws-cli configuration| 96 | |AWS_SECRET_ACCESS_KEY|-|optional, default provided by the aws-cli configuration| 97 | |SLACK_BOT_URL|error|Slack bot URL| 98 | |REDIS_SERVER|localhost:6379|Redis server address and port| 99 | |TOLERANCE|3000|Minimum amount of time instances must run to not be considered missbehave| 100 | |ALERT_TIMEFRAME|1200|Interval between checks| 101 | |SERVICE_PORT|6969|Webserver port| 102 | |REGIONS|"sa-east-1,us-east-1"|Regions were to run "describe-instances"| 103 | 104 | 105 | # API reference 106 | 107 | ``` 108 | $ lynx --dump localhost:6969/api 109 | ``` 110 | 111 | # Redis keys reference 112 | 113 | |*Starting by*|*Meaning*| 114 | |---|---| 115 | |r_*|Sum of all regions| 116 | |w_*|Sum of "wasted" instances (run for less time than $TOLERANCE)| 117 | |s_*|Sum of spot instances| 118 | |tmp_*|temporary keys| 119 | -------------------------------------------------------------------------------- /deploy/deployment.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | app: x9 6 | tier: frontend 7 | name: x9 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | app: x9 13 | tier: frontend 14 | strategy: 15 | rollingUpdate: 16 | maxSurge: 1 17 | maxUnavailable: 1 18 | type: RollingUpdate 19 | template: 20 | metadata: 21 | annotations: 22 | iam.amazonaws.com/role: "arn:aws:iam:::role/" 23 | labels: 24 | app: x9 25 | tier: frontend 26 | spec: 27 | containers: 28 | - env: 29 | - name: GET_HOSTS_FROM 30 | value: dns 31 | - name: REDIS_SERVER 32 | value: redis-x9:6379 33 | - name: SLACK_BOT_URL 34 | valueFrom: 35 | secretKeyRef: 36 | key: SLACK_BOT_URL 37 | name: x9 38 | image: vivareal/x9:master 39 | imagePullPolicy: Always 40 | name: x9 41 | ports: 42 | - containerPort: 6969 43 | protocol: TCP 44 | resources: 45 | requests: 46 | cpu: 100m 47 | memory: 100Mi 48 | terminationMessagePath: /dev/termination-log 49 | terminationMessagePolicy: File 50 | dnsPolicy: ClusterFirst 51 | restartPolicy: Always 52 | schedulerName: default-scheduler 53 | securityContext: {} 54 | terminationGracePeriodSeconds: 30 55 | -------------------------------------------------------------------------------- /deploy/redis-deploy.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: extensions/v1beta1 2 | kind: Deployment 3 | metadata: 4 | generation: 1 5 | labels: 6 | app: redis-x9 7 | role: master 8 | name: redis-x9 9 | spec: 10 | replicas: 1 11 | selector: 12 | matchLabels: 13 | app: redis-x9 14 | role: master 15 | strategy: 16 | rollingUpdate: 17 | maxSurge: 1 18 | maxUnavailable: 1 19 | type: RollingUpdate 20 | template: 21 | metadata: 22 | labels: 23 | app: redis-x9 24 | role: master 25 | spec: 26 | containers: 27 | - image: redis:3.2.10-alpine 28 | imagePullPolicy: IfNotPresent 29 | name: redis-x9 30 | ports: 31 | - containerPort: 6379 32 | protocol: TCP 33 | resources: 34 | requests: 35 | cpu: 100m 36 | memory: 100Mi 37 | terminationMessagePath: /dev/termination-log 38 | terminationMessagePolicy: File 39 | dnsPolicy: ClusterFirst 40 | restartPolicy: Always 41 | schedulerName: default-scheduler 42 | securityContext: {} 43 | terminationGracePeriodSeconds: 30 44 | -------------------------------------------------------------------------------- /deploy/redis-svc.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Service 3 | metadata: 4 | labels: 5 | app: redis-x9 6 | role: master 7 | name: redis-x9 8 | spec: 9 | ports: 10 | - port: 6379 11 | protocol: TCP 12 | targetPort: 6379 13 | selector: 14 | app: redis-x9 15 | role: master 16 | sessionAffinity: None 17 | type: ClusterIP 18 | -------------------------------------------------------------------------------- /deploy/secret.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: v1 2 | kind: Secret 3 | type: Opaque 4 | data: 5 | SLACK_BOT_URL: # The URL of Slack Webhook 6 | metadata: 7 | name: x9 8 | -------------------------------------------------------------------------------- /glide.lock: -------------------------------------------------------------------------------- 1 | hash: 53c250065090dbd951af106efee8111418e360096da79ca6a579eea17e3c8900 2 | updated: 2017-10-26T10:58:21.675965486-02:00 3 | imports: 4 | - name: github.com/araddon/dateparse 5 | version: ca7e753bd149dae747d76fa7d52b20b920f3a814 6 | - name: github.com/aws/aws-sdk-go 7 | version: f426770fd5a4bae6186b280d4af7dca83a4cdef4 8 | subpackages: 9 | - aws 10 | - aws/awserr 11 | - aws/awsutil 12 | - aws/client 13 | - aws/client/metadata 14 | - aws/corehandlers 15 | - aws/credentials 16 | - aws/credentials/ec2rolecreds 17 | - aws/credentials/endpointcreds 18 | - aws/credentials/stscreds 19 | - aws/defaults 20 | - aws/ec2metadata 21 | - aws/endpoints 22 | - aws/request 23 | - aws/session 24 | - aws/signer/v4 25 | - internal/shareddefaults 26 | - private/protocol 27 | - private/protocol/ec2query 28 | - private/protocol/query 29 | - private/protocol/query/queryutil 30 | - private/protocol/rest 31 | - private/protocol/xml/xmlutil 32 | - service/ec2 33 | - service/sts 34 | - name: github.com/go-ini/ini 35 | version: f384f410798cbe7cdce40eec40b79ed32bb4f1ad 36 | - name: github.com/go-redis/redis 37 | version: 35248155a505a84811be81a2aab1c423317447be 38 | subpackages: 39 | - internal 40 | - internal/consistenthash 41 | - internal/hashtag 42 | - internal/pool 43 | - internal/proto 44 | - name: github.com/jmespath/go-jmespath 45 | version: bd40a432e4c76585ef6b72d3fd96fb9b6dc7b68d 46 | testImports: [] 47 | -------------------------------------------------------------------------------- /glide.yaml: -------------------------------------------------------------------------------- 1 | package: github.com/grupozapvivareal/x9 2 | import: 3 | - package: github.com/araddon/dateparse 4 | - package: github.com/aws/aws-sdk-go 5 | version: ^1.12.17 6 | subpackages: 7 | - aws 8 | - aws/session 9 | - service/ec2 10 | - package: github.com/go-redis/redis 11 | version: ^6.7.2 12 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/grupozapvivareal/x9/pkg" 5 | ) 6 | 7 | func main() { 8 | pkg.Init() 9 | } 10 | -------------------------------------------------------------------------------- /pkg/alertSlack.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "net/http" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/go-redis/redis" 12 | ) 13 | 14 | func sendAlert(messageJson string) { 15 | URL := SLACK_BOT_URL 16 | if strings.HasPrefix(URL, "http") { 17 | client := http.Client{} 18 | req, _ := http.NewRequest("POST", URL, bytes.NewBufferString(messageJson)) 19 | 20 | req.Header.Set("Content-Type", "application/json") 21 | client.Do(req) 22 | req.Body.Close() 23 | fmt.Printf("%v - [Alert sent]\n", time.Now()) 24 | } else { 25 | fmt.Printf("%v [WARNING] - [Alert IGNORED, invalid or empty slack url]\n", time.Now()) 26 | } 27 | } 28 | 29 | func alertSlack() { 30 | 31 | fmt.Printf("%v - [Check alerts]\n", time.Now()) 32 | // connect redis 33 | rc := redis.NewClient(&redis.Options{ 34 | Addr: REDIS_SERVER, 35 | Password: "", // no password set 36 | DB: 0, // use default DB 37 | }) 38 | 39 | keys, _ := rc.ZRangeWithScores("alertas", 0, -1).Result() 40 | count := len(keys) 41 | 42 | if count > 0 { 43 | 44 | text := "" 45 | var qtd float64 46 | 47 | for _, k := range keys { 48 | text += k.Member.(string) + ": " + strconv.FormatFloat(k.Score, 'f', 0, 64) + "\n" 49 | qtd += k.Score 50 | 51 | } 52 | 53 | sendAlert("Warning: " + strconv.FormatFloat(qtd, 'f', 0, 64) + " instances terminated before " + TOLERANCE + " seconds\n" + text) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /pkg/ec2.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "regexp" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "github.com/araddon/dateparse" 12 | "github.com/aws/aws-sdk-go/aws" 13 | "github.com/aws/aws-sdk-go/aws/session" 14 | "github.com/aws/aws-sdk-go/service/ec2" 15 | "github.com/go-redis/redis" 16 | ) 17 | 18 | // Instance is the object that holds all configurations about a AWS instance. 19 | type Instance struct { 20 | Region string 21 | Env string 22 | App string 23 | Product string 24 | isSpot string 25 | Type string 26 | Expires int64 27 | Status string 28 | isWasted string 29 | last24Hours bool 30 | lastFrameWasted bool 31 | isASG bool 32 | Asg string 33 | PrivateIP string 34 | } 35 | 36 | func updateRedis(current map[string]*Instance) { 37 | fmt.Printf("%v - [Update redis requested]\n", time.Now()) 38 | rc := redis.NewClient(&redis.Options{ 39 | Addr: REDIS_SERVER, 40 | Password: "", // no password set 41 | DB: 0, // use default DB 42 | }) 43 | 44 | for instanceId, instance := range current { 45 | 46 | if instance.last24Hours == true { 47 | member := redis.Z{Score: float64(instance.Expires), Member: instanceId} 48 | 49 | // by waste 50 | if instance.isWasted == "1" { 51 | rc.ZAddNX("w_region-"+instance.Region, member).Result() 52 | rc.ZAddNX("w_env-"+instance.Env, member).Result() 53 | rc.ZAddNX("w_app-"+instance.App, member).Result() 54 | rc.ZAddNX("w_product-"+instance.Product, member).Result() 55 | rc.ZAddNX("w_spot-"+instance.isSpot, member).Result() 56 | rc.ZAddNX("w_type-"+instance.Type, member).Result() 57 | rc.ZAddNX("w_wasted-"+instance.isWasted, member).Result() 58 | } 59 | 60 | // by spot 61 | if instance.isSpot == "1" { 62 | rc.ZAddNX("s_region-"+instance.Region, member).Result() 63 | rc.ZAddNX("s_env-"+instance.Env, member).Result() 64 | rc.ZAddNX("s_app-"+instance.App, member).Result() 65 | rc.ZAddNX("s_product-"+instance.Product, member).Result() 66 | rc.ZAddNX("s_spot-"+instance.isSpot, member).Result() 67 | rc.ZAddNX("s_type-"+instance.Type, member).Result() 68 | rc.ZAddNX("s_wasted-"+instance.isWasted, member).Result() 69 | } else { 70 | // by regular 71 | rc.ZAddNX("r_region-"+instance.Region, member).Result() 72 | rc.ZAddNX("r_env-"+instance.Env, member).Result() 73 | rc.ZAddNX("r_app-"+instance.App, member).Result() 74 | rc.ZAddNX("r_product-"+instance.Product, member).Result() 75 | rc.ZAddNX("r_spot-"+instance.isSpot, member).Result() 76 | rc.ZAddNX("r_type-"+instance.Type, member).Result() 77 | rc.ZAddNX("r_wasted-"+instance.isWasted, member).Result() 78 | } 79 | } 80 | 81 | rc.ZIncrBy("tmp_current", 1, "Total").Result() 82 | rc.ZIncrBy("tmp_current", 1, "Region-"+instance.Region).Result() 83 | rc.ZIncrBy("tmp_current", 1, "Env-"+instance.Env).Result() 84 | rc.ZIncrBy("tmp_current", 1, "App-"+instance.App).Result() 85 | rc.ZIncrBy("tmp_current", 1, "Product-"+instance.Product).Result() 86 | rc.ZIncrBy("tmp_current", 1, "isSpot-"+instance.isSpot).Result() 87 | rc.ZIncrBy("tmp_current", 1, "Type-"+instance.Type).Result() 88 | rc.ZIncrBy("tmp_current", 1, "Status-"+instance.Status+"-"+instance.Region).Result() 89 | rc.ZIncrBy("tmp_current", 1, "Status-"+instance.Status).Result() 90 | rc.ZIncrBy("tmp_current", 1, "isWasted-"+instance.isWasted).Result() 91 | 92 | if instance.lastFrameWasted { 93 | asg := "" 94 | if instance.isASG { 95 | asg = "ASG-" 96 | rc.ZIncrBy("tmp_alertasasg", 1, "Env:"+instance.Env+"----ASG:"+instance.Asg+"----Type:"+instance.Type).Result() 97 | } 98 | rc.ZIncrBy("tmp_alertas", 1, instance.Env+"-"+instance.Product+"-"+instance.App+"-"+asg+instance.Type).Result() 99 | } 100 | 101 | if instance.PrivateIP != "" { 102 | rc.ZIncrBy("tmp_ip", 1, instance.PrivateIP+"_"+instance.Region+"_"+instance.Env+"_"+instance.Product+"_"+instance.App+"_"+instance.Type+"_"+instance.Asg) 103 | } 104 | } 105 | rc.Rename("tmp_current", "current").Result() 106 | rc.Del("alertas").Result() 107 | rc.Rename("tmp_alertas", "alertas").Result() 108 | rc.Del("alertasasg") 109 | rc.Rename("tmp_alertasasg", "alertasasg").Result() 110 | rc.Del("ip") 111 | rc.Rename("tmp_ip", "ip").Result() 112 | cleanRedisKeys() 113 | 114 | fmt.Printf("%v - [Update redis finished]\n", time.Now()) 115 | } 116 | 117 | func cleanRedisKeys() { 118 | fmt.Printf("%v - [Starting cleaning keys]\n", time.Now()) 119 | rc := redis.NewClient(&redis.Options{ 120 | Addr: REDIS_SERVER, 121 | Password: "", // no password set 122 | DB: 0, // use default DB 123 | }) 124 | 125 | keys, err := rc.Keys("[a-z]_*").Result() 126 | 127 | lessthan24hours := strconv.FormatInt(time.Now().Unix()-86400, 10) 128 | 129 | if err == redis.Nil { 130 | fmt.Println("Redis error") 131 | } 132 | 133 | for i := 0; i < len(keys); i++ { 134 | rc.ZRemRangeByScore(keys[i], "0", lessthan24hours).Result() 135 | // ZCount(key, min, max string) *IntCmd 136 | 137 | count, _ := rc.ZCount(keys[i], "-inf", "+inf").Result() 138 | if count == 0 { 139 | rc.Del(keys[i]).Result() 140 | } 141 | 142 | } 143 | fmt.Printf("%v - [Finished cleaning keys]\n", time.Now()) 144 | } 145 | 146 | func getInstances() { 147 | fmt.Printf("%v - [Starting get instances]\n", time.Now()) 148 | t := time.Now().Unix() 149 | 150 | tolerance, _ := strconv.ParseInt(TOLERANCE, 10, 64) 151 | alertframe, _ := strconv.ParseInt(ALERT_TIMEFRAME, 10, 64) 152 | 153 | sess := session.Must(session.NewSession()) 154 | 155 | Regions := strings.Split(REGIONS, ",") 156 | params := &ec2.DescribeInstancesInput{ 157 | // 158 | // Filters: []*ec2.Filter{ 159 | // { 160 | // Name: aws.String("Region"), 161 | // Values: aws.String(awsRegion) 162 | // }, 163 | // { Name: aws.String("instance-lifecycle"), // "spot" instance lifecycle 164 | // Values: []*string{aws.String("spot")}, 165 | // }, 166 | //{ 167 | // Name: aws.String("instance-state-name"), 168 | // Values: []*string{aws.String("running")}, 169 | //}, 170 | // }, 171 | } 172 | 173 | re := regexp.MustCompile("\\((.*)\\)") 174 | 175 | current := make(map[string]*Instance) 176 | for _, awsRegion := range Regions { 177 | 178 | fmt.Printf("%v - [Starting get instances in %v]\n", time.Now(), awsRegion) 179 | 180 | svc := ec2.New(sess, &aws.Config{Region: aws.String(awsRegion)}) 181 | 182 | resp, err := svc.DescribeInstances(params) 183 | 184 | if err != nil { 185 | fmt.Println("there was an error listing instances in", awsRegion, err.Error()) 186 | log.Fatal(err.Error()) 187 | } 188 | 189 | for _, reserv := range resp.Reservations { 190 | 191 | for _, inst := range reserv.Instances { 192 | 193 | //fmt.Println(inst) 194 | 195 | status := *inst.State.Name 196 | InstanceId := *inst.InstanceId 197 | InstanceType := *inst.InstanceType 198 | 199 | PrivateIP := "" 200 | if inst.PrivateIpAddress != nil { 201 | PrivateIP = *inst.PrivateIpAddress 202 | } 203 | 204 | isSpot := "0" 205 | if inst.InstanceLifecycle != nil && *inst.InstanceLifecycle == "spot" { 206 | isSpot = "1" 207 | } 208 | 209 | Env := "none" 210 | Product := "none" 211 | App := "none" 212 | isASG := false 213 | Asg := "" 214 | for _, tag := range inst.Tags { 215 | 216 | switch *tag.Key { 217 | 218 | case "Env": 219 | Env = *tag.Value 220 | case "App": 221 | App = *tag.Value 222 | case "Product": 223 | Product = *tag.Value 224 | case "aws:autoscaling:groupName": 225 | isASG = true 226 | Asg = *tag.Value 227 | } 228 | } 229 | 230 | isWasted := "0" 231 | last24Hours := false 232 | lastFrameWasted := false 233 | 234 | if status == "terminated" && len(*inst.StateTransitionReason) > 0 { 235 | datestates := re.FindAllStringSubmatch(*inst.StateTransitionReason, 1) 236 | 237 | if len(datestates) > 0 && len(datestates[0]) > 1 { 238 | 239 | datestate := datestates[0][1] 240 | // Example of datestate 241 | // `User initiated (2017-07-26 18:55:53 GMT)` 242 | terminated, err := dateparse.ParseAny(datestate) 243 | if err != nil { 244 | panic(err.Error()) 245 | } 246 | 247 | if terminated.Unix()-inst.LaunchTime.Unix() < tolerance { 248 | isWasted = "1" 249 | 250 | if t-terminated.Unix() < alertframe { 251 | lastFrameWasted = true 252 | } 253 | } 254 | } 255 | last24Hours = true 256 | 257 | } 258 | 259 | if status == "running" && t-inst.LaunchTime.Unix() < 86400 { 260 | last24Hours = true 261 | } 262 | 263 | current[InstanceId] = &Instance{ 264 | Region: awsRegion, 265 | Env: Env, 266 | App: App, 267 | Product: Product, 268 | isSpot: isSpot, 269 | Type: InstanceType, 270 | Expires: inst.LaunchTime.Unix(), 271 | Status: status, 272 | isWasted: isWasted, 273 | last24Hours: last24Hours, 274 | lastFrameWasted: lastFrameWasted, 275 | isASG: isASG, 276 | Asg: Asg, 277 | PrivateIP: PrivateIP, 278 | } 279 | 280 | } 281 | } 282 | fmt.Printf("%v - [Finished getting instances in %v]\n", time.Now(), awsRegion) 283 | } 284 | fmt.Printf("%v - [Finished getting instances]\n", time.Now()) 285 | updateRedis(current) 286 | go alertSlack() 287 | time.Sleep(time.Duration(alertframe) * time.Second) 288 | getInstances() 289 | } 290 | -------------------------------------------------------------------------------- /pkg/init.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | var SLACK_BOT_URL string // slack bot tokeninzed url 8 | var REDIS_SERVER string // redis server and port 9 | var TOLERANCE string // minimum time in seconds an instance must run 10 | var ALERT_TIMEFRAME string // checks and alerts sleep time in seconds 11 | var REGIONS string // Regions to be chceckd 12 | var SERVICE_PORT string // web http server listen port 13 | 14 | func Init() { 15 | 16 | SLACK_BOT_URL = getOptEnv("SLACK_BOT_URL", "error") 17 | REDIS_SERVER = getOptEnv("REDIS_SERVER", "localhost:6379") 18 | TOLERANCE = getOptEnv("TOLERANCE", "3000") 19 | ALERT_TIMEFRAME = getOptEnv("ALERT_TIMEFRAME", "1200") 20 | REGIONS = getOptEnv("REGIONS", "sa-east-1,us-east-1") 21 | SERVICE_PORT = getOptEnv("SERVICE_PORT", "6969") 22 | 23 | Web() 24 | } 25 | 26 | func getOptEnv(name, df string) string { 27 | value := os.Getenv(name) 28 | if value == "" { 29 | value = df 30 | } 31 | return value 32 | } 33 | -------------------------------------------------------------------------------- /pkg/web.go: -------------------------------------------------------------------------------- 1 | package pkg 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "net/http" 7 | "sort" 8 | "strings" 9 | "time" 10 | 11 | "github.com/go-redis/redis" 12 | ) 13 | 14 | // Web will start the Web Server and keeps listen for requests. 15 | func Web() { 16 | fmt.Printf("%v - [Starting web]\n", time.Now()) 17 | go getInstances() 18 | 19 | http.HandleFunc("/all/", httpRoute) 20 | http.HandleFunc("/spot/", httpRoute) 21 | http.HandleFunc("/spot/wasted/", httpRoute) 22 | http.HandleFunc("/wasted/", httpRoute) 23 | http.HandleFunc("/wasted/spot/", httpRoute) 24 | 25 | http.HandleFunc("/region/", httpRoute) 26 | http.HandleFunc("/region/spot/", httpRoute) 27 | http.HandleFunc("/region/wasted/", httpRoute) 28 | 29 | http.HandleFunc("/app/", httpRoute) 30 | http.HandleFunc("/app/spot/", httpRoute) 31 | http.HandleFunc("/app/wasted/", httpRoute) 32 | 33 | http.HandleFunc("/product/", httpRoute) 34 | http.HandleFunc("/product/spot/", httpRoute) 35 | http.HandleFunc("/product/wasted/", httpRoute) 36 | 37 | http.HandleFunc("/env/", httpRoute) 38 | http.HandleFunc("/env/spot/", httpRoute) 39 | http.HandleFunc("/env/wasted/", httpRoute) 40 | 41 | http.HandleFunc("/type/", httpRoute) 42 | http.HandleFunc("/type/spot/", httpRoute) 43 | http.HandleFunc("/type/wasted/", httpRoute) 44 | 45 | http.HandleFunc("/current/", httpSingleKey) 46 | http.HandleFunc("/ip/", httpSingleKey) 47 | http.HandleFunc("/json", httpSingleKey) 48 | http.HandleFunc("/alerts/", httpSingleKey) 49 | http.HandleFunc("/alerts/asg/", httpSingleKey) 50 | 51 | http.HandleFunc("/api/", func(w http.ResponseWriter, r *http.Request) { 52 | http.ServeFile(w, r, "views/urls.html") 53 | }) 54 | 55 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 56 | http.ServeFile(w, r, "views/urls.html") 57 | }) 58 | 59 | port := SERVICE_PORT 60 | fmt.Printf("listening on %v...\n", port) 61 | err := http.ListenAndServe(":"+port, nil) 62 | if err != nil { 63 | panic(err) 64 | } 65 | } 66 | 67 | func httpSingleKey(w http.ResponseWriter, r *http.Request) { 68 | rc := redis.NewClient(&redis.Options{ 69 | Addr: REDIS_SERVER, 70 | Password: "", // no password set 71 | DB: 0, // use default DB 72 | }) 73 | 74 | cutIt := false 75 | prefix := "" 76 | switch r.RequestURI { 77 | case "/current/", "/current/json": 78 | prefix = "current" 79 | case "/ip/", "/ip/json": 80 | prefix = "ip" 81 | case "/alerts/", "/alerts/json": 82 | prefix = "alertas" 83 | case "/alerts/asg/", "/alerts/asg/json": 84 | prefix = "alertasasg" 85 | case "/json": 86 | prefix = "virginator" 87 | cutIt = true 88 | } 89 | 90 | keys, _ := rc.ZRangeWithScores(prefix, 0, -1).Result() 91 | 92 | isJson := false 93 | if strings.HasSuffix(r.RequestURI, "json") { 94 | isJson = true 95 | w.Header()["Access-Control-Allow-Origin"] = []string{"*"} 96 | w.Header()["Content-Type"] = []string{"application/json"} 97 | } else { 98 | fmt.Fprint(w, "") 99 | } 100 | 101 | var members []string 102 | jsonmapcut := make(map[string]map[string]float64) 103 | mapa := make(map[string]float64) 104 | 105 | for _, k := range keys { 106 | members = append(members, k.Member.(string)) 107 | mapa[k.Member.(string)] = k.Score 108 | 109 | } 110 | 111 | jsonmap := make(map[string]float64) 112 | sort.Strings(members) 113 | 114 | for _, k := range members { 115 | if isJson { 116 | if cutIt { 117 | name := strings.SplitN(k, "-", 2) 118 | a := make(map[string]float64) 119 | a["us"] = mapa[name[0]+"-us"] 120 | a["sa"] = mapa[name[0]+"-sa"] 121 | jsonmapcut[name[0]] = a 122 | } else { 123 | jsonmap[k] = mapa[k] 124 | } 125 | } else { 126 | red := "" 127 | if strings.Contains(k, "none") { 128 | red = "" 129 | } 130 | 131 | fmt.Fprintf(w, "", red, k, mapa[k]) 132 | } 133 | 134 | } 135 | 136 | jsonread := "" 137 | var jsonout []byte 138 | if isJson { 139 | if cutIt { 140 | jsonout, _ = json.Marshal(jsonmapcut) 141 | } else { 142 | jsonout, _ = json.Marshal(jsonmap) 143 | } 144 | jsonread = string(jsonout[:]) 145 | fmt.Fprint(w, jsonread) 146 | } else { 147 | fmt.Fprint(w, "
%v%v :%v
") 148 | } 149 | } 150 | 151 | func httpRoute(w http.ResponseWriter, r *http.Request) { 152 | prefix := "/" 153 | noit := false 154 | switch r.RequestURI { 155 | case "/all/", "/all/json": 156 | prefix = "r_*" 157 | noit = true 158 | case "/spot/", "/spot/json": 159 | prefix = "s_*" 160 | noit = true 161 | case "/wasted/", "/wasted/json": 162 | prefix = "w_*" 163 | noit = true 164 | case "/spot/wasted/", "/sort/wasted/json": 165 | prefix = "s_wasted*" 166 | case "/wasted/spot/", "/wasted/spot/json": 167 | prefix = "w_spot*" 168 | case "/region/", "/region/json": 169 | prefix = "r_region*" 170 | case "/region/spot/", "/region/spot/json": 171 | prefix = "s_region*" 172 | case "/region/wasted/", "/region/wasted/json": 173 | prefix = "w_region*" 174 | case "/app/", "/app/json": 175 | prefix = "r_app*" 176 | case "/app/spot/", "/app/spot/json": 177 | prefix = "s_app*" 178 | case "/app/wasted/", "/app/wasted/json": 179 | prefix = "w_app*" 180 | case "/product/", "/product/json": 181 | prefix = "r_product*" 182 | case "/product/spot/", "/product/spot/json": 183 | prefix = "s_product*" 184 | case "/product/wasted/", "/product/wasted/json": 185 | prefix = "w_product*" 186 | case "/env/", "/env/json": 187 | prefix = "r_env*" 188 | case "/env/spot/", "/env/spot/json": 189 | prefix = "s_env*" 190 | case "/env/wasted/", "/env/wasted/json": 191 | prefix = "w_env*" 192 | case "/type/", "/type/json": 193 | prefix = "r_type*" //nice! 194 | case "/type/spot/", "/type/spot/json": 195 | prefix = "s_type*" 196 | case "/type/wasted/", "/type/wasted/json": 197 | prefix = "w_type*" 198 | } 199 | 200 | rc := redis.NewClient(&redis.Options{ 201 | Addr: REDIS_SERVER, 202 | Password: "", // no password set 203 | DB: 0, // use default DB 204 | }) 205 | 206 | isJson := false 207 | if strings.HasSuffix(r.RequestURI, "json") { 208 | isJson = true 209 | w.Header()["Access-Control-Allow-Origin"] = []string{"*"} 210 | w.Header()["Content-Type"] = []string{"application/json"} 211 | } else { 212 | fmt.Fprint(w, "") 213 | } 214 | 215 | keys, _ := rc.Keys(prefix).Result() 216 | sort.Strings(keys) 217 | jsonmap := make(map[string]int64) 218 | for _, k := range keys { 219 | 220 | value, _ := rc.ZCount(k, "-inf", "+inf").Result() 221 | 222 | name := strings.SplitN(k, "-", 2) 223 | 224 | if isJson { 225 | if noit { 226 | jsonmap[k] = value 227 | } else { 228 | jsonmap[name[1]] = value 229 | } 230 | 231 | } else { 232 | red := "" 233 | if strings.Contains(k, "none") { 234 | red = "" 235 | } 236 | if noit { 237 | fmt.Fprintf(w, "", red, k, value) 238 | } else { 239 | fmt.Fprintf(w, "", red, name[1], value) 240 | } 241 | } 242 | } 243 | 244 | if isJson { 245 | jsonout, _ := json.Marshal(jsonmap) 246 | jsonread := string(jsonout[:]) 247 | fmt.Fprint(w, jsonread) 248 | } else { 249 | fmt.Fprint(w, "
%v%v :%v
%v%v :%v
") 250 | } 251 | } 252 | -------------------------------------------------------------------------------- /views/urls.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 20 | 21 | Wasted means: ran less than the TOLERANCE
22 | 0 is false | 1 is true | 0+1 = total instances in that query

23 | /current - All instances running right now
24 | /current/json
25 | /ip - All instances running right now by Private IP Address
26 | /ip/json
27 |
28 | /all - all instances created today
29 | /all/json
30 | /spot - all spot instances created today
31 | /spot/json
32 | /wasted - all wasted instances today
33 | /wasted/json
34 | /spot/wasted - all spot wasted instances today (0+1 = total spots)
35 | /spot/wasted/json
36 | /wasted/spot all spot wasted instances today (0+1 = total wasted)
37 | /wasted/spot/json
38 |
39 | /region - all instances created today by region
40 | /region/json
41 | /region/spot - all spot instances created today by region
42 | /region/spot/json
43 | /region/wasted - all wasted instances create today by region
44 | /region/wasted/json
45 |
46 | /app - all instances created today by App
47 | /app/json
48 | /app/spot - all spot instances created today by App
49 | /app/spot/json
50 | /app/wasted - all wasted instances created today by App
51 | /app/wasted/json
52 |
53 | /product - all instances created today by Product
54 | /product/json
55 | /product/spot - all spot instances created today by Product
56 | /product/spot/json
57 | /product/wasted - all wasted instances created today by Product
58 | /product/wasted/json
59 |
60 | /env - all instances created today by Environment
61 | /env/json
62 | /env/spot - all spot instances created today by Environment
63 | /env/spot/json
64 | /env/wasted - all wasted instances created today by Environment
65 | /env/wasted/json
66 |
67 | /type - all instances created today by Type
68 | /type/json
69 | /type/spot - all spot instances created today by Type
70 | /type/spot/json
71 | /type/wasted - all wasted instances created today by Type
72 | /type/wasted/json
73 |
74 | /alerts - all wasted instances in last 20 minutes
75 | /alerts/json
76 | /alerts/asg - all wasted instances in last 20 minutes by ASG
77 | /alerts/asg/json 78 | 79 | 80 | 81 | 82 | --------------------------------------------------------------------------------