├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── aws-billing-es-template.json ├── docker-compose-test-rerun.yml ├── docker-compose-test.yml ├── docker-compose.yml ├── go-wrapper ├── kibana.yml ├── kibana ├── Cost_For_AmazonS3_requests.json ├── Pi-chart-for-seperate-services.json ├── S3_Api_Calls_daily.json ├── Split_bars_daily.json ├── Spot_vs_OnDemand_EC2.json ├── Total_UnblendedCost.json ├── api_call_table.json ├── discover_search.json ├── finalVisualization_5days_30min_line_split.json ├── finalVisualization_5days_30min_row_split.json ├── kibana_dashboard.json ├── kibana_visualizations.json ├── orchestrate_dashboard.sh ├── orchestrate_kibana.sh ├── orchestrate_search_mapping.sh ├── orchestrate_visualisation.sh └── top_5_used_service_split_daily.json ├── logstash.conf ├── main.go ├── main_test.go ├── orchestrate-test.py ├── orchestrate.py ├── prod.sample.env ├── screenshots ├── aws-report-usage-cost-prodenv.png ├── aws-report-usage-cost.png └── kibana-dashboard.png ├── stop.sh ├── test ├── __init__.py ├── sample │ ├── __init__.py │ └── test_ub_cost_2016-06.csv ├── scripts │ ├── Total_BlendedCost.json │ ├── add_del_modify_vis.sh │ ├── check_add_del_modify_vis.sh │ └── orchestrate_visualisation.sh └── tools │ ├── __init__.py │ ├── aggregate.json │ └── check_tools.py └── tools ├── __init__.py └── tools.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.html 2 | #Use run.py for testing purpose only 3 | run.py 4 | .DS_Store 5 | *.swp 6 | .vagrant 7 | *.pyc 8 | .project 9 | *.xml 10 | *.swo 11 | run.py 12 | *.ipynb 13 | run_tests.py 14 | *.out 15 | main 16 | *.gz 17 | *.csv 18 | !test/sample/*.csv 19 | getfile*.json 20 | prod.env 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # The cnfig for travis CI 2 | sudo: required 3 | 4 | language: python 5 | 6 | python: 7 | - "2.7" 8 | env: 9 | global: 10 | - DOCKER_VERSION=1.12.2-0~trusty 11 | 12 | services: 13 | - docker 14 | 15 | before_install: 16 | # list docker-engine versions 17 | - sudo apt-cache madison docker-engine 18 | # upgrade docker-engine to specific version 19 | - sudo apt-get -o Dpkg::Options::="--force-confnew" install -y docker-ce 20 | 21 | # reinstall docker-compose at specific version 22 | - sudo rm /usr/local/bin/docker-compose 23 | - curl -L https://github.com/docker/compose/releases/download/1.7.1/docker-compose-`uname -s`-`uname -m` > docker-compose 24 | - sudo chmod +x docker-compose 25 | - sudo mv docker-compose /usr/local/bin 26 | - sudo sysctl -w vm.max_map_count=262144 27 | 28 | script: 29 | - sudo service docker restart 30 | - sudo docker-compose -f docker-compose-test.yml up 31 | - sudo docker-compose -f docker-compose-test-rerun.yml up 32 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:14.04.4 2 | 3 | # gcc for cgo 4 | RUN apt-get update && apt-get install -y --no-install-recommends \ 5 | g++ \ 6 | gcc \ 7 | libc6-dev \ 8 | make \ 9 | && rm -rf /var/lib/apt/lists/* 10 | 11 | ENV GOLANG_VERSION 1.6.2 12 | ENV GOLANG_DOWNLOAD_URL https://golang.org/dl/go$GOLANG_VERSION.linux-amd64.tar.gz 13 | ENV GOLANG_DOWNLOAD_SHA256 e40c36ae71756198478624ed1bb4ce17597b3c19d243f3f0899bb5740d56212a 14 | 15 | RUN apt-get update 16 | RUN apt-get install -y curl 17 | RUN curl -fsSL "$GOLANG_DOWNLOAD_URL" -o golang.tar.gz \ 18 | && echo "$GOLANG_DOWNLOAD_SHA256 golang.tar.gz" | sha256sum -c - \ 19 | && tar -C /usr/local -xzf golang.tar.gz \ 20 | && rm golang.tar.gz 21 | 22 | ENV GOPATH /go 23 | ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH 24 | 25 | RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 777 "$GOPATH" 26 | 27 | COPY go-wrapper /usr/local/bin/ 28 | 29 | RUN apt-get install -y python 30 | RUN apt-get install -y python-pip 31 | RUN pip install boto3 32 | RUN pip install pyelasticsearch 33 | RUN pip install nose 34 | 35 | WORKDIR /aws-elk-billing 36 | 37 | ENV TZ=Asia/Kolkata 38 | RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone 39 | 40 | CMD ["tail -f /dev/null"] 41 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Bisect XYZ 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aws-elk-billing [![Build Status](https://travis-ci.org/PriceBoardIn/aws-elk-billing.svg?branch=master)](https://travis-ci.org/PriceBoardIn/aws-elk-billing) 2 | 3 | ![Alt text](https://github.com/PriceBoardIn/aws-elk-billing/blob/master/screenshots/kibana-dashboard.png "Overview") 4 | 5 | ## Overview 6 | 7 | aws-elk-billing is a combination of configuration snippets and tools to assist with indexing AWS programatic billing access files(CSV's) and visualizing the data using Kibana. 8 | 9 | Currently it supports `AWS Cost and Usage Report` type, although it might work for other [AWS Billing Report Types](http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/detailed-billing-reports.html#other-reports) which contains some extra columns along with all the columns from `AWS Cost and Usage Report`. 10 | 11 | ![Alt text](https://github.com/PriceBoardIn/aws-elk-billing/blob/master/screenshots/aws-report-usage-cost.png) 12 | 13 | You can create `AWS Cost and Usage Report` at https://console.aws.amazon.com/billing/home#/reports 14 | Make sure that it contains the following dimensions only **(Don't include Resource IDs)** 15 | * Account Identifiers 16 | * Invoice and Bill Information 17 | * Usage Amount and Unit 18 | * Rates and Costs 19 | * Product Attributes 20 | * Pricing Attributes 21 | * Cost Allocation Tags 22 | 23 | Follow instructions at http://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/detailed-billing-reports.html#turnonreports 24 | 25 | 26 | ### Architecture 27 | There are Four Docker containers. 28 | 29 | 1. [elasticsearch:5-alpine](https://hub.docker.com/_/elasticsearch/) 30 | 2. [priceboard/docker-alpine:kibana](https://hub.docker.com/r/priceboard/docker-alpine/) (https://github.com/PriceBoardIn/docker-alpine/tree/master/alpine-kibana) 31 | 3. [Logstash:5-alpine](https://hub.docker.com/r/_/logstash) 32 | 4. aws-elk-billing (Refer: Dockerfile of this repository) 33 | 34 | Integration among the 4 containers is done with `docker-compose.yml` 35 | 36 | 37 | ### Primary Components 38 | Task | Files 39 | ------------ | ------------- 40 | Logstash configuration | `logstash.conf` 41 | Kibana configuration | `kibana.yml` 42 | Elasticsearch index mapping | `aws-billing-es-template.json` 43 | Indexing Kibana dashboard| `kibana/orchestrate_dashboard.sh` 44 | Indexing Kibana visualisation| `kibana/orchestrate_visualisation.sh` 45 | Indexing Kibana default index (This file is just for reference purpose, we will automate this part eventually)| `kibana/orchestrate_kibana.sh` 46 | Parsing the aws-billing CSV's and sending to logstash | `main.go` 47 | Connecting the dots: `Wait` for ELK Stack to start listening on their respective ports, `downloads`, `extracts` the latest compressed billing report from S3, `XDELETE` previous index of the current month, `Index mapping`, `Index kibana_dashboard`, `Index kibana_visualization` and finally executes `main.go` | `orchestrate.py` 48 | Integrating all 4 containers | `Dockerfile`, `docker-compose.yml` 49 | 50 | ## Getting Started 51 | Clone the Repository and make sure that no process is listening to the ports used by all these dockers. 52 | 53 | Ports | Process 54 | ------------ | ------------- 55 | 9200, 9300 | Elasticsearch 56 | 5601 | Kibana 57 | 5140 | Logstash 58 | 59 | ### Set S3 credentials and AWS Billing bucket and directory name 60 | Rename [prod.sample.env](https://github.com/PriceBoardIn/aws-elk-billing/blob/master/prod.sample.env) to `prod.env` and provide values for the following keys `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `S3_BUCKET_NAME`, `S3_REPORT_PATH` 61 | 62 | ##### `S3_BUCKET_NAME` = S3 bucket name (Refer the image above) 63 | ##### `S3_REPORT_PATH` = Report path (Refer the image above) 64 | ##### `S3_REPORT_NAME` = Report name (Refer the image above) 65 | 66 | ![Alt text](https://github.com/PriceBoardIn/aws-elk-billing/blob/master/screenshots/aws-report-usage-cost-prodenv.png) 67 | 68 | `prod.env` is added in `.gitignore` so that you don't push your credentials upstream accidentally. 69 | 70 | ### Run Docker 71 | The entire process is automated through scripts and docker. All the components would be downloaded automatically inside your docker 72 | 73 | 1. ```sudo docker-compose up -d``` 74 | 2. View `Kibana` at http://localhost:5601 75 | 76 | 2.1 Use the **index pattern** as `aws-billing-*` and select the **time field** as `lineItem/UsageStartDate` 77 | 78 | 2.2 `Kibana AWS Billing Dashboard` http://localhost:5601/app/kibana#/dashboard/AWS-Billing-DashBoard 79 | 80 | 2.3 For MAC replace localhost with the ip of docker-machine 81 | To find IP of docker-machine `docker-machine ip default` 82 | 83 | 3 . `sudo docker-compose stop` to shutdown all the docker containers. 84 | 85 | 4 . `sudo docker-compose down` to shutdown and remove all the files from docker. **Note:** Next time you do a `docekr-compose up` every thing will start from scratch. Use this if you see some problems in your data or ES is timing out. 86 | 87 | ## Gotchas 88 | 89 | * `aws-elk-billing` container will take time while running the following two process `[Filename: orchestrate.py]`. 90 | 1. Downloading and extracting AWS Billing report from AWS S3. 91 | 2. Depending on the size of AWS Billing CSV report `main.go` will take time to index all the data to Elasticsearch via Logstash. 92 | * You can view the dashboard in kibana, even while `main.go` is still indexing the data. 93 | * In order to index new data, you'll have to run `docker-compose up -d` again. 94 | 95 | ## Feedback 96 | We'll love to hear feedback and ideas on how we can make it more useful. Just create an issue. 97 | 98 | Thanks 99 | -------------------------------------------------------------------------------- /aws-billing-es-template.json: -------------------------------------------------------------------------------- 1 | { 2 | "template" : "aws-billing-*", 3 | "settings" : { 4 | "index.refresh_interval" : "10s" 5 | }, 6 | "mappings" : { 7 | "_default_" : { 8 | "_all" : {"enabled" : true}, 9 | "dynamic_templates" : [ { 10 | "string_fields" : { 11 | "match" : "*", 12 | "match_mapping_type" : "text", 13 | "mapping" : { 14 | "type" : "text", "index" : "analyzed", "omit_norms" : true, 15 | "fields" : { 16 | "raw" : {"type": "text", "index" : "not_analyzed", "ignore_above" : 256} 17 | } 18 | } 19 | } 20 | } ], 21 | "properties" : { 22 | "@version": { "type": "text", "index": "not_analyzed" }, 23 | "lineItem/BlendedCost":{"type":"float"}, 24 | "lineItem/UnblendedCost":{"type":"float"} 25 | } 26 | } 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /docker-compose-test-rerun.yml: -------------------------------------------------------------------------------- 1 | #Run docker container in the background 2 | #docker-compose up -d 3 | version: '2' 4 | services: 5 | aws-elk-billing: 6 | build: . 7 | environment: 8 | - DEBIAN_FRONTEND=noninteractive 9 | - TERM=xterm 10 | - ENV=test 11 | volumes: 12 | - .:/aws-elk-billing 13 | - /var/run/docker.sock:/var/run/docker.sock 14 | depends_on: 15 | - kibana 16 | - logstash 17 | - elasticsearch 18 | links: 19 | - kibana:kibana 20 | - logstash:logstash 21 | - elasticsearch:elasticsearch 22 | command: bash -c "python -u /aws-elk-billing/orchestrate-test.py && nosetests --nologcapture test.tools.check_tools && bash test/scripts/check_add_del_modify_vis.sh && bash stop.sh" 23 | 24 | logstash: 25 | image: logstash:5-alpine 26 | ports: 27 | - 5140:5140 28 | depends_on: 29 | - elasticsearch 30 | links: 31 | - elasticsearch:elasticsearch 32 | volumes: 33 | - ./logstash.conf:/logstash.conf 34 | command: "logstash -f /logstash.conf --debug --verbose" 35 | 36 | kibana: 37 | image: priceboard/docker-alpine:kibana 38 | volumes: 39 | - ./kibana.yml:/opt/kibana/config/kibana.yml 40 | ports: 41 | - 5601:5601 42 | depends_on: 43 | - elasticsearch 44 | links: 45 | - elasticsearch:elasticsearch 46 | 47 | elasticsearch: 48 | image: elasticsearch:5-alpine 49 | ports: 50 | - 9200:9200 51 | - 9300:9300 52 | environment: 53 | ES_JAVA_OPTS: "-Xms1g -Xmx1g" 54 | -------------------------------------------------------------------------------- /docker-compose-test.yml: -------------------------------------------------------------------------------- 1 | #Run docker container in the background 2 | #docker-compose up -d 3 | version: '2' 4 | services: 5 | aws-elk-billing: 6 | build: . 7 | environment: 8 | - DEBIAN_FRONTEND=noninteractive 9 | - TERM=xterm 10 | - ENV=test 11 | volumes: 12 | - .:/aws-elk-billing 13 | - /var/run/docker.sock:/var/run/docker.sock 14 | depends_on: 15 | - kibana 16 | - logstash 17 | - elasticsearch 18 | links: 19 | - kibana:kibana 20 | - logstash:logstash 21 | - elasticsearch:elasticsearch 22 | command: bash -c "python -u /aws-elk-billing/orchestrate-test.py && nosetests --nologcapture test.tools.check_tools && bash test/scripts/add_del_modify_vis.sh && bash stop.sh" 23 | 24 | logstash: 25 | image: logstash:5-alpine 26 | ports: 27 | - 5140:5140 28 | depends_on: 29 | - elasticsearch 30 | links: 31 | - elasticsearch:elasticsearch 32 | volumes: 33 | - ./logstash.conf:/logstash.conf 34 | command: "logstash -f /logstash.conf --debug --verbose" 35 | 36 | kibana: 37 | image: priceboard/docker-alpine:kibana 38 | volumes: 39 | - ./kibana.yml:/opt/kibana/config/kibana.yml 40 | ports: 41 | - 5601:5601 42 | depends_on: 43 | - elasticsearch 44 | links: 45 | - elasticsearch:elasticsearch 46 | 47 | elasticsearch: 48 | image: elasticsearch:5-alpine 49 | ports: 50 | - 9200:9200 51 | - 9300:9300 52 | environment: 53 | ES_JAVA_OPTS: "-Xms1g -Xmx1g" 54 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | #Run docker container in the background 2 | #docker-compose up -d 3 | version: '2' 4 | services: 5 | aws-elk-billing: 6 | build: . 7 | environment: 8 | - DEBIAN_FRONTEND=noninteractive 9 | - TERM=xterm 10 | - ENV=prod 11 | volumes: 12 | - .:/aws-elk-billing 13 | depends_on: 14 | - kibana 15 | - logstash 16 | - elasticsearch 17 | links: 18 | - kibana:kibana 19 | - logstash:logstash 20 | - elasticsearch:elasticsearch 21 | env_file: 22 | - prod.env 23 | command: bash -c "python -u /aws-elk-billing/orchestrate.py" 24 | 25 | logstash: 26 | image: logstash:5-alpine 27 | ports: 28 | - 5140:5140 29 | depends_on: 30 | - elasticsearch 31 | links: 32 | - elasticsearch:elasticsearch 33 | volumes: 34 | - ./logstash.conf:/logstash.conf 35 | command: "logstash -f /logstash.conf" 36 | 37 | kibana: 38 | image: priceboard/docker-alpine:kibana 39 | volumes: 40 | - ./kibana.yml:/opt/kibana/config/kibana.yml 41 | ports: 42 | - 5601:5601 43 | depends_on: 44 | - elasticsearch 45 | links: 46 | - elasticsearch:elasticsearch 47 | 48 | elasticsearch: 49 | image: elasticsearch:5-alpine 50 | ports: 51 | - 9200:9200 52 | - 9300:9300 53 | environment: 54 | ES_JAVA_OPTS: "-Xms1g -Xmx1g" 55 | -------------------------------------------------------------------------------- /go-wrapper: -------------------------------------------------------------------------------- 1 | et -e 2 | 3 | usage() { 4 | base="$(basename "$0")" 5 | cat <&2 47 | exit 1 48 | fi 49 | 50 | goDir="$(go list -e -f '{{.ImportComment}}' 2>/dev/null || true)" 51 | 52 | if [ -z "$goDir" -a -s .godir ]; then 53 | goDir="$(cat .godir)" 54 | fi 55 | 56 | dir="$(pwd -P)" 57 | if [ "$goDir" ]; then 58 | goPath="${GOPATH%%:*}" # this just grabs the first path listed in GOPATH, if there are multiple (which is the detection logic "go get" itself uses, too) 59 | goDirPath="$goPath/src/$goDir" 60 | mkdir -p "$(dirname "$goDirPath")" 61 | if [ ! -e "$goDirPath" ]; then 62 | ln -sfv "$dir" "$goDirPath" 63 | elif [ ! -L "$goDirPath" ]; then 64 | echo >&2 "error: $goDirPath already exists but is unexpectedly not a symlink!" 65 | exit 1 66 | fi 67 | goBin="$goPath/bin/$(basename "$goDir")" 68 | else 69 | goBin="$(basename "$dir")" # likely "app" 70 | fi 71 | 72 | case "$cmd" in 73 | download) 74 | execCommand=( go get -v -d "$@" ) 75 | if [ "$goDir" ]; then execCommand+=( "$goDir" ); fi 76 | set -x; exec "${execCommand[@]}" 77 | ;; 78 | 79 | install) 80 | execCommand=( go install -v "$@" ) 81 | if [ "$goDir" ]; then execCommand+=( "$goDir" ); fi 82 | set -x; exec "${execCommand[@]}" 83 | ;; 84 | 85 | run) 86 | set -x; exec "$goBin" "$@" 87 | ;; 88 | 89 | *) 90 | echo >&2 'error: unknown command:' "$cmd" 91 | usage >&2 92 | exit 1 93 | ;; 94 | esac 95 | -------------------------------------------------------------------------------- /kibana.yml: -------------------------------------------------------------------------------- 1 | # Kibana is served by a back end server. This controls which port to use. 2 | server.port: 5601 3 | 4 | # The host to bind the server to. 5 | server.host: "0.0.0.0" 6 | 7 | # If you are running kibana behind a proxy, and want to mount it at a path, 8 | # specify that path here. The basePath can't end in a slash. 9 | # server.basePath: "" 10 | 11 | # The maximum payload size in bytes on incoming server requests. 12 | # server.maxPayloadBytes: 1048576 13 | 14 | # The Elasticsearch instance to use for all your queries. 15 | elasticsearch.url: "http://elasticsearch:9200" 16 | 17 | # preserve_elasticsearch_host true will send the hostname specified in `elasticsearch`. If you set it to false, 18 | # then the host you use to connect to *this* Kibana instance will be sent. 19 | # elasticsearch.preserveHost: true 20 | 21 | # Kibana uses an index in Elasticsearch to store saved searches, visualizations 22 | # and dashboards. It will create a new index if it doesn't already exist. 23 | # kibana.index: ".kibana" 24 | 25 | # The default application to load. 26 | # kibana.defaultAppId: "discover" 27 | 28 | # If your Elasticsearch is protected with basic auth, these are the user credentials 29 | # used by the Kibana server to perform maintenance on the kibana_index at startup. Your Kibana 30 | # users will still need to authenticate with Elasticsearch (which is proxied through 31 | # the Kibana server) 32 | # elasticsearch.username: "user" 33 | # elasticsearch.password: "pass" 34 | 35 | # SSL for outgoing requests from the Kibana Server to the browser (PEM formatted) 36 | # server.ssl.cert: /path/to/your/server.crt 37 | # server.ssl.key: /path/to/your/server.key 38 | 39 | # Optional setting to validate that your Elasticsearch backend uses the same key files (PEM formatted) 40 | # elasticsearch.ssl.cert: /path/to/your/client.crt 41 | # elasticsearch.ssl.key: /path/to/your/client.key 42 | 43 | # If you need to provide a CA certificate for your Elasticsearch instance, put 44 | # the path of the pem file here. 45 | # elasticsearch.ssl.ca: /path/to/your/CA.pem 46 | 47 | # Set to false to have a complete disregard for the validity of the SSL 48 | # certificate. 49 | # elasticsearch.ssl.verify: true 50 | 51 | # Time in milliseconds to wait for elasticsearch to respond to pings, defaults to 52 | # request_timeout setting 53 | # elasticsearch.pingTimeout: 1500 54 | 55 | # Time in milliseconds to wait for responses from the back end or elasticsearch. 56 | # This must be > 0 57 | # elasticsearch.requestTimeout: 30000 58 | 59 | # Time in milliseconds for Elasticsearch to wait for responses from shards. 60 | # Set to 0 to disable. 61 | # elasticsearch.shardTimeout: 0 62 | 63 | # Time in milliseconds to wait for Elasticsearch at Kibana startup before retrying 64 | # elasticsearch.startupTimeout: 5000 65 | 66 | # Set the path to where you would like the process id file to be created. 67 | # pid.file: /var/run/kibana.pid 68 | 69 | # If you would like to send the log output to a file you can set the path below. 70 | # logging.dest: stdout 71 | 72 | # Set this to true to suppress all logging output. 73 | # logging.silent: false 74 | 75 | # Set this to true to suppress all logging output except for error messages. 76 | # logging.quiet: false 77 | 78 | # Set this to true to log all events, including system usage information and all requests. 79 | # logging.verbose: false 80 | -------------------------------------------------------------------------------- /kibana/Cost_For_AmazonS3_requests.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Cost_For_AmazonS3_requests", 3 | "visState": "{\"title\":\"New Visualization\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"filters\",\"schema\":\"segment\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"AmazonS3\",\"analyze_wildcard\":true}}},\"label\":\"\"}]}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"product/productFamily.raw\",\"size\":3,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"vis\":{\"colors\":{\"AmazonS3\":\"#967302\",\"API Request\":\"#614D93\",\"Data Transfer\":\"#F9D9F9\",\"Storage\":\"#F9934E\"}}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/Pi-chart-for-seperate-services.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Pi chart for seperate services", 3 | "visState": "{\"title\":\"Pi chart for seperate services\",\"type\":\"pie\",\"params\":{\"addLegend\":true,\"addTooltip\":true,\"isDonut\":false,\"shareYAxis\":true},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\",\"customLabel\":\"\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":8,\"order\":\"desc\",\"orderBy\":\"1\"}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"vis\":{\"colors\":{\"AmazonRoute53\":\"#2F575E\",\"AmazonS3\":\"#967302\",\"AmazonEC2\":\"#3F6833\",\"AmazonCloudFront\":\"#99440A\"},\"legendOpen\":true}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/S3_Api_Calls_daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "S3_Api_Calls_daily", 3 | "visState": "{\"title\":\"New Visualization\",\"type\":\"line\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"showCircles\":true,\"smoothLines\":false,\"interpolate\":\"linear\",\"scale\":\"linear\",\"drawLinesBetweenPoints\":true,\"radiusRatio\":9,\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"d\",\"customInterval\":\"2d\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"filters\",\"schema\":\"group\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"API Request\",\"analyze_wildcard\":true}}},\"label\":\"\"}]}}],\"listeners\":{}}", 4 | "uiStateJSON": "{}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/Split_bars_daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Split_bars_daily_(needs imporvement)", 3 | "visState": "{\"title\":\"New Visualization\",\"type\":\"histogram\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"scale\":\"linear\",\"mode\":\"grouped\",\"times\":[],\"addTimeMarker\":false,\"defaultYExtents\":false,\"setYExtents\":false,\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"product/ProductName.raw\",\"size\":5,\"order\":\"desc\",\"orderBy\":\"1\"}},{\"id\":\"3\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"d\",\"customInterval\":\"2h\",\"min_doc_count\":1,\"extended_bounds\":{}}}],\"listeners\":{}}", 4 | "uiStateJSON": "{}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/Spot_vs_OnDemand_EC2.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Spot_vs_OnDemand_EC2", 3 | "visState": "{\"title\":\"Spot_vs_OnDemand_EC2\",\"type\":\"pie\",\"params\":{\"shareYAxis\":true,\"addTooltip\":true,\"addLegend\":true,\"isDonut\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"filters\",\"schema\":\"segment\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"*AmazonEC2*\",\"analyze_wildcard\":true}}},\"label\":\"all Amazon EC2 Instances\"}]}},{\"id\":\"3\",\"type\":\"filters\",\"schema\":\"segment\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"*Spot*\",\"analyze_wildcard\":true}}},\"label\":\"Spot EC2 instaIces\"},{\"input\":{\"query\":{\"query_string\":{\"query\":\"NOT *Spot*\",\"analyze_wildcard\":true}}},\"label\":\"OnDemand EC2 Instances\"}]}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"vis\":{\"colors\":{\"all Amazon EC2 Instances\":\"#3F6833\",\"Spot EC2 instaIces\":\"#F29191\",\"OnDemand EC2 Instances\":\"#584477\"}}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/Total_UnblendedCost.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Total_UnblendedCost", 3 | "visState": "{\"title\":\"Total_UnblendedCost\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\",\"customLabel\":\"Total Cost\"}}],\"listeners\":{}}", 4 | "uiStateJSON": "{}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/api_call_table.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "api_call_table", 3 | "visState": "{\"title\":\"New Visualization\",\"type\":\"table\",\"params\":{\"perPage\":10,\"showPartialRows\":false,\"showMeticsAtAllLevels\":false},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"bucket\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"d\",\"customInterval\":\"2d\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"filters\",\"schema\":\"bucket\",\"params\":{\"filters\":[{\"input\":{\"query\":{\"query_string\":{\"query\":\"API Request\",\"analyze_wildcard\":true}}},\"label\":\"\"}]}}],\"listeners\":{}}", 4 | "uiStateJSON": "{}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/discover_search.json: -------------------------------------------------------------------------------- 1 | { 2 | "properties": { 3 | "columns": { 4 | "type": "string" 5 | }, 6 | "description": { 7 | "type": "string" 8 | }, 9 | "hits": { 10 | "type": "integer" 11 | }, 12 | "kibanaSavedObjectMeta": { 13 | "properties": { 14 | "searchSourceJSON": { 15 | "type": "string" 16 | } 17 | } 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /kibana/finalVisualization_5days_30min_line_split.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "finalVisualization_5days_30min_line_split", 3 | "visState": "{\"title\":\"finalVisualization_5days_30min_line_split\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"defaultYExtents\":true,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":\"3\",\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"custom\",\"customInterval\":\"30m\",\"min_doc_count\":1,\"extended_bounds\":{}}},{\"id\":\"3\",\"type\":\"significant_terms\",\"schema\":\"group\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":10}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"AmazonEC2\":\"#3F6833\",\"AmazonS3\":\"#967302\",\"AmazonRoute53\":\"#2F575E\",\"AmazonCloudFront\":\"#99440A\",\"AmazonSNS\":\"#58140C\"}}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/finalVisualization_5days_30min_row_split.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "finalVisualization_5days_30min_row_split", 3 | "visState": "{\"title\":\"finalVisualization_5days_30min_row_split\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"defaultYExtents\":true,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"custom\",\"customInterval\":\"30m\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"3\",\"type\":\"significant_terms\",\"schema\":\"group\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":6,\"customLabel\":\"view\"}},{\"id\":\"4\",\"type\":\"significant_terms\",\"schema\":\"split\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":6,\"customLabel\":\"service\",\"row\":true}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"AmazonEC2\":\"#3F6833\",\"AmazonRoute53\":\"#2F575E\",\"AmazonCloudFront\":\"#99440A\",\"AmazonS3\":\"#967302\",\"AmazonSNS\":\"#58140C\"}}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /kibana/kibana_dashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "AWS-Billing-DashBoard", 3 | "hits": 0, 4 | "description": "", 5 | "panelsJSON": "[{\"col\":1,\"id\":\"Pi-chart-for-seperate-services\",\"panelIndex\":1,\"row\":3,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":5,\"id\":\"Spot_vs_OnDemand_EC2\",\"panelIndex\":2,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":9,\"id\":\"Cost_For_AmazonS3_requests\",\"panelIndex\":5,\"row\":1,\"size_x\":4,\"size_y\":3,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Total_UnblendedCost\",\"panelIndex\":6,\"row\":1,\"size_x\":4,\"size_y\":2,\"type\":\"visualization\"},{\"col\":5,\"id\":\"S3_Api_Calls_daily\",\"panelIndex\":7,\"row\":4,\"size_x\":8,\"size_y\":2,\"type\":\"visualization\"},{\"col\":1,\"id\":\"Top-5-used-service-split-daily\",\"panelIndex\":8,\"row\":6,\"size_x\":12,\"size_y\":7,\"type\":\"visualization\"}]", 6 | "optionsJSON": "{\"darkTheme\":false}", 7 | "uiStateJSON": "{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"AmazonCloudFront\":\"#99440A\",\"AmazonEC2\":\"#3F6833\",\"AmazonRoute53\":\"#2F575E\",\"AmazonS3\":\"#967302\",\"AmazonSNS\":\"#58140C\"}}}", 8 | "version": 1, 9 | "timeRestore": false, 10 | "kibanaSavedObjectMeta": { 11 | "searchSourceJSON": "{\"filter\":[{\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}}}]}" 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /kibana/kibana_visualizations.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "_id": "finalVisualization_5days_30min_line_split", 4 | "_type": "visualization", 5 | "_source": 6 | }, 7 | { 8 | "_id": "Split_bars_daily_(needs-imporvement)", 9 | "_type": "visualization", 10 | "_source": 11 | }, 12 | { 13 | "_id": "Cost_For_AmazonS3_requests", 14 | "_type": "visualization", 15 | "_source": 16 | }, 17 | { 18 | "_id": "Pi-chart-for-seperate-services", 19 | "_type": "visualization", 20 | "_source": 21 | }, 22 | { 23 | "_id": "Spot_vs_OnDemand_EC2", 24 | "_type": "visualization", 25 | "_source": 26 | }, 27 | { 28 | "_id": "api_call_table", 29 | "_type": "visualization", 30 | "_source": 31 | }, 32 | { 33 | "_id": "Total_UnblendedCost", 34 | "_type": "visualization", 35 | "_source": 36 | }, 37 | { 38 | "_id": "S3_Api_Calls_daily", 39 | "_type": "visualization", 40 | "_source": 41 | } 42 | ] 43 | -------------------------------------------------------------------------------- /kibana/orchestrate_dashboard.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Import the default AWS-Billing-DashBoard json file if there is no such dashboard present 4 | 5 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/dashboard/AWS-Billing-DashBoard"`; 6 | 7 | if [[ $CONTENT == *'"found":true'* ]] 8 | then 9 | echo "Dashboard With Default Name Is Already There!"; 10 | else 11 | echo "Default Dashboard Is Being Created!"; 12 | curl -XPUT "http://elasticsearch:9200/.kibana/dashboard/AWS-Billing-DashBoard" -d "`cat kibana_dashboard.json`"; 13 | fi 14 | -------------------------------------------------------------------------------- /kibana/orchestrate_kibana.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Set default index without going to Kibana frontend 4 | 5 | curl "http://elasticsearch:9200/.kibana/config/4.5.0/_update" -d '{"doc": {"buildNum": 9889, "defaultIndex": "aws-billing*"}}' 6 | 7 | #Create kibana index_pattern 8 | # The content for this JSON is taken from template type 9 | 10 | curl "http://elasticsearch:9200/.kibana/index-pattern/aws-billing-*" -d '{"title" : "aws-billing-*", "timeFieldName" : "lineItem/UsageStartDate", "fields" : "[{\"name\":\"lineItem/ProductCode.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/toLocation.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/UsageAmount.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/BlendedRate.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/servicecode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"identity/LineItemId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/description.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/productFamily\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/Operation\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/maxThroughputvolume\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/volumeType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/availability.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/AvailabilityZone\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/groupDescription\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/Operation.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/InvoiceId.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"product/fromLocation.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"identity/LineItemId.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/vcpu\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/processorFeatures.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/UsageType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/locationType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/maxVolumeSize.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/BillType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"@timestamp\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/location\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/maxIopsBurstPerformance\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"bill/InvoiceId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/networkPerformance.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/physicalProcessor\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/instanceType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"port\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/maxIopsBurstPerformance.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/PayerAccountId.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/sku.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"@version\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/volumeType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/processorArchitecture\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/UsageAccountId.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/memory\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"host.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/UnblendedRate\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/AvailabilityZone.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/TaxType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/routingTarget\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/LineItemType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/operatingSystem\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/storageClass\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/sku\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/UnblendedCost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/fromLocationType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"identity/TimeInterval.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/availability\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/requestDescription.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"type.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/memory.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/storageMedia.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/BillingPeriodStartDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/maxVolumeSize\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/storage\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/UsageType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/CurrencyCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/ProductName\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/routingType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/preInstalledSw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/instanceType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"_source\",\"type\":\"_source\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"lineItem/UsageEndDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/maxIopsvolume\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/BlendedCost\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pricing/term.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/licenseModel.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/BillingEntity.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/requestDescription\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"bill/BillType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/servicecode.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/clockSpeed.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/maxIopsvolume.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/BlendedRate\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/productFamily.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/LineItemDescription.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/UsageAccountId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/fromLocationType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/instanceFamily\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"index\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/routingType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/UsageAmount\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/durability.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/LineItemDescription\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/storage.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/vcpu.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/TaxType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/networkPerformance\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/operatingSystem.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/clockSpeed\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/preInstalledSw.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"pricing/term\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/usagetype.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/operation.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/tenancy\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/toLocationType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/BillingPeriodEndDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/transferType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/description\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/transferType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"lineItem/UnblendedRate.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/tenancy.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/usagetype\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/storageMedia\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/toLocation\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/UsageStartDate\",\"type\":\"date\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/durability\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/fromLocation\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/routingTarget.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"identity/TimeInterval\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/group\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"bill/BillingEntity\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/group.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/licenseModel\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/requestType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/storageClass.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/toLocationType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/operation\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/ProductCode\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/CurrencyCode.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/processorArchitecture.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/locationType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"bill/PayerAccountId\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"lineItem/LineItemType\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/groupDescription.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/requestType.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/instanceFamily.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"index.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/processorFeatures\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":true,\"doc_values\":false},{\"name\":\"product/physicalProcessor.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/maxThroughputvolume.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/location.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"product/ProductName.raw\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":true,\"analyzed\":false,\"doc_values\":true},{\"name\":\"_id\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_type\",\"type\":\"string\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false},{\"name\":\"_score\",\"type\":\"number\",\"count\":0,\"scripted\":false,\"indexed\":false,\"analyzed\":false,\"doc_values\":false}]"}' 11 | -------------------------------------------------------------------------------- /kibana/orchestrate_search_mapping.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Default search mapping for dashboard to work 4 | curl -XPUT "http://elasticsearch:9200/.kibana/_mappings/search" -d "`cat discover_search.json`" 5 | -------------------------------------------------------------------------------- /kibana/orchestrate_visualisation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Import visualisation json file if that doesn't exist 4 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_row_split"`; 5 | if [[ $CONTENT == *'"found":false'* ]] 6 | then 7 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_row_split" -d "`cat finalVisualization_5days_30min_row_split.json`"; 8 | fi 9 | 10 | 11 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_line_split"`; 12 | if [[ $CONTENT == *'"found":false'* ]] 13 | then 14 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_line_split" -d "`cat finalVisualization_5days_30min_line_split.json`"; 15 | fi 16 | 17 | 18 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/api_call_table"`; 19 | if [[ $CONTENT == *'"found":false'* ]] 20 | then 21 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/api_call_table" -d "`cat api_call_table.json`"; 22 | fi 23 | 24 | 25 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost"`; 26 | if [[ $CONTENT == *'"found":false'* ]] 27 | then 28 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost" -d "`cat Total_UnblendedCost.json`"; 29 | fi 30 | 31 | 32 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Spot_vs_OnDemand_EC2"`; 33 | if [[ $CONTENT == *'"found":false'* ]] 34 | then 35 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Spot_vs_OnDemand_EC2" -d "`cat Spot_vs_OnDemand_EC2.json`"; 36 | fi 37 | 38 | 39 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Split_bars_daily"`; 40 | if [[ $CONTENT == *'"found":false'* ]] 41 | then 42 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Split_bars_daily" -d "`cat Split_bars_daily.json`"; 43 | fi 44 | 45 | 46 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/S3_Api_Calls_daily"`; 47 | if [[ $CONTENT == *'"found":false'* ]] 48 | then 49 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/S3_Api_Calls_daily" -d "`cat S3_Api_Calls_daily.json`"; 50 | fi 51 | 52 | 53 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Pi-chart-for-seperate-services"`; 54 | if [[ $CONTENT == *'"found":false'* ]] 55 | then 56 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Pi-chart-for-seperate-services" -d "`cat Pi-chart-for-seperate-services.json`"; 57 | fi 58 | 59 | 60 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Cost_For_AmazonS3_requests"`; 61 | if [[ $CONTENT == *'"found":false'* ]] 62 | then 63 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Cost_For_AmazonS3_requests" -d "`cat Cost_For_AmazonS3_requests.json`"; 64 | fi 65 | 66 | 67 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Top-5-used-service-split-daily"`; 68 | if [[ $CONTENT == *'"found":false'* ]] 69 | then 70 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Top-5-used-service-split-daily" -d "`cat top_5_used_service_split_daily.json`"; 71 | fi 72 | -------------------------------------------------------------------------------- /kibana/top_5_used_service_split_daily.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Top 5 used service split daily", 3 | "visState": "{\"title\":\"Top 5 used service split daily\",\"type\":\"line\",\"params\":{\"addLegend\":true,\"addTimeMarker\":true,\"addTooltip\":true,\"defaultYExtents\":true,\"drawLinesBetweenPoints\":true,\"interpolate\":\"linear\",\"radiusRatio\":9,\"scale\":\"linear\",\"setYExtents\":false,\"shareYAxis\":true,\"showCircles\":true,\"smoothLines\":true,\"times\":[],\"yAxis\":{}},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/UnblendedCost\"}},{\"id\":\"2\",\"type\":\"date_histogram\",\"schema\":\"segment\",\"params\":{\"field\":\"lineItem/UsageStartDate\",\"interval\":\"d\",\"customInterval\":\"30m\",\"min_doc_count\":1,\"extended_bounds\":{},\"customLabel\":\"\"}},{\"id\":\"3\",\"type\":\"terms\",\"schema\":\"group\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":31,\"order\":\"asc\",\"orderBy\":\"1\"}},{\"id\":\"4\",\"type\":\"significant_terms\",\"schema\":\"split\",\"params\":{\"field\":\"lineItem/ProductCode.raw\",\"size\":31,\"customLabel\":\"service\",\"row\":true}}],\"listeners\":{}}", 4 | "uiStateJSON": "{\"spy\":{\"mode\":{\"fill\":false,\"name\":null}},\"vis\":{\"colors\":{\"AmazonEC2\":\"#3F6833\",\"AmazonRoute53\":\"#2F575E\",\"AmazonCloudFront\":\"#99440A\",\"AmazonS3\":\"#967302\",\"AmazonSNS\":\"#58140C\"}}}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"analyze_wildcard\":true,\"query\":\"*\"}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /logstash.conf: -------------------------------------------------------------------------------- 1 | input { 2 | tcp { 3 | port => 5140 4 | codec => json 5 | } 6 | } 7 | 8 | filter { 9 | if [type] == "aws_billing_hourly" or [type] == "aws_billing_monthly" { 10 | grok { 11 | match => { "lineItem/UsageStartDate" => "%{YEAR:billing_year}-%{MONTHNUM2:billing_month}" } 12 | } 13 | 14 | mutate { 15 | replace => [ "index", "aws-billing-%{billing_year}.%{billing_month}" ] 16 | remove_field => [ "message" ] 17 | } 18 | } 19 | } 20 | 21 | output { 22 | elasticsearch { 23 | hosts => [ "elasticsearch:9200" ] 24 | index => "%{index}" 25 | } 26 | #stdout { codec => rubydebug } 27 | } 28 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // aws-billing project main.go 2 | package main 3 | import ( 4 | "encoding/csv" 5 | "encoding/json" 6 | "flag" 7 | "fmt" 8 | "io" 9 | "math" 10 | "net" 11 | "os" 12 | "regexp" 13 | "runtime" 14 | "strconv" 15 | "strings" 16 | "sync" 17 | "time" 18 | ) 19 | 20 | var ( 21 | tagMatcher = regexp.MustCompile("(user|aws):.*") 22 | dateMonthMatcher = regexp.MustCompile(`\d{4}-\d{2}`) 23 | monthlyCostAllocationMatcher = regexp.MustCompile(`.*aws-cost-allocation.*`) 24 | detailedBillingWithResourcesMatcher = regexp.MustCompile(`.*billing-report.*`) 25 | ReportMatchers = []*regexp.Regexp{monthlyCostAllocationMatcher, detailedBillingWithResourcesMatcher} 26 | wg = sync.WaitGroup{} 27 | 28 | // Flag variables 29 | file = flag.String("file", "billing_report_2016-05.csv", "CSV billing file to load.") 30 | logstashAddress = flag.String("logstash-address", "logstash:5140", "Address and port for the logstash tcp listener.") 31 | concurrency = flag.Int("concurrency", 2, "Number of cores to use.") 32 | //accountsFile = flag.String("accounts-file", "aws-cost-allocation-2014-06.csv", "CSV file containing LinkedAccountId and account LinkedAccountName") 33 | ) 34 | 35 | var FieldTypes = map[string]func(s string, report *BillingReport) interface{} { 36 | "PayerAccountId": ParseInt, 37 | "RateId": ParseInt, 38 | "SubscriptionId": ParseInt, 39 | "PricingPlanId": ParseInt, 40 | "UsageQuantity": ParseFloat, 41 | "BlendedRate": ParseFloat, 42 | "BlendedCost": ParseFloat, 43 | "UnBlendedRate": ParseFloat, 44 | "UnBlendedCost": ParseFloat, 45 | "TotalCost": ParseFloat, 46 | "Cost": ParseFloat, 47 | "CostBeforeTax": ParseFloat, 48 | "Credits": ParseFloat, 49 | "UsageStartDate": ParseDate, 50 | "UsageEndDate": ParseDate, 51 | "BillingPeriodStartDate": ParseBillingPeriodDate, 52 | "BillingPeriodEndDate": ParseBillingPeriodDate, 53 | } 54 | 55 | type DetailedLineItem struct { 56 | InvoiceID string 57 | PayerAccountId int 58 | LinkedAccountId int 59 | } 60 | 61 | const ( 62 | DetailedBillingWithResourcesAndTags = iota 63 | MonthlyCostAllocation = iota 64 | ) 65 | 66 | type FieldMapper map[string]map[interface{}]map[string]interface{} 67 | 68 | type BillingReport struct { 69 | CsvFileName string 70 | InvoicePeriod time.Time 71 | ReportType int 72 | Fields []string 73 | csvReader *csv.Reader 74 | Mapper FieldMapper 75 | } 76 | 77 | func main() { 78 | flag.Parse() 79 | fmt.Println("AWS billing CSV report parser!") 80 | var out = make(chan []byte) 81 | fmt.Println("out after make(chan []byte) is %v", out) 82 | go PublishRecord(out) 83 | //var csvFileName = "376681487066-aws-billing-detailed-line-items-with-resources-and-tags-2014-06.csv" 84 | 85 | report := OpenBillingReport(*file) 86 | 87 | /*if *accountsFile != "" { 88 | accountReport := OpenBillingReport(*accountsFile) 89 | report.Mapper = ReportToAccountFieldMap(accountReport) 90 | }*/ 91 | 92 | valuesSink := make(chan []string) 93 | reportSinks := make([]chan map[string]interface{}, 0, 0) 94 | jsonSinks := make([]chan []byte, 0, 0) 95 | 96 | for i := 0; i < runtime.GOMAXPROCS(*concurrency); i++ { 97 | reportSink := make(chan map[string]interface{}) 98 | jsonSink := make(chan []byte) 99 | reportSinks = append(reportSinks, reportSink) 100 | jsonSinks = append(jsonSinks, jsonSink) 101 | go ParseRecord(valuesSink, reportSink, report) 102 | go RecordToJson(reportSink, jsonSink) 103 | go PublishRecord(jsonSink) 104 | wg.Add(3) 105 | } 106 | 107 | var recordNum int 108 | for { 109 | values, err := report.csvReader.Read() 110 | if err == io.EOF { 111 | break 112 | } 113 | valuesSink <- values 114 | recordNum += 1 115 | if math.Mod(float64(recordNum), 10000) == 0 { 116 | fmt.Println(recordNum) 117 | } 118 | } 119 | close(valuesSink) 120 | wg.Wait() 121 | } 122 | 123 | func ReportToAccountFieldMap(report *BillingReport) FieldMapper { 124 | mapper := FieldMapper{} 125 | mapper["LinkedAccountId"] = make(map[interface{}]map[string]interface{}) 126 | for { 127 | values, err := report.csvReader.Read() 128 | if err == io.EOF { 129 | break 130 | } else if err != nil { 131 | panic(err.Error()) 132 | } 133 | 134 | var linkedAccountId string 135 | var linkedAccountName string 136 | for i, field := range report.Fields { 137 | if field == "LinkedAccountId" { 138 | linkedAccountId = values[i] 139 | } 140 | if field == "LinkedAccountName" { 141 | linkedAccountName = values[i] 142 | } 143 | } 144 | if _, ok := mapper["LinkedAccountId"][linkedAccountId]; !ok { 145 | fmt.Println("Creating map for: ", linkedAccountId) 146 | mapper["LinkedAccountId"][linkedAccountId] = make(map[string]interface{}) 147 | } 148 | mapper["LinkedAccountId"][linkedAccountId]["LinkedAccountName"] = linkedAccountName 149 | } 150 | return mapper 151 | } 152 | 153 | func ParseRecord(in chan []string, out chan map[string]interface{}, report *BillingReport) { 154 | var parsedValue interface{} 155 | for values := range in { 156 | var record = make(map[string]interface{}) 157 | record["Tags"] = make(map[string]interface{}) 158 | record["type"] = report.TypeString() 159 | for i, field := range report.Fields { 160 | if f, ok := FieldTypes[field]; ok { 161 | parsedValue = f(values[i], report) 162 | record[field] = parsedValue 163 | } else if tagMatcher.MatchString(field) { 164 | record["Tags"].(map[string]interface{})[ParseTag(field, report).(string)] = strings.ToLower(values[i]) 165 | parsedValue = strings.ToLower(values[i]) 166 | } else { 167 | record[field] = values[i] 168 | parsedValue = values[i] 169 | } 170 | if valueMap, ok := report.Mapper[field][parsedValue]; ok { 171 | for k, v := range valueMap { 172 | record[k] = v 173 | } 174 | } 175 | } 176 | out <- record 177 | } 178 | close(out) 179 | wg.Done() 180 | } 181 | 182 | func RecordToJson(in chan map[string]interface{}, out chan []byte) { 183 | for record := range in { 184 | jbRecord, err := json.Marshal(record) 185 | if err != nil { 186 | panic(err.Error()) 187 | } 188 | out <- jbRecord 189 | } 190 | close(out) 191 | wg.Done() 192 | } 193 | 194 | func (report *BillingReport) TypeString() string { 195 | switch report.ReportType { 196 | case DetailedBillingWithResourcesAndTags: 197 | return "aws_billing_hourly" 198 | case MonthlyCostAllocation: 199 | return "aws_billing_monthly" 200 | } 201 | return "" 202 | } 203 | 204 | func OpenBillingReport(csvFileName string) *BillingReport { 205 | fmt.Println("Opening billing report filename", csvFileName) 206 | /* 207 | AWS billing report filename must contain at least yyyy-mm 208 | */ 209 | invoicePeriod, err := time.Parse("2006-01", dateMonthMatcher.FindString(csvFileName)) 210 | if err != nil { 211 | panic(err.Error()) 212 | } 213 | file, err := os.Open(csvFileName) 214 | if err != nil { 215 | panic(err.Error()) 216 | } 217 | reader := csv.NewReader(file) 218 | report := &BillingReport{ 219 | CsvFileName: csvFileName, 220 | InvoicePeriod: invoicePeriod, 221 | csvReader: reader} 222 | 223 | switch { 224 | case detailedBillingWithResourcesMatcher.MatchString(csvFileName): 225 | report.ReportType = DetailedBillingWithResourcesAndTags 226 | case monthlyCostAllocationMatcher.MatchString(csvFileName): 227 | report.ReportType = MonthlyCostAllocation 228 | } 229 | 230 | if report.ReportType == MonthlyCostAllocation { 231 | //Burn one due to comment 232 | fmt.Println("Monthly cost allocation, skipping comment:") 233 | fmt.Println(reader.Read()) 234 | reader.FieldsPerRecord = 0 235 | } 236 | fields, err := reader.Read() 237 | if err != nil { 238 | panic(err.Error()) 239 | } 240 | report.Fields = fields 241 | return report 242 | } 243 | 244 | func ParseTag(s string, report *BillingReport) interface{} { 245 | tagParts := strings.Split(s, ":") 246 | return strings.Join(tagParts, "_") 247 | } 248 | 249 | func ParseInt(s string, report *BillingReport) interface{} { 250 | value, _ := strconv.ParseInt(s, 0, 0) 251 | return value 252 | } 253 | 254 | func ParseUint(s string, report *BillingReport) interface{} { 255 | value, _ := strconv.ParseUint(s, 0, 0) 256 | return value 257 | } 258 | 259 | func ParseFloat(s string, report *BillingReport) interface{} { 260 | value, _ := strconv.ParseFloat(s, 0) 261 | return value 262 | } 263 | 264 | func ParseDate(s string, report *BillingReport) interface{} { 265 | var returnTime time.Time 266 | var err error 267 | switch s { 268 | case "": 269 | returnTime = report.InvoicePeriod 270 | default: 271 | returnTime, err = time.Parse("2006-01-02 15:04:05", s) 272 | if err != nil { 273 | panic(err.Error()) 274 | } 275 | } 276 | return returnTime.Format(time.RFC3339) 277 | } 278 | 279 | func ParseBillingPeriodDate(s string, report *BillingReport) interface{} { 280 | var returnTime time.Time 281 | var err error 282 | switch s { 283 | case "": 284 | returnTime = report.InvoicePeriod 285 | default: 286 | returnTime, err = time.Parse("2006/01/02 15:04:05", s) 287 | if err != nil { 288 | panic(err.Error()) 289 | } 290 | } 291 | return returnTime.Format(time.RFC3339) 292 | } 293 | 294 | func PublishRecord(in chan []byte) { 295 | fmt.Println("Publishing record to logstash %v", in) 296 | con, err := net.DialTimeout("tcp", *logstashAddress, (5 * time.Second)) 297 | if err != nil { 298 | panic(err.Error()) 299 | } 300 | for record := range in { 301 | _, err = con.Write(record) 302 | if err != nil { 303 | panic(err.Error()) 304 | } 305 | _, err = con.Write([]byte("\n")) 306 | if err != nil { 307 | panic(err.Error()) 308 | } 309 | } 310 | wg.Done() 311 | } 312 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | _ "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestCostAllocationMatcher(t *testing.T) { 9 | if !monthlyCostAllocationMatcher.MatchString("376681487066-aws-cost-allocation-2013-06.csv") { 10 | t.Fail() 11 | } 12 | } 13 | 14 | func TestDetailedBillingWithResourcesMatcher(t *testing.T) { 15 | if !detailedBillingWithResourcesMatcher.MatchString("376681487066-aws-billing-detailed-line-items-with-resources-and-tags-2014-03.csv") { 16 | t.Fail() 17 | } 18 | } 19 | 20 | func TestBillingReportTypeString(t *testing.T) { 21 | report := &BillingReport{} 22 | report.ReportType = MonthlyCostAllocation 23 | if report.TypeString() != "aws_billing_monthly" { 24 | t.Fail() 25 | } 26 | report.ReportType = DetailedBillingWithResourcesAndTags 27 | if report.TypeString() != "aws_billing_hourly" { 28 | t.Fail() 29 | } 30 | } 31 | 32 | func TestParseReport(t *testing.T) { 33 | in := make(chan []string) 34 | out := make(chan map[string]interface{}) 35 | report := OpenBillingReport("test-2014-06.csv") 36 | report.Mapper = FieldMapper{ 37 | "LinkedAccountId": { 38 | "041869798014": { 39 | "AccountName": "MYOB Advanced", 40 | }, 41 | }, 42 | } 43 | go ParseRecord(in, out, report) 44 | values, _ := report.csvReader.Read() 45 | in <- values 46 | close(in) 47 | } 48 | -------------------------------------------------------------------------------- /orchestrate-test.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append(os.path.join(os.path.dirname(__file__))) 3 | #pwd = os.path.dirname(__file__) 4 | ''' 5 | added head source directory in path for import from any location and relative testing and pwd for open() relative files 6 | ''' 7 | from tools.tools import Tools 8 | import boto3 9 | import subprocess 10 | import time 11 | 12 | if __name__ == '__main__': 13 | 14 | print('Orchestrate-test Running') 15 | #initialize the tools class 16 | tools = Tools() 17 | 18 | # checking for established connections between E-L-K 19 | tools.check_elk_connection() 20 | 21 | # function to index the default template mapping of the data 22 | tools.index_template() 23 | 24 | # index a sample test file with sum of unblended cost 1.24185686 25 | tools.index_csv('test/sample/test_ub_cost_2016-06.csv', '20160601-20160701') 26 | # rows of data in the csv, must be given as string 27 | data_count = '300' 28 | while(True): 29 | index_names = subprocess.check_output(['curl -XGET "elasticsearch:9200/_cat/indices/"'], shell=True, stderr=subprocess.PIPE) 30 | if 'aws-billing-2016.06' in index_names and data_count in index_names: 31 | break 32 | 33 | index_names = subprocess.check_output(['curl -XGET "elasticsearch:9200/_cat/indices/"'], shell=True, stderr=subprocess.PIPE) 34 | print(index_names) 35 | 36 | tools.index_kibana() 37 | tools.delete_csv_json_files() 38 | -------------------------------------------------------------------------------- /orchestrate.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append(os.path.join(os.path.dirname(__file__))) 3 | #pwd = os.path.dirname(__file__) 4 | ''' 5 | added head source directory in path for import from any location and relative testing and pwd for open() relative files 6 | ''' 7 | from tools.tools import Tools 8 | import boto3 9 | 10 | 11 | if __name__ == '__main__': 12 | 13 | # you must provide your credentials in the recomanded way, here we are 14 | # passing it by ENV variables 15 | s3 = boto3.client('s3') 16 | 17 | #initialize the tools class 18 | tools = Tools(s3) 19 | 20 | # checking for established connections between E-L-K 21 | tools.check_elk_connection() 22 | 23 | # function to index the default template mapping of the data 24 | tools.index_template() 25 | 26 | # getting the required buckets names to index from get_s3_bucket_dir_to_index() 27 | s3_dir_to_index = tools.get_s3_bucket_dir_to_index() 28 | if s3_dir_to_index == 1: 29 | print 'I could not find any billing report under Bucket ', os.environ['S3_BUCKET_NAME'], ' under Path ', os.environ['S3_REPORT_PATH'] 30 | sys.exit(1) 31 | 32 | # downloading the csv file with get_req_csv_from_s3() and then calling the index_csv() to index it in our elasticsearch 33 | for dir_name in s3_dir_to_index: 34 | gzip_filename = tools.get_latest_zip_filename(dir_name) 35 | csv_filename = tools.get_req_csv_from_s3(dir_name, gzip_filename) 36 | print(gzip_filename,csv_filename) 37 | tools.index_csv(csv_filename, dir_name) 38 | 39 | # function to index deafualt dashboards, viasualization and search mapping in the .kibana index of elasticsearch 40 | # kibana is indexed at last because the data will be ready to index at this time 41 | tools.index_kibana() 42 | 43 | # delete the intermediate files 44 | tools.delete_csv_json_files() 45 | -------------------------------------------------------------------------------- /prod.sample.env: -------------------------------------------------------------------------------- 1 | AWS_ACCESS_KEY_ID=YOUR_ACCESS_ID 2 | AWS_SECRET_ACCESS_KEY=YOUR_SECRET_KEY 3 | S3_BUCKET_NAME=BILLING_BUCKET_NAME 4 | S3_REPORT_PATH=PATH/AS/CONFIGURED/IN/AWS/REPORTS 5 | S3_REPORT_NAME=REPORT_NAME 6 | -------------------------------------------------------------------------------- /screenshots/aws-report-usage-cost-prodenv.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/screenshots/aws-report-usage-cost-prodenv.png -------------------------------------------------------------------------------- /screenshots/aws-report-usage-cost.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/screenshots/aws-report-usage-cost.png -------------------------------------------------------------------------------- /screenshots/kibana-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/screenshots/kibana-dashboard.png -------------------------------------------------------------------------------- /stop.sh: -------------------------------------------------------------------------------- 1 | echo -e "POST /containers/awselkbilling_kibana_1/kill?signal=SIGKILL HTTP/1.0\r\n" | nc -U /var/run/docker.sock; 2 | echo -e "POST /containers/awselkbilling_logstash_1/kill?signal=SIGKILL HTTP/1.0\r\n" | nc -U /var/run/docker.sock; 3 | echo -e "POST /containers/awselkbilling_elasticsearch_1/kill?signal=SIGKILL HTTP/1.0\r\n" | nc -U /var/run/docker.sock; 4 | -------------------------------------------------------------------------------- /test/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/test/__init__.py -------------------------------------------------------------------------------- /test/sample/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/test/sample/__init__.py -------------------------------------------------------------------------------- /test/scripts/Total_BlendedCost.json: -------------------------------------------------------------------------------- 1 | { 2 | "title": "Total_BlendedCost", 3 | "visState": "{\"title\":\"Total_BlendedCost\",\"type\":\"metric\",\"params\":{\"handleNoResults\":true,\"fontSize\":60},\"aggs\":[{\"id\":\"1\",\"type\":\"sum\",\"schema\":\"metric\",\"params\":{\"field\":\"lineItem/BlendedCost\",\"customLabel\":\"Total Cost\"}}],\"listeners\":{}}", 4 | "uiStateJSON": "{}", 5 | "description": "", 6 | "version": 1, 7 | "kibanaSavedObjectMeta": { 8 | "searchSourceJSON": "{\"index\":\"aws-billing-*\",\"query\":{\"query_string\":{\"query\":\"*\",\"analyze_wildcard\":true}},\"filter\":[]}" 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /test/scripts/add_del_modify_vis.sh: -------------------------------------------------------------------------------- 1 | #This script modify the Default visualizations for checking the persistance in the other test script 'check_add_del_modify.sh' 2 | #check there must not exist visualization Total_BlendedCost 3 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Total_BlendedCost"`; 4 | if [[ $CONTENT == *'"found":true'* ]] 5 | then 6 | echo 'Total_BlendedCost visualization exists, something is wrong... aborting' 7 | exit 1; 8 | fi 9 | echo 'Total_BlendedCost visualization doesnt exist, will create it...' 10 | 11 | #create new visualization Tolal_BlendedCost 12 | CREATE_VIS=$(curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Total_BlendedCost" -d "`cat test/scripts/Total_BlendedCost.json`"); 13 | #delete default visualization Total_UnblendedCost 14 | DELETE_VIS=`curl -XDELETE "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost"`; 15 | 16 | -------------------------------------------------------------------------------- /test/scripts/check_add_del_modify_vis.sh: -------------------------------------------------------------------------------- 1 | #This scripts check the persistance of the modification made by test script 'add_del_modify_vis.sh' 2 | #check if the default visualization Total_UnblendedCost recreated or not 3 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost"`; 4 | if [[ $CONTENT == *'"found":false'* ]] 5 | then 6 | echo 'The default visualization Total_UnblendedCost could not be recreated, test unsuccessful :(' 7 | exit 1; 8 | fi 9 | echo 'The default visualization Total_UnblendedCost recreated, test success :)' 10 | 11 | #check if the newly created visualization Total_BlendedCost persists or not 12 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Total_BlendedCost"`; 13 | if [[ $CONTENT == *'"found":false'* ]] 14 | then 15 | echo 'The newly created visualization Total_BlendedCost did not persist, test unsuccessful :(' 16 | exit 1; 17 | fi 18 | echo 'The newly created visualization Total_BlendedCost persists, test success :)' 19 | 20 | #delete the all index to make sure local teardown takes place 21 | DELETE_INDEX=`curl -XDELETE "http://elasticsearch:9200/*"` 22 | exit 0; 23 | -------------------------------------------------------------------------------- /test/scripts/orchestrate_visualisation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | #Import visualisation json file if that doesn't exist 4 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_row_split"`; 5 | if [[ $CONTENT == *'"found":false'* ]] 6 | then 7 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_row_split" -d "`cat finalVisualization_5days_30min_row_split.json`"; 8 | fi 9 | 10 | 11 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_line_split"`; 12 | if [[ $CONTENT == *'"found":false'* ]] 13 | then 14 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/finalVisualization_5days_30min_line_split" -d "`cat finalVisualization_5days_30min_line_split.json`"; 15 | fi 16 | 17 | 18 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/api_call_table"`; 19 | if [[ $CONTENT == *'"found":false'* ]] 20 | then 21 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/api_call_table" -d "`cat api_call_table.json`"; 22 | fi 23 | 24 | 25 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost"`; 26 | if [[ $CONTENT == *'"found":false'* ]] 27 | then 28 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Total_UnblendedCost" -d "`cat Total_UnblendedCost.json`"; 29 | fi 30 | 31 | 32 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Spot_vs_OnDemand_EC2"`; 33 | if [[ $CONTENT == *'"found":false'* ]] 34 | then 35 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Spot_vs_OnDemand_EC2" -d "`cat Spot_vs_OnDemand_EC2.json`"; 36 | fi 37 | 38 | 39 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Split_bars_daily"`; 40 | if [[ $CONTENT == *'"found":false'* ]] 41 | then 42 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Split_bars_daily" -d "`cat Split_bars_daily.json`"; 43 | fi 44 | 45 | 46 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/S3_Api_Calls_daily"`; 47 | if [[ $CONTENT == *'"found":false'* ]] 48 | then 49 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/S3_Api_Calls_daily" -d "`cat S3_Api_Calls_daily.json`"; 50 | fi 51 | 52 | 53 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Pi-chart-for-seperate-services"`; 54 | if [[ $CONTENT == *'"found":false'* ]] 55 | then 56 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Pi-chart-for-seperate-services" -d "`cat Pi-chart-for-seperate-services.json`"; 57 | fi 58 | 59 | 60 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Cost_For_AmazonS3_requests"`; 61 | if [[ $CONTENT == *'"found":false'* ]] 62 | then 63 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Cost_For_AmazonS3_requests" -d "`cat Cost_For_AmazonS3_requests.json`"; 64 | fi 65 | 66 | 67 | CONTENT=`curl -XGET "http://elasticsearch:9200/.kibana/visualization/Top-5-used-service-split-daily"`; 68 | if [[ $CONTENT == *'"found":false'* ]] 69 | then 70 | curl -XPUT "http://elasticsearch:9200/.kibana/visualization/Top-5-used-service-split-daily" -d "`cat top_5_used_service_split_daily.json`"; 71 | fi 72 | -------------------------------------------------------------------------------- /test/tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/test/tools/__init__.py -------------------------------------------------------------------------------- /test/tools/aggregate.json: -------------------------------------------------------------------------------- 1 | { 2 | "size": 0, 3 | "aggregations": { 4 | "sum_ub_cost": { 5 | "sum": { 6 | "field": "lineItem/UnblendedCost" 7 | } 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /test/tools/check_tools.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append(os.path.join(os.path.dirname(__file__),'..','..','..')) 3 | ''' 4 | added head source directory in path for import from any location 5 | ''' 6 | from unittest import TestCase 7 | from nose.tools import assert_equals 8 | from nose.tools import assert_items_equal 9 | import logging 10 | from tools.tools import Tools 11 | import boto3 12 | import subprocess 13 | import simplejson 14 | 15 | log = logging.getLogger(__name__) 16 | TestCase.maxDiff = None 17 | TestCase.longMessage = True 18 | 19 | 20 | class Test_Functions: 21 | 22 | def __init__(self): 23 | self.check_tools = Tools() 24 | 25 | def test_check_elk_connection(self): 26 | assert_equals( 27 | self.check_tools.check_elk_connection(), 28 | True, 29 | 'Must return True for E-L-K connections successfull' 30 | ) 31 | 32 | def test_ub_cost(self): 33 | result = subprocess.check_output(['curl -XGET "http://elasticsearch:9200/aws-billing-2016.06/_search" -d "`cat /aws-elk-billing/test/tools/aggregate.json`"'], 34 | shell=True, stderr=subprocess.PIPE) 35 | result = subprocess.check_output(['curl -XGET "http://elasticsearch:9200/aws-billing-2016.06/_search" -d "`cat /aws-elk-billing/test/tools/aggregate.json`"'], 36 | shell=True, stderr=subprocess.PIPE) 37 | result = subprocess.check_output(['curl -XGET "http://elasticsearch:9200/aws-billing-2016.06/_search" -d "`cat /aws-elk-billing/test/tools/aggregate.json`"'], 38 | shell=True, stderr=subprocess.PIPE) 39 | print(result) 40 | result = simplejson.loads(result) 41 | sum_ub_cost = result["aggregations"]["sum_ub_cost"]["value"] 42 | assert_equals( 43 | float(format(sum_ub_cost,'.3f')), 44 | 0.201, 45 | 'Must return the exact sum as the csv file' 46 | ) 47 | -------------------------------------------------------------------------------- /tools/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/300Geeks/aws-elk-billing/5672d2c9780f34b6c45a80e1f5a7362ec60bfeea/tools/__init__.py -------------------------------------------------------------------------------- /tools/tools.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | sys.path.append(os.path.join(os.path.dirname(__file__))) 3 | ''' 4 | added head source directory in path for import from any location and relative testing and pwd for open() relative files 5 | ''' 6 | import pyelasticsearch as pyes 7 | import boto3 8 | import time 9 | import socket 10 | import sys 11 | from datetime import datetime as dtdt 12 | from datetime import date as dtd 13 | from dateutil.relativedelta import relativedelta 14 | import subprocess 15 | import os 16 | 17 | class Tools: 18 | 19 | def __init__(self, s3=None): 20 | if s3: 21 | self.bucketname = os.environ['S3_BUCKET_NAME'] 22 | self.path_name_s3_billing = [ i for i in os.environ['S3_REPORT_PATH'].split('/') if i] 23 | self.s3_report_name = os.environ['S3_REPORT_NAME'] 24 | self.s3 = s3 25 | else: 26 | pass 27 | 28 | def check_elk_connection(self): 29 | elasticsearch_socket = socket.socket() 30 | logstash_socket = socket.socket() 31 | kibana_socket = socket.socket() 32 | connection_ok = False 33 | for _ in range(15): 34 | try: 35 | print 'Checking if Elasticsearch container has started to listen to 9200' 36 | elasticsearch_socket.connect(('elasticsearch', 9200)) 37 | print 'Great Elasticsearch is listening on 9200, 9300 :)' 38 | connection_ok = True 39 | break 40 | except Exception as e: 41 | print( 42 | "Something's wrong with Elasticsearch. Exception is %s" % (e)) 43 | print 'I will retry after 4 seconds' 44 | connection_ok = True 45 | time.sleep(4) 46 | 47 | for _ in range(15): 48 | try: 49 | print 'Checking if Logstash container has started to listen to 5140' 50 | logstash_socket.connect(('logstash', 5140)) 51 | print 'Great Logstash is listening on 5140 :)' 52 | connection_ok = True 53 | break 54 | except Exception as e: 55 | print("Something's wrong with Logstash. Exception is %s" % (e)) 56 | print 'I will retry after 4 seconds' 57 | connection_ok = True 58 | time.sleep(4) 59 | 60 | for _ in range(15): 61 | try: 62 | print 'Checking if Kibana container has started to listen to 5601' 63 | kibana_socket.connect(('kibana', 5601)) 64 | print 'Great Kibana is listening on 5601 :)' 65 | connection_ok = True 66 | break 67 | except Exception as e: 68 | print("Something's wrong with Kibana. Exception is %s" % (e)) 69 | print 'I will retry after 4 seconds' 70 | connection_ok = True 71 | time.sleep(4) 72 | 73 | elasticsearch_socket.close() 74 | logstash_socket.close() 75 | kibana_socket.close() 76 | 77 | return connection_ok 78 | 79 | def index_template(self): 80 | out = subprocess.check_output(['curl --head "elasticsearch:9200/_template/aws_billing"'], shell=True, stderr=subprocess.PIPE) 81 | if '200 OK' not in out: 82 | status = subprocess.Popen( 83 | ['curl -XPUT elasticsearch:9200/_template/aws_billing -d "`cat /aws-elk-billing/aws-billing-es-template.json`"'], 84 | shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 85 | if status.wait() != 0: 86 | print 'Something went wrong while creating mapping index' 87 | sys.exit(1) 88 | else: 89 | print 'ES mapping created :)' 90 | else: 91 | print 'Template already exists' 92 | 93 | def get_s3_bucket_dir_to_index(self): 94 | if len(self.path_name_s3_billing) == 1: 95 | prefix = '/' + '/'.join(self.path_name_s3_billing) + '/' 96 | else: 97 | prefix='/'.join(self.path_name_s3_billing) + '/' 98 | 99 | key_names = self.s3.list_objects( 100 | Bucket=self.bucketname, 101 | Prefix=prefix, 102 | Delimiter='/') 103 | s3_dir_names = [] 104 | 105 | if 'CommonPrefixes' not in key_names: 106 | return 1 107 | 108 | for keys in key_names['CommonPrefixes']: 109 | s3_dir_names.append(keys['Prefix'].split('/')[-2]) 110 | 111 | s3_dir_names.sort() 112 | es = pyes.ElasticSearch('http://elasticsearch:9200') 113 | index_list = es.get_mapping('aws-billing*').keys() 114 | index_time = [] 115 | for i in index_list: 116 | if i: 117 | index_time.append(es.search(index=i, size=1, query={"query": {"match_all": {}}})[ 118 | 'hits']['hits'][0]['_source']['@timestamp']) 119 | 120 | index_time.sort(reverse=True) 121 | 122 | dir_start = 0 123 | dir_end = None 124 | 125 | if index_time: 126 | current_dir = dtd.today().strftime('%Y%m01') + '-' + (dtd.today() + \ 127 | relativedelta(months=1)).strftime('%Y%m01') 128 | 129 | last_ind_dir = index_time[0].split('T')[0].replace('-', '') 130 | last_ind_dir = dtdt.strptime(last_ind_dir, '%Y%m%d').strftime('%Y%m01') + '-' + ( 131 | dtdt.strptime(last_ind_dir, '%Y%m%d') + relativedelta(months=1)).strftime('%Y%m01') 132 | dir_start = s3_dir_names.index(last_ind_dir) 133 | dir_end = s3_dir_names.index(current_dir) + 1 134 | 135 | s3_dir_to_index = s3_dir_names[dir_start:dir_end] 136 | print('Months to be indexed: {}'.format(', '.join(s3_dir_to_index))) 137 | # returning only the dirnames which are to be indexed 138 | return s3_dir_to_index 139 | 140 | def get_latest_zip_filename(self, monthly_dir_name): 141 | # monthly_dir_name for aws s3 directory format for getting the correct json file 142 | # json file name 143 | if len(self.path_name_s3_billing) == 1: 144 | latest_json_file_name = '/' + '/'.join(self.path_name_s3_billing + [monthly_dir_name, self.s3_report_name + '-Manifest.json']) 145 | else: 146 | latest_json_file_name = '/'.join(self.path_name_s3_billing + [monthly_dir_name, self.s3_report_name + '-Manifest.json']) 147 | 148 | # download the jsonfile as getfile_$time.json from s3 149 | print('Downloading {}...'.format(latest_json_file_name)) 150 | self.s3.download_file( 151 | self.bucketname, 152 | latest_json_file_name, 153 | 'getfile.json') 154 | 155 | # read the json file to get the latest updated version of csv 156 | f = open('getfile.json', 'r') 157 | content = eval(f.read()) 158 | latest_gzip_filename = content['reportKeys'][0] 159 | f.close() 160 | return latest_gzip_filename 161 | 162 | def get_req_csv_from_s3(self, monthly_dir_name, latest_gzip_filename): 163 | # the local filename formated for compatibility with the go lang code billing_report_yyyy-mm.csv 164 | local_gz_filename = 'billing_report_' + \ 165 | dtdt.strptime(monthly_dir_name.split('-')[0], '%Y%m%d').strftime('%Y-%m') + '.csv.gz' 166 | local_csv_filename = local_gz_filename[:-3] 167 | 168 | # downloading the zipfile from s3 169 | self.s3.download_file(self.bucketname, latest_gzip_filename, local_gz_filename) 170 | 171 | # upzip and replace the .gz file with .csv file 172 | print("Extracting latest csv file") 173 | process_gunzip = subprocess.Popen(['gunzip -v ' + local_gz_filename], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 174 | 175 | return local_csv_filename 176 | 177 | def index_csv(self, filename, dir_name): 178 | 179 | # DELETE earlier aws-billing* index if exists for the current indexing month 180 | # current month index format (name) 181 | index_format = dtdt.strptime( 182 | dir_name.split('-')[0], 183 | '%Y%m%d').strftime('%Y.%m') 184 | os.environ['file_y_m'] = index_format 185 | # have to change the name of the index in logstash index=>indexname 186 | 187 | status = subprocess.Popen( 188 | ['curl -XDELETE elasticsearch:9200/aws-billing-' + index_format], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 189 | if status.wait() != 0: 190 | print 'I think there are no aws-billing* indice or it is outdated, its OK main golang code will create a new one for you :)' 191 | else: 192 | print 'aws-billing* indice deleted or Not found, its OK main golang code will create a new one for you :)' 193 | 194 | # Run the main golang code to parse the billing file and send it to 195 | # Elasticsearch over Logstash 196 | status = subprocess.Popen( 197 | ['go run /aws-elk-billing/main.go --file /aws-elk-billing/' + filename], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 198 | print(status.stdout.read()) 199 | print(status.stderr.read()) 200 | if status.wait() != 0: 201 | print 'Something went wrong while getting the file reference or while talking with logstash' 202 | sys.exit(1) 203 | else: 204 | print 'AWS Billing report sucessfully parsed and indexed in Elasticsearch via Logstash :)' 205 | 206 | def index_kibana(self): 207 | # Index the search mapping for Discover to work 208 | status = subprocess.Popen( 209 | ['(cd /aws-elk-billing/kibana; bash orchestrate_search_mapping.sh)'], 210 | shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 211 | if status.wait() != 0: 212 | print 'The Discover Search mapping failed to be indexed to .kibana index in Elasticsearch' 213 | sys.exit(1) 214 | else: 215 | print 'The Discover Search mapping sucessfully indexed to .kibana index in Elasticsearch, Kept intact if user already used it :)' 216 | 217 | 218 | # Index Kibana dashboard 219 | status = subprocess.Popen( 220 | ['(cd /aws-elk-billing/kibana; bash orchestrate_dashboard.sh)'], 221 | shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 222 | if status.wait() != 0: 223 | print 'AWS-Billing-DashBoard default dashboard failed to indexed to .kibana index in Elasticsearch' 224 | sys.exit(1) 225 | else: 226 | print 'AWS-Billing-DashBoard default dashboard sucessfully indexed to .kibana index in Elasticsearch, Kept intact if user already used it :)' 227 | 228 | # Index Kibana visualization 229 | status = subprocess.Popen( 230 | ['(cd /aws-elk-billing/kibana; bash orchestrate_visualisation.sh)'], 231 | shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 232 | if status.wait() != 0: 233 | print 'Kibana default visualizations failed to indexed to .kibana index in Elasticsearch' 234 | sys.exit(1) 235 | else: 236 | print 'Kibana default visualizations sucessfully indexed to .kibana index in Elasticsearch, Kept intact if user have already used it :)' 237 | 238 | def delete_csv_json_files(self): 239 | # delete all getfile json, csv files and part downloading files after indexing over 240 | process_delete_csv = subprocess.Popen( 241 | ["find /aws-elk-billing -name 'billing_report_*' -exec rm -f {} \;"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 242 | process_delete_json = subprocess.Popen( 243 | ["find /aws-elk-billing -name 'getfile*' -exec rm -f {} \;"], shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 244 | --------------------------------------------------------------------------------