├── .github └── workflows │ ├── main.yml │ └── release.yaml ├── .gitignore ├── Dockerfile ├── README.md ├── app ├── go.mod ├── go.sum └── main.go ├── docker-compose.yml ├── etc ├── alertmanager │ └── config.yml ├── cortex │ └── single-process-config.yaml ├── grafana │ └── provisioning │ │ ├── dashboards │ │ ├── Docker Prometheus Monitoring-1571332751387.json │ │ └── dashboard.yml │ │ └── datasources │ │ └── datasource.yml ├── mysqld_exporter │ └── .my.cnf └── prometheus │ ├── alert.rules │ └── prometheus.yml └── ops └── database └── schema.sql /.github/workflows/main.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: [master] 6 | pull_request: 7 | branches: [master] 8 | 9 | jobs: 10 | 11 | build: 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Stop MySql 16 | run: sudo service mysql stop 17 | - name: Run docker-compose 18 | run: docker-compose up -d 19 | - name: build app 20 | run: cd app && go build main.go 21 | -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | release: 3 | types: 4 | - created 5 | name: Build Release 6 | jobs: 7 | generate: 8 | name: Generate cross-platform builds 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: Checkout the repository 12 | uses: actions/checkout@v2 13 | - name: Generate build files 14 | uses: thatisuday/go-cross-build@v1 15 | with: 16 | platforms: 'linux/amd64, darwin/amd64, windows/amd64' 17 | package: 'app/main.go' 18 | name: 'app' 19 | compress: 'true' 20 | dest: 'dist' 21 | - name: Upload build-artifacts 22 | uses: skx/github-action-publish-binaries@master 23 | env: 24 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 25 | with: 26 | args: "./dist/*.tar.gz" 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /var 2 | main 3 | .idea -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:latest AS builder 2 | ADD ./app /app 3 | WORKDIR /app 4 | RUN go mod download 5 | RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o /main . 6 | 7 | FROM alpine:latest 8 | RUN apk --no-cache add ca-certificates 9 | COPY --from=builder /main ./ 10 | RUN chmod +x ./main 11 | ENTRYPOINT ["./main"] 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 📶 Prometheus/Grafana Docker + Sample Go App 2 | 3 | Monitoring System build using **Prometheus** (with some companion modules like **Alert Manager** and **MySQL Exporter**). It uses **Grafana** as the visualization dashboard. 4 | 5 | [![Actions Status](https://github.com/rubencougil/prometheus-go/workflows/Build/badge.svg)](https://github.com/rubencougil/prometheus-go/actions) 6 | [![Build Release](https://github.com/rubencougil/prometheus-go/workflows/Build%20Release/badge.svg)](https://github.com/rubencougil/prometheus-go/actions) 7 | 8 | ![Clase 3 - Practica Prometheus](https://github.com/rubencougil/prometheus-go/assets/1073799/b4d7ba04-cbf2-4e06-b072-d0cbf9bf1974) 9 | 10 | ## How to run it? 11 | 12 | To spin up needed services 13 | 14 | `docker-compose up -d` 15 | 16 | Once done, the Sample Go App will automatically send some metrics to Prometheus. Can be visualized in **Grafana** at: 17 | 18 | ``` 19 | http://localhost:3000 20 | User: admin 21 | Pass: foobar 22 | ``` 23 | 24 | ## Cortex 25 | 26 | Data Source URL for Grafana: http://cortex:9009/api/prom 27 | -------------------------------------------------------------------------------- /app/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/rubencougil/geekshubs/prometheus/app 2 | 3 | go 1.12 4 | 5 | require github.com/prometheus/client_golang v1.11.1 6 | -------------------------------------------------------------------------------- /app/go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= 2 | github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 3 | github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= 4 | github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 5 | github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= 6 | github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= 7 | github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= 8 | github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= 9 | github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= 10 | github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= 11 | github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= 12 | github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= 13 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 14 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 15 | github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 16 | github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= 17 | github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= 18 | github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= 19 | github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= 20 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 21 | github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= 22 | github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= 23 | github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 24 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 25 | github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 26 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 27 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 28 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 29 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 30 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 31 | github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 32 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 33 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 34 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 35 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 36 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 37 | github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 38 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 39 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 40 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 41 | github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= 42 | github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= 43 | github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 44 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 45 | github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= 46 | github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= 47 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 48 | github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 49 | github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= 50 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 51 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 52 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 53 | github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= 54 | github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= 55 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 56 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 57 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 58 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 59 | github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 60 | github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= 61 | github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 62 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 63 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 64 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 65 | github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= 66 | github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= 67 | github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= 68 | github.com/prometheus/client_golang v1.11.1 h1:+4eQaD7vAZ6DsfsxB15hbE0odUjGI5ARs9yskGu1v4s= 69 | github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= 70 | github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= 71 | github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 72 | github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= 73 | github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= 74 | github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= 75 | github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= 76 | github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= 77 | github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= 78 | github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= 79 | github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= 80 | github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= 81 | github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= 82 | github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= 83 | github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= 84 | github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= 85 | github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 88 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 89 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 90 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 91 | golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 92 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 93 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 94 | golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 95 | golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 96 | golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= 97 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 98 | golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 99 | golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= 100 | golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= 101 | golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 102 | golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 103 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 104 | golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 105 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 106 | golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 107 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 108 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 109 | golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 110 | golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 115 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= 116 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 117 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 118 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 119 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 120 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 121 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 122 | google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= 123 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 124 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 125 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 126 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 127 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 128 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 129 | google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= 130 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 131 | gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= 132 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 133 | gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 134 | gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 135 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 136 | gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 137 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 138 | gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 139 | -------------------------------------------------------------------------------- /app/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/prometheus/client_golang/prometheus" 7 | "github.com/prometheus/client_golang/prometheus/promauto" 8 | "github.com/prometheus/client_golang/prometheus/push" 9 | "log" 10 | "math/rand" 11 | "net/http" 12 | "time" 13 | 14 | "github.com/prometheus/client_golang/prometheus/promhttp" 15 | ) 16 | 17 | var addr = flag.String("listen-address", ":8081", "The address to listen on for HTTP requests.") 18 | 19 | var ( 20 | c = promauto.NewCounter(prometheus.CounterOpts{ 21 | Name: "geekshubs_app_sample_metric", 22 | Help: "Sample metric for GeeksHubs devops bootcamp", 23 | }) 24 | 25 | h = promauto.NewHistogram(prometheus.HistogramOpts{ 26 | Name: "geekshubs_app_sample_histogram", 27 | Help: "Sample histogram for GeeksHubs devops bootcamp", 28 | }) 29 | 30 | d = promauto.NewCounterVec(prometheus.CounterOpts{ 31 | Name: "geekshubs_app_sample_devices", 32 | Help: "Sample counter opts devices for GeeksHubs devops bootcamp"}, []string{"device"}) 33 | 34 | e = promauto.NewCounter(prometheus.CounterOpts{ 35 | Name: "geekshubs_app_push_metric", 36 | Help: "Sample metric for Geekshubs course (push)", 37 | }) 38 | ) 39 | 40 | func main() { 41 | 42 | go func() { 43 | for { 44 | rand.Seed(time.Now().UnixNano()) 45 | h.Observe(float64(rand.Intn(100-0+1) + 0)) 46 | d.With(prometheus.Labels{"device": "/dev/sda"}).Inc() 47 | c.Inc() 48 | fmt.Print(".") 49 | time.Sleep(1 * time.Second) 50 | } 51 | }() 52 | 53 | go func() { 54 | for { 55 | // Example of metric push 56 | err := push.New("http://pushgateway:9091", "geekshub_job").Collector(e).Add() 57 | if err != nil { 58 | _ = fmt.Errorf("%v", err) 59 | } 60 | e.Inc() 61 | fmt.Print("_") 62 | time.Sleep(1 * time.Second) 63 | } 64 | }() 65 | 66 | http.Handle("/metrics", promhttp.Handler()) 67 | log.Fatal(http.ListenAndServe(*addr, nil)) 68 | } 69 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | services: 3 | app: 4 | build: . 5 | restart: always 6 | ports: 7 | - "8081:8081" 8 | 9 | prometheus: 10 | image: prom/prometheus:v2.45.2 11 | #image: prom/prometheus-linux-arm64:v2.40.2 12 | user: root 13 | volumes: 14 | - "./etc/prometheus/:/etc/prometheus/" 15 | - "./var/prometheus/prometheus_data:/prometheus" 16 | command: 17 | - "--config.file=/etc/prometheus/prometheus.yml" 18 | - "--storage.tsdb.path=/prometheus" 19 | - "--web.console.libraries=/usr/share/prometheus/console_libraries" 20 | - "--web.console.templates=/usr/share/prometheus/consoles" 21 | - "--web.enable-lifecycle" 22 | ports: 23 | - "9090:9090" 24 | 25 | grafana: 26 | image: grafana/grafana 27 | user: root 28 | ports: 29 | - "3000:3000" 30 | volumes: 31 | - "./var/grafana/grafana_data:/var/lib/grafana" 32 | - "./etc/grafana/provisioning/:/etc/grafana/provisioning/" 33 | environment: 34 | - GF_SECURITY_ADMIN_PASSWORD=foobar 35 | - GF_USERS_ALLOW_SIGN_UP=false 36 | 37 | alertmanager: 38 | image: prom/alertmanager:v0.24.0 39 | ports: 40 | - 9093:9093 41 | volumes: 42 | - ./etc/alertmanager/:/etc/alertmanager/ 43 | command: 44 | - "--config.file=/etc/alertmanager/config.yml" 45 | - "--storage.path=/alertmanager" 46 | 47 | db: 48 | image: mysql:5.7 49 | #platform: linux/amd64 50 | restart: always 51 | environment: 52 | MYSQL_DATABASE: "db" 53 | MYSQL_USER: "user" 54 | MYSQL_PASSWORD: "password" 55 | MYSQL_ROOT_PASSWORD: "password" 56 | ports: 57 | - "3306:3306" 58 | volumes: 59 | - "./var/mysql:/var/lib/mysql" 60 | - "./ops/database:/docker-entrypoint-initdb.d" 61 | - "./var/log/mysql:/var/log/mysql" 62 | 63 | mysqld_exporter: 64 | image: prom/mysqld-exporter 65 | user: root 66 | ports: 67 | - "9104:9104" 68 | volumes: 69 | - "./etc/mysqld_exporter/.my.cnf:/.my.cnf" 70 | environment: 71 | - "collect.info_schema.tablestats=true" 72 | - "collect.info_schema.userstats=true" 73 | - "collect.info_schema.query_response_time=true" 74 | - "collect.auto_increment.columns=true" 75 | - "collect.perf_schema.tableiowaits=true" 76 | - "collect.perf_schema.tablelocks=true" 77 | depends_on: 78 | - db 79 | 80 | pushgateway: 81 | image: prom/pushgateway 82 | ports: 83 | - "9091:9091" 84 | depends_on: 85 | - prometheus 86 | 87 | #https://logz.io/blog/devops/prometheus-architecture-at-scale/ 88 | cortex: 89 | image: cortexproject/cortex:v0.1.0 90 | #image: ubuntu/cortex:latest 91 | ports: 92 | - "9009:9009" 93 | command: "-config.file=/etc/single-process-config.yaml" 94 | depends_on: 95 | - prometheus 96 | volumes: 97 | - "./etc/cortex/single-process-config.yaml:/etc/single-process-config.yaml" 98 | -------------------------------------------------------------------------------- /etc/alertmanager/config.yml: -------------------------------------------------------------------------------- 1 | 2 | route: 3 | group_by: [alertname, job] 4 | receiver: 'slack' 5 | repeat_interval: 5s 6 | 7 | global: 8 | # Also possible to place this URL in a file. 9 | # Ex: `slack_api_url_file: '/etc/alertmanager/slack_url'` 10 | slack_api_url: 'http://slack.me' 11 | 12 | receivers: 13 | - name: 'slack' 14 | slack_configs: 15 | - send_resolved: true 16 | title: |- 17 | [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }} 18 | {{- if gt (len .CommonLabels) (len .GroupLabels) -}} 19 | {{" "}}( 20 | {{- with .CommonLabels.Remove .GroupLabels.Names }} 21 | {{- range $index, $label := .SortedPairs -}} 22 | {{ if $index }}, {{ end }} 23 | {{- $label.Name }}="{{ $label.Value -}}" 24 | {{- end }} 25 | {{- end -}} 26 | ) 27 | {{- end }} 28 | text: >- 29 | {{ with index .Alerts 0 -}} 30 | :chart_with_upwards_trend: *<{{ .GeneratorURL }}|Graph>* 31 | {{- if .Annotations.runbook }} :notebook: *<{{ .Annotations.runbook }}|Runbook>*{{ end }} 32 | {{ end }} 33 | 34 | *Alert details*: 35 | 36 | {{ range .Alerts -}} 37 | *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} 38 | *Description:* {{ .Annotations.description }} 39 | *Details:* 40 | {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` 41 | {{ end }} 42 | {{ end }} -------------------------------------------------------------------------------- /etc/cortex/single-process-config.yaml: -------------------------------------------------------------------------------- 1 | # Configuration for running Cortex in single-process mode. 2 | # This should not be used in production. It is only for getting started 3 | # and development. 4 | 5 | # Disable the requirement that every request to Cortex has a 6 | # X-Scope-OrgID header. `fake` will be substituted in instead. 7 | auth_enabled: false 8 | 9 | server: 10 | http_listen_port: 9009 11 | 12 | # Configure the server to allow messages up to 100MB. 13 | grpc_server_max_recv_msg_size: 104857600 14 | grpc_server_max_send_msg_size: 104857600 15 | grpc_server_max_concurrent_streams: 1000 16 | 17 | distributor: 18 | shard_by_all_labels: true 19 | pool: 20 | health_check_ingesters: true 21 | 22 | ingester_client: 23 | grpc_client_config: 24 | # Configure the client to allow messages up to 100MB. 25 | max_recv_msg_size: 104857600 26 | max_send_msg_size: 104857600 27 | use_gzip_compression: true 28 | 29 | ingester: 30 | #chunk_idle_period: 15m 31 | 32 | lifecycler: 33 | # The address to advertise for this ingester. Will be autodiscovered by 34 | # looking up address on eth0 or en0; can be specified if this fails. 35 | # address: 127.0.0.1 36 | 37 | # We want to start immediately and flush on shutdown. 38 | join_after: 0 39 | claim_on_rollout: false 40 | final_sleep: 0s 41 | num_tokens: 512 42 | 43 | # Use an in memory ring store, so we don't need to launch a Consul. 44 | ring: 45 | kvstore: 46 | store: inmemory 47 | replication_factor: 1 48 | 49 | # Use local storage - BoltDB for the index, and the filesystem 50 | # for the chunks. 51 | schema: 52 | configs: 53 | - from: 2019-03-25 54 | store: boltdb 55 | object_store: filesystem 56 | schema: v10 57 | index: 58 | prefix: index_ 59 | period: 168h 60 | 61 | storage: 62 | boltdb: 63 | directory: /tmp/cortex/index 64 | 65 | filesystem: 66 | directory: /tmp/cortex/chunks -------------------------------------------------------------------------------- /etc/grafana/provisioning/dashboards/Docker Prometheus Monitoring-1571332751387.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "description": "Docker Monitoring Template", 16 | "editable": true, 17 | "gnetId": 179, 18 | "graphTooltip": 1, 19 | "id": 1, 20 | "iteration": 1571330223815, 21 | "links": [], 22 | "panels": [ 23 | { 24 | "collapsed": false, 25 | "datasource": null, 26 | "gridPos": { 27 | "h": 1, 28 | "w": 24, 29 | "x": 0, 30 | "y": 0 31 | }, 32 | "id": 17, 33 | "panels": [], 34 | "title": "Host Info", 35 | "type": "row" 36 | }, 37 | { 38 | "cacheTimeout": null, 39 | "colorBackground": false, 40 | "colorValue": false, 41 | "colors": [ 42 | "#299c46", 43 | "rgba(237, 129, 40, 0.89)", 44 | "#d44a3a" 45 | ], 46 | "datasource": "Prometheus", 47 | "decimals": null, 48 | "format": "s", 49 | "gauge": { 50 | "maxValue": 100, 51 | "minValue": 0, 52 | "show": false, 53 | "thresholdLabels": false, 54 | "thresholdMarkers": true 55 | }, 56 | "gridPos": { 57 | "h": 7, 58 | "w": 3, 59 | "x": 0, 60 | "y": 1 61 | }, 62 | "id": 15, 63 | "interval": null, 64 | "links": [], 65 | "mappingType": 1, 66 | "mappingTypes": [ 67 | { 68 | "name": "value to text", 69 | "value": 1 70 | }, 71 | { 72 | "name": "range to text", 73 | "value": 2 74 | } 75 | ], 76 | "maxDataPoints": 100, 77 | "nullPointMode": "connected", 78 | "nullText": null, 79 | "options": {}, 80 | "postfix": "", 81 | "postfixFontSize": "50%", 82 | "prefix": "", 83 | "prefixFontSize": "50%", 84 | "rangeMaps": [ 85 | { 86 | "from": "null", 87 | "text": "N/A", 88 | "to": "null" 89 | } 90 | ], 91 | "sparkline": { 92 | "fillColor": "rgba(31, 118, 189, 0.18)", 93 | "full": false, 94 | "lineColor": "rgb(31, 120, 193)", 95 | "show": false 96 | }, 97 | "tableColumn": "", 98 | "targets": [ 99 | { 100 | "expr": "time() - process_start_time_seconds{job=\"prometheus\"}", 101 | "format": "time_series", 102 | "intervalFactor": 1, 103 | "refId": "A" 104 | } 105 | ], 106 | "thresholds": "", 107 | "title": "Uptime", 108 | "type": "singlestat", 109 | "valueFontSize": "80%", 110 | "valueMaps": [ 111 | { 112 | "op": "=", 113 | "text": "N/A", 114 | "value": "null" 115 | } 116 | ], 117 | "valueName": "current" 118 | }, 119 | { 120 | "cacheTimeout": null, 121 | "colorBackground": false, 122 | "colorValue": false, 123 | "colors": [ 124 | "#299c46", 125 | "rgba(237, 129, 40, 0.89)", 126 | "#d44a3a" 127 | ], 128 | "datasource": "Prometheus", 129 | "format": "short", 130 | "gauge": { 131 | "maxValue": 100, 132 | "minValue": 0, 133 | "show": false, 134 | "thresholdLabels": false, 135 | "thresholdMarkers": true 136 | }, 137 | "gridPos": { 138 | "h": 4, 139 | "w": 3, 140 | "x": 3, 141 | "y": 1 142 | }, 143 | "id": 35, 144 | "interval": null, 145 | "links": [], 146 | "mappingType": 1, 147 | "mappingTypes": [ 148 | { 149 | "name": "value to text", 150 | "value": 1 151 | }, 152 | { 153 | "name": "range to text", 154 | "value": 2 155 | } 156 | ], 157 | "maxDataPoints": 100, 158 | "nullPointMode": "connected", 159 | "nullText": null, 160 | "options": {}, 161 | "postfix": "", 162 | "postfixFontSize": "50%", 163 | "prefix": "", 164 | "prefixFontSize": "50%", 165 | "rangeMaps": [ 166 | { 167 | "from": "null", 168 | "text": "N/A", 169 | "to": "null" 170 | } 171 | ], 172 | "sparkline": { 173 | "fillColor": "rgba(31, 118, 189, 0.18)", 174 | "full": false, 175 | "lineColor": "rgb(31, 120, 193)", 176 | "show": false, 177 | "ymax": null, 178 | "ymin": null 179 | }, 180 | "tableColumn": "", 181 | "targets": [ 182 | { 183 | "expr": "count(count(node_cpu_seconds_total{instance=~\"$node\", mode='system'}) by (cpu))", 184 | "instant": true, 185 | "refId": "A" 186 | } 187 | ], 188 | "thresholds": "", 189 | "timeFrom": null, 190 | "timeShift": null, 191 | "title": "CPU Cores", 192 | "type": "singlestat", 193 | "valueFontSize": "80%", 194 | "valueMaps": [ 195 | { 196 | "op": "=", 197 | "text": "N/A", 198 | "value": "null" 199 | } 200 | ], 201 | "valueName": "current" 202 | }, 203 | { 204 | "cacheTimeout": null, 205 | "colorBackground": true, 206 | "colorValue": false, 207 | "colors": [ 208 | "#299c46", 209 | "rgba(237, 129, 40, 0.89)", 210 | "#d44a3a" 211 | ], 212 | "datasource": "Prometheus", 213 | "format": "none", 214 | "gauge": { 215 | "maxValue": 100, 216 | "minValue": 0, 217 | "show": false, 218 | "thresholdLabels": false, 219 | "thresholdMarkers": true 220 | }, 221 | "gridPos": { 222 | "h": 7, 223 | "w": 5, 224 | "x": 6, 225 | "y": 1 226 | }, 227 | "id": 13, 228 | "interval": null, 229 | "links": [], 230 | "mappingType": 1, 231 | "mappingTypes": [ 232 | { 233 | "name": "value to text", 234 | "value": 1 235 | }, 236 | { 237 | "name": "range to text", 238 | "value": 2 239 | } 240 | ], 241 | "maxDataPoints": 100, 242 | "nullPointMode": "connected", 243 | "nullText": null, 244 | "options": {}, 245 | "postfix": "", 246 | "postfixFontSize": "50%", 247 | "prefix": "", 248 | "prefixFontSize": "50%", 249 | "rangeMaps": [ 250 | { 251 | "from": "null", 252 | "text": "N/A", 253 | "to": "null" 254 | } 255 | ], 256 | "sparkline": { 257 | "fillColor": "rgba(31, 118, 189, 0.18)", 258 | "full": false, 259 | "lineColor": "rgb(31, 120, 193)", 260 | "show": false 261 | }, 262 | "tableColumn": "", 263 | "targets": [ 264 | { 265 | "expr": "sum(ALERTS)", 266 | "format": "time_series", 267 | "intervalFactor": 1, 268 | "refId": "A" 269 | } 270 | ], 271 | "thresholds": "0,1", 272 | "title": "Alerts", 273 | "type": "singlestat", 274 | "valueFontSize": "80%", 275 | "valueMaps": [ 276 | { 277 | "op": "=", 278 | "text": "N/A", 279 | "value": "0" 280 | } 281 | ], 282 | "valueName": "avg" 283 | }, 284 | { 285 | "cacheTimeout": null, 286 | "colorBackground": true, 287 | "colorValue": false, 288 | "colors": [ 289 | "#d44a3a", 290 | "rgba(237, 129, 40, 0.89)", 291 | "#299c46" 292 | ], 293 | "datasource": "Prometheus", 294 | "format": "none", 295 | "gauge": { 296 | "maxValue": 100, 297 | "minValue": 0, 298 | "show": false, 299 | "thresholdLabels": false, 300 | "thresholdMarkers": true 301 | }, 302 | "gridPos": { 303 | "h": 7, 304 | "w": 4, 305 | "x": 11, 306 | "y": 1 307 | }, 308 | "id": 11, 309 | "interval": null, 310 | "links": [], 311 | "mappingType": 1, 312 | "mappingTypes": [ 313 | { 314 | "name": "value to text", 315 | "value": 1 316 | }, 317 | { 318 | "name": "range to text", 319 | "value": 2 320 | } 321 | ], 322 | "maxDataPoints": 100, 323 | "nullPointMode": "connected", 324 | "nullText": null, 325 | "options": {}, 326 | "postfix": "", 327 | "postfixFontSize": "50%", 328 | "prefix": "", 329 | "prefixFontSize": "50%", 330 | "rangeMaps": [ 331 | { 332 | "from": "null", 333 | "text": "N/A", 334 | "to": "null" 335 | } 336 | ], 337 | "sparkline": { 338 | "fillColor": "rgba(31, 118, 189, 0.18)", 339 | "full": false, 340 | "lineColor": "rgb(31, 120, 193)", 341 | "show": false 342 | }, 343 | "tableColumn": "", 344 | "targets": [ 345 | { 346 | "expr": "sum(up)", 347 | "format": "time_series", 348 | "intervalFactor": 1, 349 | "refId": "A" 350 | } 351 | ], 352 | "thresholds": "0,1", 353 | "title": "Targets Online", 354 | "type": "singlestat", 355 | "valueFontSize": "80%", 356 | "valueMaps": [ 357 | { 358 | "op": "=", 359 | "text": "N/A", 360 | "value": "null" 361 | } 362 | ], 363 | "valueName": "current" 364 | }, 365 | { 366 | "cacheTimeout": null, 367 | "colorBackground": false, 368 | "colorValue": false, 369 | "colors": [ 370 | "#d44a3a", 371 | "rgba(237, 129, 40, 0.89)", 372 | "#299c46" 373 | ], 374 | "datasource": "Prometheus", 375 | "format": "none", 376 | "gauge": { 377 | "maxValue": 100, 378 | "minValue": 0, 379 | "show": false, 380 | "thresholdLabels": false, 381 | "thresholdMarkers": true 382 | }, 383 | "gridPos": { 384 | "h": 7, 385 | "w": 4, 386 | "x": 15, 387 | "y": 1 388 | }, 389 | "id": 31, 390 | "interval": null, 391 | "links": [], 392 | "mappingType": 1, 393 | "mappingTypes": [ 394 | { 395 | "name": "value to text", 396 | "value": 1 397 | }, 398 | { 399 | "name": "range to text", 400 | "value": 2 401 | } 402 | ], 403 | "maxDataPoints": 100, 404 | "nullPointMode": "connected", 405 | "nullText": null, 406 | "options": {}, 407 | "postfix": "", 408 | "postfixFontSize": "50%", 409 | "prefix": "", 410 | "prefixFontSize": "50%", 411 | "rangeMaps": [ 412 | { 413 | "from": "null", 414 | "text": "N/A", 415 | "to": "null" 416 | } 417 | ], 418 | "sparkline": { 419 | "fillColor": "rgba(31, 118, 189, 0.18)", 420 | "full": false, 421 | "lineColor": "rgb(31, 120, 193)", 422 | "show": true 423 | }, 424 | "tableColumn": "", 425 | "targets": [ 426 | { 427 | "expr": "count(rate(container_last_seen{job=\"cadvisor\", name!=\"\"}[5m]))", 428 | "format": "time_series", 429 | "intervalFactor": 1, 430 | "refId": "A" 431 | } 432 | ], 433 | "thresholds": "0,1", 434 | "title": "Running Containers", 435 | "type": "singlestat", 436 | "valueFontSize": "80%", 437 | "valueMaps": [ 438 | { 439 | "op": "=", 440 | "text": "N/A", 441 | "value": "null" 442 | } 443 | ], 444 | "valueName": "current" 445 | }, 446 | { 447 | "cacheTimeout": null, 448 | "colorBackground": false, 449 | "colorValue": false, 450 | "colors": [ 451 | "#299c46", 452 | "rgba(237, 129, 40, 0.89)", 453 | "#d44a3a" 454 | ], 455 | "datasource": null, 456 | "decimals": null, 457 | "format": "decbytes", 458 | "gauge": { 459 | "maxValue": 100, 460 | "minValue": 0, 461 | "show": false, 462 | "thresholdLabels": false, 463 | "thresholdMarkers": true 464 | }, 465 | "gridPos": { 466 | "h": 3, 467 | "w": 3, 468 | "x": 3, 469 | "y": 5 470 | }, 471 | "id": 37, 472 | "interval": null, 473 | "links": [], 474 | "mappingType": 1, 475 | "mappingTypes": [ 476 | { 477 | "name": "value to text", 478 | "value": 1 479 | }, 480 | { 481 | "name": "range to text", 482 | "value": 2 483 | } 484 | ], 485 | "maxDataPoints": 100, 486 | "nullPointMode": "connected", 487 | "nullText": null, 488 | "options": {}, 489 | "postfix": "", 490 | "postfixFontSize": "50%", 491 | "prefix": "", 492 | "prefixFontSize": "50%", 493 | "rangeMaps": [ 494 | { 495 | "from": "null", 496 | "text": "N/A", 497 | "to": "null" 498 | } 499 | ], 500 | "sparkline": { 501 | "fillColor": "rgba(31, 118, 189, 0.18)", 502 | "full": false, 503 | "lineColor": "rgb(31, 120, 193)", 504 | "show": false, 505 | "ymax": null, 506 | "ymin": null 507 | }, 508 | "tableColumn": "", 509 | "targets": [ 510 | { 511 | "expr": "node_memory_MemTotal_bytes{instance=~\"$node\"}", 512 | "refId": "A" 513 | } 514 | ], 515 | "thresholds": "", 516 | "timeFrom": null, 517 | "timeShift": null, 518 | "title": "Host Memory", 519 | "type": "singlestat", 520 | "valueFontSize": "80%", 521 | "valueMaps": [ 522 | { 523 | "op": "=", 524 | "text": "N/A", 525 | "value": "null" 526 | } 527 | ], 528 | "valueName": "current" 529 | }, 530 | { 531 | "cacheTimeout": null, 532 | "colorBackground": false, 533 | "colorValue": false, 534 | "colors": [ 535 | "rgba(50, 172, 45, 0.97)", 536 | "rgba(237, 129, 40, 0.89)", 537 | "rgba(245, 54, 54, 0.9)" 538 | ], 539 | "datasource": "Prometheus", 540 | "editable": true, 541 | "error": false, 542 | "format": "percent", 543 | "gauge": { 544 | "maxValue": 100, 545 | "minValue": 0, 546 | "show": true, 547 | "thresholdLabels": false, 548 | "thresholdMarkers": true 549 | }, 550 | "gridPos": { 551 | "h": 6, 552 | "w": 6, 553 | "x": 0, 554 | "y": 8 555 | }, 556 | "id": 4, 557 | "interval": null, 558 | "isNew": true, 559 | "links": [], 560 | "mappingType": 1, 561 | "mappingTypes": [ 562 | { 563 | "name": "value to text", 564 | "value": 1 565 | }, 566 | { 567 | "name": "range to text", 568 | "value": 2 569 | } 570 | ], 571 | "maxDataPoints": 100, 572 | "nullPointMode": "connected", 573 | "nullText": null, 574 | "options": {}, 575 | "postfix": "", 576 | "postfixFontSize": "50%", 577 | "prefix": "", 578 | "prefixFontSize": "50%", 579 | "rangeMaps": [ 580 | { 581 | "from": "null", 582 | "text": "N/A", 583 | "to": "null" 584 | } 585 | ], 586 | "sparkline": { 587 | "fillColor": "rgba(31, 118, 189, 0.18)", 588 | "full": false, 589 | "lineColor": "rgb(31, 120, 193)", 590 | "show": false 591 | }, 592 | "tableColumn": "", 593 | "targets": [ 594 | { 595 | "expr": "(sum(node_memory_MemTotal_bytes) - sum(node_memory_MemFree_bytes +node_memory_Buffers_bytes + node_memory_Cached_bytes) ) / sum(node_memory_MemTotal_bytes) * 100", 596 | "format": "time_series", 597 | "interval": "10s", 598 | "intervalFactor": 1, 599 | "refId": "A", 600 | "step": 10 601 | } 602 | ], 603 | "thresholds": "65, 90", 604 | "title": "Memory usage", 605 | "type": "singlestat", 606 | "valueFontSize": "80%", 607 | "valueMaps": [ 608 | { 609 | "op": "=", 610 | "text": "N/A", 611 | "value": "null" 612 | } 613 | ], 614 | "valueName": "current" 615 | }, 616 | { 617 | "cacheTimeout": null, 618 | "colorBackground": false, 619 | "colorValue": false, 620 | "colors": [ 621 | "rgba(50, 172, 45, 0.97)", 622 | "rgba(237, 129, 40, 0.89)", 623 | "rgba(245, 54, 54, 0.9)" 624 | ], 625 | "datasource": "Prometheus", 626 | "decimals": 2, 627 | "editable": true, 628 | "error": false, 629 | "format": "percent", 630 | "gauge": { 631 | "maxValue": 100, 632 | "minValue": 0, 633 | "show": true, 634 | "thresholdLabels": false, 635 | "thresholdMarkers": true 636 | }, 637 | "gridPos": { 638 | "h": 6, 639 | "w": 6, 640 | "x": 6, 641 | "y": 8 642 | }, 643 | "id": 6, 644 | "interval": null, 645 | "isNew": true, 646 | "links": [], 647 | "mappingType": 1, 648 | "mappingTypes": [ 649 | { 650 | "name": "value to text", 651 | "value": 1 652 | }, 653 | { 654 | "name": "range to text", 655 | "value": 2 656 | } 657 | ], 658 | "maxDataPoints": 100, 659 | "nullPointMode": "connected", 660 | "nullText": null, 661 | "options": {}, 662 | "postfix": "", 663 | "postfixFontSize": "50%", 664 | "prefix": "", 665 | "prefixFontSize": "50%", 666 | "rangeMaps": [ 667 | { 668 | "from": "null", 669 | "text": "N/A", 670 | "to": "null" 671 | } 672 | ], 673 | "sparkline": { 674 | "fillColor": "rgba(31, 118, 189, 0.18)", 675 | "full": false, 676 | "lineColor": "rgb(31, 120, 193)", 677 | "show": false 678 | }, 679 | "tableColumn": "", 680 | "targets": [ 681 | { 682 | "expr": "100 - (avg(irate(node_cpu_seconds_total{instance=~\"$node\",mode=\"idle\"}[5m])) * 100)", 683 | "format": "time_series", 684 | "interval": "1m", 685 | "intervalFactor": 1, 686 | "legendFormat": "", 687 | "refId": "A", 688 | "step": 10 689 | } 690 | ], 691 | "thresholds": "65, 90", 692 | "title": "CPU usage", 693 | "type": "singlestat", 694 | "valueFontSize": "80%", 695 | "valueMaps": [ 696 | { 697 | "op": "=", 698 | "text": "N/A", 699 | "value": "null" 700 | } 701 | ], 702 | "valueName": "current" 703 | }, 704 | { 705 | "cacheTimeout": null, 706 | "colorBackground": false, 707 | "colorValue": false, 708 | "colors": [ 709 | "rgba(50, 172, 45, 0.97)", 710 | "rgba(237, 129, 40, 0.89)", 711 | "rgba(245, 54, 54, 0.9)" 712 | ], 713 | "datasource": "Prometheus", 714 | "decimals": 2, 715 | "editable": true, 716 | "error": false, 717 | "format": "percent", 718 | "gauge": { 719 | "maxValue": 100, 720 | "minValue": 0, 721 | "show": true, 722 | "thresholdLabels": false, 723 | "thresholdMarkers": true 724 | }, 725 | "gridPos": { 726 | "h": 6, 727 | "w": 7, 728 | "x": 12, 729 | "y": 8 730 | }, 731 | "id": 7, 732 | "interval": null, 733 | "isNew": true, 734 | "links": [], 735 | "mappingType": 1, 736 | "mappingTypes": [ 737 | { 738 | "name": "value to text", 739 | "value": 1 740 | }, 741 | { 742 | "name": "range to text", 743 | "value": 2 744 | } 745 | ], 746 | "maxDataPoints": 100, 747 | "nullPointMode": "connected", 748 | "nullText": null, 749 | "options": {}, 750 | "postfix": "", 751 | "postfixFontSize": "50%", 752 | "prefix": "", 753 | "prefixFontSize": "50%", 754 | "rangeMaps": [ 755 | { 756 | "from": "null", 757 | "text": "N/A", 758 | "to": "null" 759 | } 760 | ], 761 | "sparkline": { 762 | "fillColor": "rgba(31, 118, 189, 0.18)", 763 | "full": false, 764 | "lineColor": "rgb(31, 120, 193)", 765 | "show": false 766 | }, 767 | "tableColumn": "", 768 | "targets": [ 769 | { 770 | "expr": "avg( node_filesystem_avail_bytes {mountpoint=\"/\"} / node_filesystem_size_bytes{mountpoint=\"/\"})", 771 | "interval": "10s", 772 | "intervalFactor": 1, 773 | "metric": "", 774 | "refId": "A", 775 | "step": 10 776 | } 777 | ], 778 | "thresholds": "65, 90", 779 | "title": "Filesystem usage", 780 | "type": "singlestat", 781 | "valueFontSize": "80%", 782 | "valueMaps": [ 783 | { 784 | "op": "=", 785 | "text": "N/A", 786 | "value": "null" 787 | } 788 | ], 789 | "valueName": "current" 790 | }, 791 | { 792 | "aliasColors": { 793 | "RECEIVE": "#ea6460", 794 | "SENT": "#1f78c1", 795 | "TRANSMIT": "#1f78c1" 796 | }, 797 | "bars": false, 798 | "dashLength": 10, 799 | "dashes": false, 800 | "datasource": "Prometheus", 801 | "fill": 4, 802 | "fillGradient": 0, 803 | "gridPos": { 804 | "h": 9, 805 | "w": 6, 806 | "x": 0, 807 | "y": 14 808 | }, 809 | "id": 25, 810 | "legend": { 811 | "avg": false, 812 | "current": false, 813 | "max": false, 814 | "min": false, 815 | "show": true, 816 | "total": false, 817 | "values": false 818 | }, 819 | "lines": true, 820 | "linewidth": 1, 821 | "links": [], 822 | "nullPointMode": "null", 823 | "options": { 824 | "dataLinks": [] 825 | }, 826 | "percentage": false, 827 | "pointradius": 5, 828 | "points": false, 829 | "renderer": "flot", 830 | "seriesOverrides": [], 831 | "spaceLength": 10, 832 | "stack": false, 833 | "steppedLine": false, 834 | "targets": [ 835 | { 836 | "expr": "sum(rate(container_network_receive_bytes_total{id=\"/\"}[$interval])) by (id)", 837 | "format": "time_series", 838 | "interval": "2m", 839 | "intervalFactor": 2, 840 | "legendFormat": "RECEIVE", 841 | "refId": "A" 842 | }, 843 | { 844 | "expr": "- sum(rate(container_network_transmit_bytes_total{id=\"/\"}[$interval])) by (id)", 845 | "format": "time_series", 846 | "interval": "2m", 847 | "intervalFactor": 2, 848 | "legendFormat": "TRANSMIT", 849 | "refId": "B" 850 | } 851 | ], 852 | "thresholds": [], 853 | "timeFrom": null, 854 | "timeRegions": [], 855 | "timeShift": null, 856 | "title": "Node Network Traffic", 857 | "tooltip": { 858 | "shared": true, 859 | "sort": 0, 860 | "value_type": "cumulative" 861 | }, 862 | "type": "graph", 863 | "xaxis": { 864 | "buckets": null, 865 | "mode": "time", 866 | "name": null, 867 | "show": true, 868 | "values": [] 869 | }, 870 | "yaxes": [ 871 | { 872 | "format": "Bps", 873 | "label": null, 874 | "logBase": 1, 875 | "max": null, 876 | "min": null, 877 | "show": true 878 | }, 879 | { 880 | "format": "s", 881 | "label": null, 882 | "logBase": 1, 883 | "max": null, 884 | "min": null, 885 | "show": true 886 | } 887 | ], 888 | "yaxis": { 889 | "align": false, 890 | "alignLevel": null 891 | } 892 | }, 893 | { 894 | "aliasColors": { 895 | "Available Memory": "#508642", 896 | "Used Memory": "#bf1b00" 897 | }, 898 | "bars": false, 899 | "dashLength": 10, 900 | "dashes": false, 901 | "datasource": "Prometheus", 902 | "fill": 3, 903 | "fillGradient": 0, 904 | "gridPos": { 905 | "h": 9, 906 | "w": 6, 907 | "x": 6, 908 | "y": 14 909 | }, 910 | "id": 27, 911 | "legend": { 912 | "avg": false, 913 | "current": false, 914 | "max": false, 915 | "min": false, 916 | "show": true, 917 | "total": false, 918 | "values": false 919 | }, 920 | "lines": true, 921 | "linewidth": 1, 922 | "links": [], 923 | "nullPointMode": "null", 924 | "options": { 925 | "dataLinks": [] 926 | }, 927 | "percentage": false, 928 | "pointradius": 5, 929 | "points": false, 930 | "renderer": "flot", 931 | "seriesOverrides": [], 932 | "spaceLength": 10, 933 | "stack": true, 934 | "steppedLine": false, 935 | "targets": [ 936 | { 937 | "expr": "sum(node_memory_MemTotal_bytes) - sum(node_memory_MemAvailable_bytes)", 938 | "format": "time_series", 939 | "interval": "2m", 940 | "intervalFactor": 2, 941 | "legendFormat": "Used Memory", 942 | "refId": "B" 943 | }, 944 | { 945 | "expr": "sum(node_memory_MemAvailable_bytes)", 946 | "format": "time_series", 947 | "interval": "2m", 948 | "intervalFactor": 2, 949 | "legendFormat": "Available Memory", 950 | "refId": "A" 951 | } 952 | ], 953 | "thresholds": [], 954 | "timeFrom": null, 955 | "timeRegions": [], 956 | "timeShift": null, 957 | "title": "Node Mermory", 958 | "tooltip": { 959 | "shared": true, 960 | "sort": 0, 961 | "value_type": "individual" 962 | }, 963 | "type": "graph", 964 | "xaxis": { 965 | "buckets": null, 966 | "mode": "time", 967 | "name": null, 968 | "show": true, 969 | "values": [] 970 | }, 971 | "yaxes": [ 972 | { 973 | "format": "decbytes", 974 | "label": null, 975 | "logBase": 1, 976 | "max": null, 977 | "min": null, 978 | "show": true 979 | }, 980 | { 981 | "format": "s", 982 | "label": null, 983 | "logBase": 1, 984 | "max": null, 985 | "min": null, 986 | "show": true 987 | } 988 | ], 989 | "yaxis": { 990 | "align": false, 991 | "alignLevel": null 992 | } 993 | }, 994 | { 995 | "aliasColors": { 996 | "Available Memory": "#508642", 997 | "Free Storage": "#447ebc", 998 | "Total Storage Available": "#508642", 999 | "Used Memory": "#bf1b00", 1000 | "Used Storage": "#bf1b00" 1001 | }, 1002 | "bars": false, 1003 | "dashLength": 10, 1004 | "dashes": false, 1005 | "datasource": "Prometheus", 1006 | "fill": 3, 1007 | "fillGradient": 0, 1008 | "gridPos": { 1009 | "h": 9, 1010 | "w": 7, 1011 | "x": 12, 1012 | "y": 14 1013 | }, 1014 | "id": 28, 1015 | "legend": { 1016 | "avg": false, 1017 | "current": false, 1018 | "max": false, 1019 | "min": false, 1020 | "show": true, 1021 | "total": false, 1022 | "values": false 1023 | }, 1024 | "lines": true, 1025 | "linewidth": 1, 1026 | "links": [], 1027 | "nullPointMode": "null", 1028 | "options": { 1029 | "dataLinks": [] 1030 | }, 1031 | "percentage": false, 1032 | "pointradius": 5, 1033 | "points": false, 1034 | "renderer": "flot", 1035 | "seriesOverrides": [], 1036 | "spaceLength": 10, 1037 | "stack": true, 1038 | "steppedLine": false, 1039 | "targets": [ 1040 | { 1041 | "expr": "sum(node_filesystem_free_bytes {job=\"node-exporter\", instance=~\".*9100\", device=~\"/dev/.*\", mountpoint!=\"/var/lib/docker/aufs\"}) ", 1042 | "format": "time_series", 1043 | "interval": "2m", 1044 | "intervalFactor": 2, 1045 | "legendFormat": "Free Storage", 1046 | "refId": "A" 1047 | } 1048 | ], 1049 | "thresholds": [], 1050 | "timeFrom": null, 1051 | "timeRegions": [], 1052 | "timeShift": null, 1053 | "title": "Filesystem Available", 1054 | "tooltip": { 1055 | "shared": true, 1056 | "sort": 0, 1057 | "value_type": "individual" 1058 | }, 1059 | "type": "graph", 1060 | "xaxis": { 1061 | "buckets": null, 1062 | "mode": "time", 1063 | "name": null, 1064 | "show": true, 1065 | "values": [] 1066 | }, 1067 | "yaxes": [ 1068 | { 1069 | "decimals": null, 1070 | "format": "decbytes", 1071 | "label": null, 1072 | "logBase": 1, 1073 | "max": null, 1074 | "min": null, 1075 | "show": true 1076 | }, 1077 | { 1078 | "format": "s", 1079 | "label": null, 1080 | "logBase": 1, 1081 | "max": null, 1082 | "min": null, 1083 | "show": true 1084 | } 1085 | ], 1086 | "yaxis": { 1087 | "align": false, 1088 | "alignLevel": null 1089 | } 1090 | }, 1091 | { 1092 | "collapsed": false, 1093 | "datasource": null, 1094 | "gridPos": { 1095 | "h": 1, 1096 | "w": 24, 1097 | "x": 0, 1098 | "y": 23 1099 | }, 1100 | "id": 19, 1101 | "panels": [], 1102 | "repeat": null, 1103 | "title": "Container Performance", 1104 | "type": "row" 1105 | }, 1106 | { 1107 | "aliasColors": {}, 1108 | "bars": false, 1109 | "dashLength": 10, 1110 | "dashes": false, 1111 | "datasource": "Prometheus", 1112 | "decimals": 3, 1113 | "editable": true, 1114 | "error": false, 1115 | "fill": 0, 1116 | "fillGradient": 0, 1117 | "grid": {}, 1118 | "gridPos": { 1119 | "h": 10, 1120 | "w": 6, 1121 | "x": 0, 1122 | "y": 24 1123 | }, 1124 | "id": 3, 1125 | "isNew": true, 1126 | "legend": { 1127 | "alignAsTable": true, 1128 | "avg": true, 1129 | "current": true, 1130 | "max": false, 1131 | "min": false, 1132 | "rightSide": false, 1133 | "show": true, 1134 | "sort": "current", 1135 | "sortDesc": true, 1136 | "total": false, 1137 | "values": true 1138 | }, 1139 | "lines": true, 1140 | "linewidth": 2, 1141 | "links": [], 1142 | "nullPointMode": "connected", 1143 | "options": { 1144 | "dataLinks": [] 1145 | }, 1146 | "percentage": false, 1147 | "pointradius": 5, 1148 | "points": false, 1149 | "renderer": "flot", 1150 | "seriesOverrides": [], 1151 | "spaceLength": 10, 1152 | "stack": false, 1153 | "steppedLine": false, 1154 | "targets": [ 1155 | { 1156 | "expr": "sum(rate(container_cpu_usage_seconds_total{image!=\"\"}[1m])) by (id,name)", 1157 | "format": "time_series", 1158 | "interval": "10s", 1159 | "intervalFactor": 1, 1160 | "legendFormat": "{{ name }}", 1161 | "metric": "container_cpu_user_seconds_total", 1162 | "refId": "A", 1163 | "step": 10 1164 | } 1165 | ], 1166 | "thresholds": [], 1167 | "timeFrom": null, 1168 | "timeRegions": [], 1169 | "timeShift": null, 1170 | "title": "Container CPU usage", 1171 | "tooltip": { 1172 | "msResolution": true, 1173 | "shared": true, 1174 | "sort": 0, 1175 | "value_type": "cumulative" 1176 | }, 1177 | "type": "graph", 1178 | "xaxis": { 1179 | "buckets": null, 1180 | "mode": "time", 1181 | "name": null, 1182 | "show": true, 1183 | "values": [] 1184 | }, 1185 | "yaxes": [ 1186 | { 1187 | "format": "percentunit", 1188 | "label": null, 1189 | "logBase": 1, 1190 | "max": null, 1191 | "min": null, 1192 | "show": true 1193 | }, 1194 | { 1195 | "format": "short", 1196 | "label": null, 1197 | "logBase": 1, 1198 | "max": null, 1199 | "min": null, 1200 | "show": true 1201 | } 1202 | ], 1203 | "yaxis": { 1204 | "align": false, 1205 | "alignLevel": null 1206 | } 1207 | }, 1208 | { 1209 | "aliasColors": {}, 1210 | "bars": false, 1211 | "dashLength": 10, 1212 | "dashes": false, 1213 | "datasource": "Prometheus", 1214 | "decimals": 2, 1215 | "editable": true, 1216 | "error": false, 1217 | "fill": 0, 1218 | "fillGradient": 0, 1219 | "grid": {}, 1220 | "gridPos": { 1221 | "h": 10, 1222 | "w": 6, 1223 | "x": 6, 1224 | "y": 24 1225 | }, 1226 | "id": 2, 1227 | "isNew": true, 1228 | "legend": { 1229 | "alignAsTable": true, 1230 | "avg": true, 1231 | "current": true, 1232 | "max": false, 1233 | "min": false, 1234 | "rightSide": false, 1235 | "show": true, 1236 | "sort": "current", 1237 | "sortDesc": true, 1238 | "total": false, 1239 | "values": true 1240 | }, 1241 | "lines": true, 1242 | "linewidth": 2, 1243 | "links": [], 1244 | "nullPointMode": "connected", 1245 | "options": { 1246 | "dataLinks": [] 1247 | }, 1248 | "percentage": false, 1249 | "pointradius": 5, 1250 | "points": false, 1251 | "renderer": "flot", 1252 | "seriesOverrides": [], 1253 | "spaceLength": 10, 1254 | "stack": false, 1255 | "steppedLine": false, 1256 | "targets": [ 1257 | { 1258 | "expr": "container_memory_max_usage_bytes{image!=\"\"}", 1259 | "format": "time_series", 1260 | "interval": "10s", 1261 | "intervalFactor": 1, 1262 | "legendFormat": "{{ name }}", 1263 | "metric": "container_memory_usage:sort_desc", 1264 | "refId": "A", 1265 | "step": 10 1266 | } 1267 | ], 1268 | "thresholds": [], 1269 | "timeFrom": null, 1270 | "timeRegions": [], 1271 | "timeShift": null, 1272 | "title": "Container Memory Usage", 1273 | "tooltip": { 1274 | "msResolution": false, 1275 | "shared": true, 1276 | "sort": 0, 1277 | "value_type": "cumulative" 1278 | }, 1279 | "type": "graph", 1280 | "xaxis": { 1281 | "buckets": null, 1282 | "mode": "time", 1283 | "name": null, 1284 | "show": true, 1285 | "values": [] 1286 | }, 1287 | "yaxes": [ 1288 | { 1289 | "format": "bytes", 1290 | "label": null, 1291 | "logBase": 1, 1292 | "max": null, 1293 | "min": null, 1294 | "show": true 1295 | }, 1296 | { 1297 | "format": "short", 1298 | "label": null, 1299 | "logBase": 1, 1300 | "max": null, 1301 | "min": null, 1302 | "show": true 1303 | } 1304 | ], 1305 | "yaxis": { 1306 | "align": false, 1307 | "alignLevel": null 1308 | } 1309 | }, 1310 | { 1311 | "columns": [], 1312 | "datasource": "Prometheus", 1313 | "fontSize": "100%", 1314 | "gridPos": { 1315 | "h": 13, 1316 | "w": 10, 1317 | "x": 12, 1318 | "y": 24 1319 | }, 1320 | "id": 23, 1321 | "links": [], 1322 | "options": {}, 1323 | "pageSize": null, 1324 | "scroll": true, 1325 | "showHeader": true, 1326 | "sort": { 1327 | "col": 0, 1328 | "desc": true 1329 | }, 1330 | "styles": [ 1331 | { 1332 | "alias": "Time", 1333 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 1334 | "pattern": "Time", 1335 | "type": "date" 1336 | }, 1337 | { 1338 | "alias": "", 1339 | "colorMode": null, 1340 | "colors": [ 1341 | "rgba(245, 54, 54, 0.9)", 1342 | "rgba(237, 129, 40, 0.89)", 1343 | "rgba(50, 172, 45, 0.97)" 1344 | ], 1345 | "decimals": 2, 1346 | "pattern": "/.*/", 1347 | "thresholds": [], 1348 | "type": "number", 1349 | "unit": "short" 1350 | } 1351 | ], 1352 | "targets": [ 1353 | { 1354 | "expr": "ALERTS", 1355 | "format": "table", 1356 | "intervalFactor": 1, 1357 | "refId": "A" 1358 | } 1359 | ], 1360 | "title": "Alerts", 1361 | "transform": "table", 1362 | "type": "table" 1363 | }, 1364 | { 1365 | "aliasColors": {}, 1366 | "bars": false, 1367 | "dashLength": 10, 1368 | "dashes": false, 1369 | "datasource": "Prometheus", 1370 | "decimals": 2, 1371 | "editable": true, 1372 | "error": false, 1373 | "fill": 0, 1374 | "fillGradient": 0, 1375 | "grid": {}, 1376 | "gridPos": { 1377 | "h": 14, 1378 | "w": 6, 1379 | "x": 0, 1380 | "y": 34 1381 | }, 1382 | "id": 8, 1383 | "isNew": true, 1384 | "legend": { 1385 | "alignAsTable": true, 1386 | "avg": true, 1387 | "current": true, 1388 | "max": false, 1389 | "min": false, 1390 | "rightSide": false, 1391 | "show": true, 1392 | "sort": "current", 1393 | "sortDesc": true, 1394 | "total": false, 1395 | "values": true 1396 | }, 1397 | "lines": true, 1398 | "linewidth": 2, 1399 | "links": [], 1400 | "nullPointMode": "connected", 1401 | "options": { 1402 | "dataLinks": [] 1403 | }, 1404 | "percentage": false, 1405 | "pointradius": 5, 1406 | "points": false, 1407 | "renderer": "flot", 1408 | "seriesOverrides": [], 1409 | "spaceLength": 10, 1410 | "stack": false, 1411 | "steppedLine": false, 1412 | "targets": [ 1413 | { 1414 | "expr": "sort_desc(sum by (name) (rate(container_network_receive_bytes_total{image!=\"\"}[1m] ) ))", 1415 | "interval": "10s", 1416 | "intervalFactor": 1, 1417 | "legendFormat": "{{ name }}", 1418 | "metric": "container_network_receive_bytes_total", 1419 | "refId": "A", 1420 | "step": 10 1421 | } 1422 | ], 1423 | "thresholds": [], 1424 | "timeFrom": null, 1425 | "timeRegions": [], 1426 | "timeShift": null, 1427 | "title": "Container Network Input", 1428 | "tooltip": { 1429 | "msResolution": false, 1430 | "shared": true, 1431 | "sort": 0, 1432 | "value_type": "cumulative" 1433 | }, 1434 | "type": "graph", 1435 | "xaxis": { 1436 | "buckets": null, 1437 | "mode": "time", 1438 | "name": null, 1439 | "show": true, 1440 | "values": [] 1441 | }, 1442 | "yaxes": [ 1443 | { 1444 | "format": "bytes", 1445 | "label": null, 1446 | "logBase": 1, 1447 | "max": null, 1448 | "min": null, 1449 | "show": true 1450 | }, 1451 | { 1452 | "format": "short", 1453 | "label": null, 1454 | "logBase": 1, 1455 | "max": null, 1456 | "min": null, 1457 | "show": true 1458 | } 1459 | ], 1460 | "yaxis": { 1461 | "align": false, 1462 | "alignLevel": null 1463 | } 1464 | }, 1465 | { 1466 | "aliasColors": {}, 1467 | "bars": false, 1468 | "dashLength": 10, 1469 | "dashes": false, 1470 | "datasource": "Prometheus", 1471 | "decimals": 2, 1472 | "editable": true, 1473 | "error": false, 1474 | "fill": 0, 1475 | "fillGradient": 0, 1476 | "grid": {}, 1477 | "gridPos": { 1478 | "h": 14, 1479 | "w": 6, 1480 | "x": 6, 1481 | "y": 34 1482 | }, 1483 | "id": 9, 1484 | "isNew": true, 1485 | "legend": { 1486 | "alignAsTable": true, 1487 | "avg": true, 1488 | "current": true, 1489 | "max": false, 1490 | "min": false, 1491 | "rightSide": false, 1492 | "show": true, 1493 | "sort": "current", 1494 | "sortDesc": true, 1495 | "total": false, 1496 | "values": true 1497 | }, 1498 | "lines": true, 1499 | "linewidth": 2, 1500 | "links": [], 1501 | "nullPointMode": "connected", 1502 | "options": { 1503 | "dataLinks": [] 1504 | }, 1505 | "percentage": false, 1506 | "pointradius": 5, 1507 | "points": false, 1508 | "renderer": "flot", 1509 | "seriesOverrides": [], 1510 | "spaceLength": 10, 1511 | "stack": false, 1512 | "steppedLine": false, 1513 | "targets": [ 1514 | { 1515 | "expr": "sort_desc(sum by (name) (rate(container_network_transmit_bytes_total{image!=\"\"}[1m] ) ))", 1516 | "format": "time_series", 1517 | "intervalFactor": 2, 1518 | "legendFormat": "{{ name }}", 1519 | "metric": "container_network_transmit_bytes_total", 1520 | "refId": "B", 1521 | "step": 4 1522 | } 1523 | ], 1524 | "thresholds": [], 1525 | "timeFrom": null, 1526 | "timeRegions": [], 1527 | "timeShift": null, 1528 | "title": "Container Network Output", 1529 | "tooltip": { 1530 | "msResolution": false, 1531 | "shared": true, 1532 | "sort": 0, 1533 | "value_type": "cumulative" 1534 | }, 1535 | "type": "graph", 1536 | "xaxis": { 1537 | "buckets": null, 1538 | "mode": "time", 1539 | "name": null, 1540 | "show": true, 1541 | "values": [] 1542 | }, 1543 | "yaxes": [ 1544 | { 1545 | "format": "bytes", 1546 | "label": null, 1547 | "logBase": 1, 1548 | "max": null, 1549 | "min": null, 1550 | "show": true 1551 | }, 1552 | { 1553 | "format": "short", 1554 | "label": null, 1555 | "logBase": 1, 1556 | "max": null, 1557 | "min": null, 1558 | "show": false 1559 | } 1560 | ], 1561 | "yaxis": { 1562 | "align": false, 1563 | "alignLevel": null 1564 | } 1565 | }, 1566 | { 1567 | "columns": [], 1568 | "datasource": "Prometheus", 1569 | "fontSize": "100%", 1570 | "gridPos": { 1571 | "h": 10, 1572 | "w": 10, 1573 | "x": 12, 1574 | "y": 37 1575 | }, 1576 | "id": 30, 1577 | "links": [], 1578 | "options": {}, 1579 | "pageSize": 10, 1580 | "scroll": true, 1581 | "showHeader": true, 1582 | "sort": { 1583 | "col": 0, 1584 | "desc": true 1585 | }, 1586 | "styles": [ 1587 | { 1588 | "alias": "Time", 1589 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 1590 | "link": false, 1591 | "linkUrl": "", 1592 | "pattern": "Time", 1593 | "type": "date" 1594 | }, 1595 | { 1596 | "alias": "", 1597 | "colorMode": null, 1598 | "colors": [ 1599 | "rgba(245, 54, 54, 0.9)", 1600 | "rgba(237, 129, 40, 0.89)", 1601 | "rgba(50, 172, 45, 0.97)" 1602 | ], 1603 | "decimals": 2, 1604 | "pattern": "/.*/", 1605 | "thresholds": [], 1606 | "type": "number", 1607 | "unit": "short" 1608 | } 1609 | ], 1610 | "targets": [ 1611 | { 1612 | "expr": "cadvisor_version_info", 1613 | "format": "table", 1614 | "instant": false, 1615 | "interval": "15m", 1616 | "intervalFactor": 2, 1617 | "legendFormat": "cAdvisor Version: {{cadvisorVersion}}", 1618 | "refId": "A" 1619 | }, 1620 | { 1621 | "expr": "prometheus_build_info", 1622 | "format": "table", 1623 | "interval": "15m", 1624 | "intervalFactor": 2, 1625 | "legendFormat": "Prometheus Version: {{version}}", 1626 | "refId": "B" 1627 | }, 1628 | { 1629 | "expr": "node_exporter_build_info", 1630 | "format": "table", 1631 | "interval": "15m", 1632 | "intervalFactor": 2, 1633 | "legendFormat": "Node-Exporter Version: {{version}}", 1634 | "refId": "C" 1635 | } 1636 | ], 1637 | "title": "Running Versions", 1638 | "transform": "table", 1639 | "type": "table" 1640 | } 1641 | ], 1642 | "refresh": "10s", 1643 | "schemaVersion": 20, 1644 | "style": "dark", 1645 | "tags": [ 1646 | "docker", 1647 | "prometheus, ", 1648 | "node-exporter", 1649 | "cadvisor" 1650 | ], 1651 | "templating": { 1652 | "list": [ 1653 | { 1654 | "auto": false, 1655 | "auto_count": 30, 1656 | "auto_min": "10s", 1657 | "current": { 1658 | "text": "1m", 1659 | "value": "1m" 1660 | }, 1661 | "hide": 0, 1662 | "label": "interval", 1663 | "name": "interval", 1664 | "options": [ 1665 | { 1666 | "selected": true, 1667 | "text": "1m", 1668 | "value": "1m" 1669 | }, 1670 | { 1671 | "selected": false, 1672 | "text": "10m", 1673 | "value": "10m" 1674 | }, 1675 | { 1676 | "selected": false, 1677 | "text": "30m", 1678 | "value": "30m" 1679 | }, 1680 | { 1681 | "selected": false, 1682 | "text": "1h", 1683 | "value": "1h" 1684 | }, 1685 | { 1686 | "selected": false, 1687 | "text": "6h", 1688 | "value": "6h" 1689 | }, 1690 | { 1691 | "selected": false, 1692 | "text": "12h", 1693 | "value": "12h" 1694 | }, 1695 | { 1696 | "selected": false, 1697 | "text": "1d", 1698 | "value": "1d" 1699 | }, 1700 | { 1701 | "selected": false, 1702 | "text": "7d", 1703 | "value": "7d" 1704 | }, 1705 | { 1706 | "selected": false, 1707 | "text": "14d", 1708 | "value": "14d" 1709 | }, 1710 | { 1711 | "selected": false, 1712 | "text": "30d", 1713 | "value": "30d" 1714 | } 1715 | ], 1716 | "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", 1717 | "refresh": 2, 1718 | "skipUrlSync": false, 1719 | "type": "interval" 1720 | }, 1721 | { 1722 | "allValue": null, 1723 | "current": { 1724 | "text": "All", 1725 | "value": "$__all" 1726 | }, 1727 | "datasource": "Prometheus", 1728 | "definition": "label_values(node_exporter_build_info{name=~'$name'},instance)", 1729 | "hide": 0, 1730 | "includeAll": true, 1731 | "label": "IP", 1732 | "multi": true, 1733 | "name": "node", 1734 | "options": [], 1735 | "query": "label_values(node_exporter_build_info{name=~'$name'},instance)", 1736 | "refresh": 2, 1737 | "regex": "", 1738 | "skipUrlSync": false, 1739 | "sort": 1, 1740 | "tagValuesQuery": "", 1741 | "tags": [], 1742 | "tagsQuery": "", 1743 | "type": "query", 1744 | "useTags": false 1745 | }, 1746 | { 1747 | "allValue": null, 1748 | "current": { 1749 | "text": "All", 1750 | "value": "$__all" 1751 | }, 1752 | "datasource": "Prometheus", 1753 | "definition": "label_values(node_exporter_build_info,env)", 1754 | "hide": 0, 1755 | "includeAll": true, 1756 | "label": "Env", 1757 | "multi": true, 1758 | "name": "env", 1759 | "options": [], 1760 | "query": "label_values(node_exporter_build_info,env)", 1761 | "refresh": 2, 1762 | "regex": "", 1763 | "skipUrlSync": false, 1764 | "sort": 0, 1765 | "tagValuesQuery": "", 1766 | "tags": [], 1767 | "tagsQuery": "", 1768 | "type": "query", 1769 | "useTags": false 1770 | }, 1771 | { 1772 | "allValue": null, 1773 | "current": { 1774 | "text": "All", 1775 | "value": "$__all" 1776 | }, 1777 | "datasource": "Prometheus", 1778 | "definition": "label_values(node_exporter_build_info{env=~'$env'},name)", 1779 | "hide": 0, 1780 | "includeAll": true, 1781 | "label": "CPU Name", 1782 | "multi": true, 1783 | "name": "name", 1784 | "options": [], 1785 | "query": "label_values(node_exporter_build_info{env=~'$env'},name)", 1786 | "refresh": 2, 1787 | "regex": "", 1788 | "skipUrlSync": false, 1789 | "sort": 0, 1790 | "tagValuesQuery": "", 1791 | "tags": [], 1792 | "tagsQuery": "", 1793 | "type": "query", 1794 | "useTags": false 1795 | } 1796 | ] 1797 | }, 1798 | "time": { 1799 | "from": "now-5m", 1800 | "to": "now" 1801 | }, 1802 | "timepicker": { 1803 | "refresh_intervals": [ 1804 | "5s", 1805 | "10s", 1806 | "30s", 1807 | "1m", 1808 | "5m", 1809 | "15m", 1810 | "30m", 1811 | "1h", 1812 | "2h", 1813 | "1d" 1814 | ], 1815 | "time_options": [ 1816 | "5m", 1817 | "15m", 1818 | "1h", 1819 | "6h", 1820 | "12h", 1821 | "24h", 1822 | "2d", 1823 | "7d", 1824 | "30d" 1825 | ] 1826 | }, 1827 | "timezone": "browser", 1828 | "title": "Docker Prometheus Monitoring", 1829 | "uid": "64nrElFmk", 1830 | "version": 2 1831 | } -------------------------------------------------------------------------------- /etc/grafana/provisioning/dashboards/dashboard.yml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'Prometheus' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: true 10 | options: 11 | path: /etc/grafana/provisioning/dashboards 12 | -------------------------------------------------------------------------------- /etc/grafana/provisioning/datasources/datasource.yml: -------------------------------------------------------------------------------- 1 | # config file version 2 | apiVersion: 1 3 | 4 | # list of datasources that should be deleted from the database 5 | deleteDatasources: 6 | - name: Prometheus 7 | orgId: 1 8 | 9 | # list of datasources to insert/update depending 10 | # whats available in the database 11 | datasources: 12 | # name of the datasource. Required 13 | - name: Prometheus 14 | # datasource type. Required 15 | type: prometheus 16 | # access mode. direct or proxy. Required 17 | access: proxy 18 | # org id. will default to orgId 1 if not specified 19 | orgId: 1 20 | # url 21 | url: http://prometheus:9090 22 | # database password, if used 23 | password: 24 | # database user, if used 25 | user: 26 | # database name, if used 27 | database: 28 | # enable/disable basic auth 29 | basicAuth: false 30 | # basic auth username, if used 31 | basicAuthUser: 32 | # basic auth password, if used 33 | basicAuthPassword: 34 | # enable/disable with credentials headers 35 | withCredentials: 36 | # mark as default datasource. Max one per org 37 | isDefault: true 38 | # fields that will be converted to json and stored in json_data 39 | jsonData: 40 | graphiteVersion: "1.1" 41 | tlsAuth: false 42 | tlsAuthWithCACert: false 43 | # json object of data that will be encrypted. 44 | secureJsonData: 45 | tlsCACert: "..." 46 | tlsClientCert: "..." 47 | tlsClientKey: "..." 48 | version: 1 49 | # allow users to edit datasources from the UI. 50 | editable: true 51 | -------------------------------------------------------------------------------- /etc/mysqld_exporter/.my.cnf: -------------------------------------------------------------------------------- 1 | [client] 2 | user = root 3 | password = password 4 | host = db 5 | port = 3306 -------------------------------------------------------------------------------- /etc/prometheus/alert.rules: -------------------------------------------------------------------------------- 1 | groups: 2 | - name: example 3 | rules: 4 | 5 | # Go routines gt 10 6 | - alert: goRoutinesGt10 7 | expr: avg(go_goroutines) > 10 8 | for: 1m 9 | labels: 10 | severity: page 11 | annotations: 12 | summary: "Go Routines AVG grater than 10" 13 | 14 | # Go routines gt 100000 15 | - alert: goRoutinesGt100000 16 | expr: avg(go_goroutines) > 100000 17 | for: 1m 18 | labels: 19 | severity: page 20 | annotations: 21 | summary: "Go Routines AVG grater than 100000" 22 | 23 | - alert: MysqlDown 24 | expr: mysql_up == 0 25 | for: 0m 26 | labels: 27 | severity: critical 28 | annotations: 29 | summary: MySQL down (instance {{ $labels.instance }}) 30 | description: "MySQL instance is down on {{ $labels.instance }}\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" 31 | 32 | -------------------------------------------------------------------------------- /etc/prometheus/prometheus.yml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: 15s 3 | evaluation_interval: 15s 4 | external_labels: 5 | monitor: 'geekshubs' 6 | 7 | rule_files: 8 | - 'alert.rules' 9 | 10 | # Grafana URL: http://cortex:9009/api/prom 11 | remote_write: 12 | - url: http://cortex:9009/api/prom/push 13 | 14 | alerting: 15 | alertmanagers: 16 | - scheme: http 17 | static_configs: 18 | - targets: 19 | - "alertmanager:9093" 20 | 21 | scrape_configs: 22 | 23 | - job_name: 'prometheus' 24 | scrape_interval: 5s 25 | static_configs: 26 | - targets: ['localhost:9090'] 27 | 28 | # Doc: https://docs.docker.com/config/daemon/prometheus/ 29 | # Linux Workaround: https://stackoverflow.com/a/67158212/1341168 30 | - job_name: 'docker' 31 | scrape_interval: 5s 32 | static_configs: 33 | - targets: ['host.docker.internal:9323'] 34 | 35 | - job_name: 'geekshubs' 36 | scrape_interval: 5s 37 | static_configs: 38 | - targets: ['app:8081'] 39 | 40 | - job_name: 'mysqld_exporter' 41 | scrape_interval: 5s 42 | static_configs: 43 | - targets: ['mysqld_exporter:9104'] 44 | 45 | - job_name: 'pushgateway' 46 | scrape_interval: 10s 47 | honor_labels: true 48 | static_configs: 49 | - targets: ['pushgateway:9091'] 50 | -------------------------------------------------------------------------------- /ops/database/schema.sql: -------------------------------------------------------------------------------- 1 | CREATE USER 'mysqld_exporter'@'localhost' IDENTIFIED BY 'password' WITH MAX_USER_CONNECTIONS 3; 2 | 3 | CREATE TABLE IF NOT EXISTS `users` ( 4 | `id` INT NOT NULL AUTO_INCREMENT, 5 | `name` VARCHAR(255), 6 | `email` VARCHAR(255), 7 | PRIMARY KEY (`id`) 8 | ); --------------------------------------------------------------------------------