├── .gitignore ├── LICENSE ├── README.md ├── Vagrantfile ├── ansible ├── group_vars │ └── all ├── hosts ├── roles │ ├── common │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── ntp.conf.j2 │ ├── grafana │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── arm.yml │ │ │ ├── main.yml │ │ │ └── x86.yml │ │ └── templates │ │ │ └── grafana.ini.j2 │ ├── influxdb │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ └── main.yml │ │ └── templates │ │ │ └── influxdb.conf.j2 │ ├── ispstats │ │ ├── handlers │ │ │ └── main.yml │ │ ├── tasks │ │ │ ├── grafana_dashboard.yml │ │ │ ├── grafana_datasource.yml │ │ │ ├── grafana_user.yml │ │ │ ├── influxdb.yml │ │ │ └── main.yml │ │ └── templates │ │ │ ├── dashboard.json.j2 │ │ │ ├── ispstats.service.j2 │ │ │ └── ispstats.timer.j2 │ └── nginx │ │ ├── handlers │ │ └── main.yml │ │ ├── tasks │ │ └── main.yml │ │ └── templates │ │ └── default.conf.j2 └── site.yml ├── images └── grafana.png └── stats.py /.gitignore: -------------------------------------------------------------------------------- 1 | .git 2 | .DS_Store 3 | .DS_Store? 4 | ansible/site.retry 5 | .vagrant/ 6 | ansible/grafana_key 7 | ansible/grafana_password 8 | ansible/influxdb_password 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Christian Mehlmauer 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ISP STATS 2 | 3 | ![Grafana](/images/grafana.png?raw=true "Grafana") 4 | 5 | ## Description 6 | The following package is meant to provide an easy way to regularly measure your internet connection speed and create some stats. 7 | You can easily set up this script on a raspberry pi with raspbian and keep it running in the background. 8 | 9 | The data can be used to confront your ISP if your speed is way below the promised speed. 10 | 11 | For setting up the machine you need a linux machine because ansible does not run on windows. 12 | 13 | The setup runs [speedtest-cli](https://github.com/sivel/speedtest-cli) every hour and collects the stats from the nearest [speedtest.net](http://www.speedtest.net/) server. 14 | The data is written into a time series database ([InfluxDB](https://www.influxdata.com/time-series-platform/influxdb/)) and displayed in a preconfigured dashboard using [Grafana](http://grafana.org/). You can also export all data from the dashboard as CSV for further offline processing. 15 | 16 | **Hint:** If you want to use a raspberry older than model `3 B+` and have a fast internet connection (>100Mbps) you can not use the device as it can only handle 100Mbps at a maximum. To handle greater speeds you need to use a [Raspberry 3 B+](https://www.raspberrypi.org/products/raspberry-pi-3-model-b-plus/). 17 | 18 | ## Using Ansible 19 | * Install python and git 20 | * Install Ansible: http://docs.ansible.com/ansible/intro_installation.html#installing-the-control-machine 21 | * Clone repo `git clone https://github.com/FireFart/isp_stats.git` 22 | * edit `ansible/hosts` and change IP address 23 | * Be sure `root` login via SSH key is enabled on the remote machine and your public key is installed 24 | * `ansible-playbook -i ansible/hosts ansible/site.yml` 25 | * Open `http://ÌP/dashboard/db/isp-stats` in your browser 26 | * The admin User is `admin`, the password can be found in `ansible/grafana_password` (do not delete this file!) 27 | * All passwords and keys are generated on the first run so there are no default passwords to change after installation 28 | 29 | ## Using Vagrant 30 | `vagrant up` 31 | 32 | ## Install on a raspberry pi 33 | * download the raspbian-lite zip and extract it https://www.raspberrypi.org/downloads/raspbian/ 34 | * write the extracted image to sdcard https://www.raspberrypi.org/documentation/installation/installing-images/ 35 | * Create an empty file in the mounted `boot` partition with name `ssh` (without an extension) to enable the ssh deamon on start 36 | * Connect the raspberry using an ethernet cable to your router. Don't use WIFI as it is probably slower than your internet connection and would give false statistics. If your internet connection is faster or around 100Mbps you will need the use a [Raspberry 3 B+](https://www.raspberrypi.org/products/raspberry-pi-3-model-b-plus/) or later as older raspberries only have a 10/100 ethernet interface and a shared USB-Bus which limits the overall speed. Model 3B+ added a Gigabit Interface so the device is now able to handle greater speeds. 37 | * Connect power 38 | * Login via SSH with user `pi` and password `raspberry` 39 | * sudo raspi-config 40 | * Expand Filesystem 41 | * Change User Password 42 | * Change Locale 43 | * Change Timezone 44 | * Finish --> Reboot 45 | * `mkdir /root/.ssh` and add your public key to `/root/.ssh/authorized_keys` 46 | * Run ansible as described above 47 | 48 | ## My current Raspberry Pi Configuration (not working at full speed at the moment) 49 | * [Raspberry 3 B+](https://www.raspberrypi.org/products/raspberry-pi-3-model-b-plus/) 50 | * 8GB micro SD card 51 | * OS is raspbian-lite (no GUI) -------------------------------------------------------------------------------- /Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure(2) do |config| 2 | config.vm.box = "debian/jessie64" 3 | 4 | config.vm.provision "ansible" do |ansible| 5 | ansible.playbook = "ansible/site.yml" 6 | ansible.sudo = true 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /ansible/group_vars/all: -------------------------------------------------------------------------------- 1 | --- 2 | # influxdb settings 3 | db_name: stats 4 | influxdb_username: admin 5 | influxdb_password: "{{ lookup('password', 'influxdb_password chars=ascii_letters') }}" 6 | 7 | # grafana settings 8 | grafana_username: admin 9 | grafana_password: "{{ lookup('password', 'grafana_password chars=ascii_letters') }}" 10 | grafana_key: "{{ lookup('password', 'grafana_key chars=ascii_letters,digits,hexdigits') }}" 11 | grafana_version: 5.0.3 12 | grafana_arm_deb : https://github.com/fg2it/grafana-on-raspberry/releases/download/v{{ grafana_version }}/grafana_{{ grafana_version }}_armhf.deb 13 | 14 | # ispstats settings 15 | ispstats_user: stats 16 | ispstats_path: /opt/ispstats 17 | speedtest_cli_path: /opt/speedtest-cli 18 | speedtest_repo: https://github.com/sivel/speedtest-cli.git 19 | 20 | # nginx settings - change if needed 21 | nginx_processes: 1 22 | 23 | # ntp servers. change if needed 24 | ntp_servers: 25 | - time1.google.com 26 | - time2.google.com 27 | - time3.google.com 28 | - time4.google.com 29 | 30 | # how often should the speedtest run? 31 | # See https://www.freedesktop.org/software/systemd/man/systemd.time.html#Calendar%20Events for a reference 32 | run_interval: hourly 33 | -------------------------------------------------------------------------------- /ansible/hosts: -------------------------------------------------------------------------------- 1 | [servers] 2 | 10.0.0.100 -------------------------------------------------------------------------------- /ansible/roles/common/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart ntp 3 | service: name=ntp state=restarted 4 | -------------------------------------------------------------------------------- /ansible/roles/common/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: install system updates 3 | apt: upgrade=dist update_cache=yes 4 | 5 | - name: Install common packages 6 | apt: name={{ item }} state=present 7 | with_items: 8 | - vim 9 | - curl 10 | - git 11 | - python 12 | - python3 13 | - python-setuptools 14 | - apt-transport-https 15 | 16 | - name: Remove useless packages 17 | apt: name={{ item }} state=absent purge=yes autoremove=yes force=yes 18 | with_items: 19 | - exim4 20 | - exim4-base 21 | - exim4-config 22 | 23 | - name: Check if nfs-common exists 24 | stat: path=/etc/init.d/nfs-common 25 | register: nfs_status 26 | 27 | - name: stop nfs-common 28 | systemd: name=nfs-common state=stopped enabled=no 29 | when: nfs_status.stat.exists 30 | changed_when: false 31 | 32 | - name: Check if rpcbind exists 33 | stat: path=/etc/init.d/rpcbind 34 | register: rpcbind_status 35 | 36 | - name: stop rpcbind 37 | systemd: name=rpcbind state=stopped enabled=no 38 | when: rpcbind_status.stat.exists 39 | changed_when: false 40 | 41 | - name: Check if hciuart exists 42 | stat: path=/lib/systemd/system/hciuart.service 43 | register: hciuart_status 44 | 45 | - name: stop hciuart 46 | systemd: name=hciuart state=stopped enabled=no 47 | when: hciuart_status.stat.exists 48 | changed_when: false 49 | 50 | - name: install pip 51 | easy_install: name=pip state=latest 52 | changed_when: false 53 | 54 | - name: install python packages 55 | pip: name={{ item }} state=latest 56 | with_items: 57 | - requests 58 | - influxdb 59 | 60 | - name: Install ntp 61 | apt: name=ntp state=present 62 | tags: ntp 63 | 64 | - name: Configure ntp file 65 | template: src=ntp.conf.j2 dest=/etc/ntp.conf 66 | tags: ntp 67 | notify: restart ntp 68 | 69 | - name: Start the ntp service 70 | service: name=ntp state=started enabled=yes 71 | tags: ntp 72 | -------------------------------------------------------------------------------- /ansible/roles/common/templates/ntp.conf.j2: -------------------------------------------------------------------------------- 1 | # /etc/ntp.conf, configuration for ntpd; see ntp.conf(5) for help 2 | 3 | driftfile /var/lib/ntp/ntp.drift 4 | 5 | statistics loopstats peerstats clockstats 6 | filegen loopstats file loopstats type day enable 7 | filegen peerstats file peerstats type day enable 8 | filegen clockstats file clockstats type day enable 9 | 10 | {% for host in ntp_servers %} 11 | server {{ host }} iburst 12 | {% endfor %} 13 | 14 | # By default, exchange time with everybody, but don't allow configuration. 15 | restrict -4 default kod notrap nomodify nopeer noquery 16 | restrict -6 default kod notrap nomodify nopeer noquery 17 | 18 | # Local users may interrogate the ntp server more closely. 19 | restrict 127.0.0.1 20 | restrict ::1 21 | -------------------------------------------------------------------------------- /ansible/roles/grafana/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart grafana 3 | systemd: name=grafana-server state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /ansible/roles/grafana/tasks/arm.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Installing Grafana 3 | apt: 4 | deb: "{{ grafana_arm_deb }}" 5 | state: present 6 | notify: restart grafana 7 | -------------------------------------------------------------------------------- /ansible/roles/grafana/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - debug: 3 | msg: "Installing Grafana for architecture {{ ansible_architecture }}" 4 | 5 | - include_tasks: x86.yml 6 | when: ansible_architecture == "i386" or ansible_architecture == "x86_64" 7 | 8 | - include_tasks: arm.yml 9 | when: ansible_architecture == "armv7l" or ansible_architecture == "armv6l" 10 | 11 | - name: Copy Grafana configuration 12 | template: src=grafana.ini.j2 dest=/etc/grafana/grafana.ini 13 | notify: restart grafana 14 | -------------------------------------------------------------------------------- /ansible/roles/grafana/tasks/x86.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Import Grafana GPG signing key 3 | apt_key: url=https://packagecloud.io/gpg.key state=present 4 | 5 | - name: Add Grafana repository 6 | apt_repository: 7 | repo: 'deb https://packagecloud.io/grafana/stable/debian/ jessie main' 8 | state: present 9 | update_cache: yes 10 | 11 | - name: Install Grafana packages 12 | apt: name=grafana state=present 13 | notify: restart grafana 14 | -------------------------------------------------------------------------------- /ansible/roles/grafana/templates/grafana.ini.j2: -------------------------------------------------------------------------------- 1 | [server] 2 | http_addr = 127.0.0.1 3 | 4 | [analytics] 5 | reporting_enabled = false 6 | 7 | [security] 8 | admin_user = {{ grafana_username }} 9 | admin_password = {{ grafana_password }} 10 | secret_key = {{ grafana_key }} 11 | 12 | [users] 13 | allow_sign_up = false 14 | 15 | [auth.anonymous] 16 | enabled = true 17 | 18 | [metrics] 19 | enabled = false 20 | -------------------------------------------------------------------------------- /ansible/roles/influxdb/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart influxdb 3 | systemd: name=influxdb state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /ansible/roles/influxdb/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Import InfluxDB GPG signing key 3 | apt_key: url=https://repos.influxdata.com/influxdb.key state=present 4 | 5 | - name: Add InfluxDB repository 6 | apt_repository: 7 | repo: 'deb https://repos.influxdata.com/debian jessie stable' 8 | state: present 9 | update_cache: yes 10 | 11 | - name: Install InfluxDB packages 12 | apt: name=influxdb state=present 13 | notify: restart influxdb 14 | 15 | - name: Copy InfluxDB configuration 16 | template: src=influxdb.conf.j2 dest=/etc/influxdb/influxdb.conf 17 | notify: restart influxdb 18 | -------------------------------------------------------------------------------- /ansible/roles/influxdb/templates/influxdb.conf.j2: -------------------------------------------------------------------------------- 1 | reporting-disabled = true 2 | bind-address = "127.0.0.1:8088" 3 | 4 | [meta] 5 | dir = "/var/lib/influxdb/meta" 6 | 7 | [data] 8 | dir = "/var/lib/influxdb/data" 9 | wal-dir = "/var/lib/influxdb/wal" 10 | 11 | [http] 12 | # Determines whether HTTP endpoint is enabled. 13 | enabled = true 14 | # The bind address used by the HTTP service. 15 | bind-address = "127.0.0.1:8086" 16 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart ispstats 3 | systemd: 4 | state: restarted 5 | enabled: yes 6 | name: ispstats.timer 7 | daemon_reload: yes 8 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/tasks/grafana_dashboard.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Delete dashboard 3 | uri: 4 | url: http://localhost:3000/api/dashboards/db/isp-stats 5 | method: DELETE 6 | user: "{{ grafana_username }}" 7 | password: "{{ grafana_password }}" 8 | status_code: 200,404 9 | force_basic_auth: yes 10 | 11 | - name: Create dashboard 12 | uri: 13 | url: http://localhost:3000/api/dashboards/db 14 | method: POST 15 | user: "{{ grafana_username }}" 16 | password: "{{ grafana_password }}" 17 | status_code: 200 18 | body_format: json 19 | force_basic_auth: yes 20 | body: 21 | dashboard: "{{ lookup('template', 'dashboard.json.j2') }}" 22 | overwrite: true 23 | changed_when: false 24 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/tasks/grafana_datasource.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check if datasource is present 3 | uri: 4 | url: http://localhost:3000/api/datasources/name/{{ db_name }} 5 | user: "{{ grafana_username }}" 6 | password: "{{ grafana_password }}" 7 | force_basic_auth: yes 8 | status_code: 200,404 9 | register: grafana_status 10 | changed_when: false 11 | 12 | - name: Create Datasource if not present 13 | uri: 14 | url: http://localhost:3000/api/datasources 15 | method: POST 16 | user: "{{ grafana_username }}" 17 | password: "{{ grafana_password }}" 18 | force_basic_auth: yes 19 | body: 20 | name: "{{ db_name }}" 21 | type: "influxdb" 22 | url: "http://localhost:8086" 23 | access: "proxy" 24 | isDefault: true 25 | database: "{{ db_name }}" 26 | user: "{{ influxdb_username }}" 27 | password: "{{ influxdb_password }}" 28 | status_code: 200 29 | body_format: json 30 | when: grafana_status.status == 404 31 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/tasks/grafana_user.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Check password 3 | uri: 4 | url: http://localhost:3000/api/users/1 5 | user: "{{ grafana_username }}" 6 | password: "{{ grafana_password }}" 7 | force_basic_auth: yes 8 | status_code: 200 9 | changed_when: false 10 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/tasks/influxdb.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Create database 3 | influxdb_database: 4 | hostname: "127.0.0.1" 5 | database_name: "{{ db_name }}" 6 | state: present 7 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # influxdb config 3 | - include_tasks: influxdb.yml 4 | 5 | # grafana config 6 | - include_tasks: grafana_user.yml 7 | - include_tasks: grafana_datasource.yml 8 | - include_tasks: grafana_dashboard.yml 9 | 10 | - name: add user 11 | user: 12 | name: "{{ ispstats_user }}" 13 | comment: "User for ISP Stats" 14 | createhome: no 15 | shell: /usr/sbin/nologin 16 | 17 | - name: create install dir 18 | file: 19 | path: "{{ ispstats_path }}" 20 | state: directory 21 | owner: "{{ ispstats_user }}" 22 | group: "{{ ispstats_user }}" 23 | 24 | - name: copy application 25 | template: 26 | src: ../stats.py 27 | dest: "{{ ispstats_path }}/stats.py" 28 | owner: "{{ ispstats_user }}" 29 | group: "{{ ispstats_user }}" 30 | mode: 0754 31 | 32 | - name: checkout speedtest-cli 33 | git: 34 | repo: "{{ speedtest_repo }}" 35 | dest: "{{ speedtest_cli_path }}" 36 | 37 | - name: change ownership of speedtest-cli 38 | file: 39 | dest: "{{ speedtest_cli_path }}" 40 | owner: "{{ ispstats_user }}" 41 | group: "{{ ispstats_user }}" 42 | recurse: yes 43 | changed_when: false 44 | 45 | - name: Check if ispstats.timer exists 46 | stat: path=/etc/systemd/system/ispstats.timer 47 | register: ispstats_timer_status 48 | 49 | - name: Stop ispstats.timer 50 | systemd: name=ispstats.timer state=stopped enabled=no 51 | when: ispstats_timer_status.stat.exists 52 | changed_when: false 53 | 54 | - name: Check if ispstats.service exists 55 | stat: path=/etc/systemd/system/ispstats.service 56 | register: ispstats_service_status 57 | 58 | - name: Stop ispstats.service 59 | systemd: name=ispstats.service state=stopped enabled=no 60 | when: ispstats_service_status.stat.exists 61 | changed_when: false 62 | notify: restart ispstats 63 | 64 | - name: install ispstats systemd unit file 65 | template: 66 | src: ispstats.service.j2 67 | dest: /etc/systemd/system/ispstats.service 68 | mode: 0664 69 | owner: root 70 | group: root 71 | notify: restart ispstats 72 | 73 | - name: install ispstats systemd timer file 74 | template: 75 | src: ispstats.timer.j2 76 | dest: /etc/systemd/system/ispstats.timer 77 | mode: 0664 78 | owner: root 79 | group: root 80 | notify: restart ispstats 81 | 82 | - name: enable ispstats timer 83 | systemd: 84 | enabled: yes 85 | name: ispstats.timer 86 | daemon_reload: yes 87 | 88 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/templates/dashboard.json.j2: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [] 4 | }, 5 | "editable": true, 6 | "gnetId": null, 7 | "graphTooltip": 0, 8 | "hideControls": false, 9 | "id": null, 10 | "links": [], 11 | "refresh": "1h", 12 | "rows": [ 13 | { 14 | "collapse": false, 15 | "height": 250, 16 | "panels": [ 17 | { 18 | "aliasColors": {}, 19 | "bars": false, 20 | "datasource": "{{ db_name }}", 21 | "fill": 2, 22 | "id": 1, 23 | "legend": { 24 | "alignAsTable": false, 25 | "avg": true, 26 | "current": false, 27 | "hideEmpty": false, 28 | "hideZero": false, 29 | "max": false, 30 | "min": false, 31 | "show": true, 32 | "total": false, 33 | "values": true 34 | }, 35 | "lines": true, 36 | "linewidth": 1, 37 | "links": [], 38 | "nullPointMode": "null as zero", 39 | "percentage": false, 40 | "pointradius": 1, 41 | "points": true, 42 | "renderer": "flot", 43 | "seriesOverrides": [], 44 | "span": 12, 45 | "stack": false, 46 | "steppedLine": false, 47 | "targets": [ 48 | { 49 | "alias": "$col", 50 | "dsType": "influxdb", 51 | "groupBy": [ 52 | { 53 | "params": [ 54 | "auto" 55 | ], 56 | "type": "time" 57 | }, 58 | { 59 | "params": [ 60 | "none" 61 | ], 62 | "type": "fill" 63 | } 64 | ], 65 | "hide": false, 66 | "measurement": "speedtest", 67 | "policy": "default", 68 | "refId": "A", 69 | "resultFormat": "time_series", 70 | "select": [ 71 | [ 72 | { 73 | "params": [ 74 | "download_speed" 75 | ], 76 | "type": "field" 77 | }, 78 | { 79 | "params": [], 80 | "type": "mean" 81 | }, 82 | { 83 | "params": [ 84 | "Download" 85 | ], 86 | "type": "alias" 87 | } 88 | ], 89 | [ 90 | { 91 | "params": [ 92 | "upload_speed" 93 | ], 94 | "type": "field" 95 | }, 96 | { 97 | "params": [], 98 | "type": "mean" 99 | }, 100 | { 101 | "params": [ 102 | "Upload" 103 | ], 104 | "type": "alias" 105 | } 106 | ] 107 | ], 108 | "tags": [ 109 | { 110 | "key": "server_host", 111 | "operator": "=~", 112 | "value": "/^$Server$/" 113 | } 114 | ] 115 | } 116 | ], 117 | "thresholds": [], 118 | "timeFrom": null, 119 | "timeShift": null, 120 | "title": "Speed", 121 | "tooltip": { 122 | "shared": true, 123 | "sort": 0, 124 | "value_type": "individual" 125 | }, 126 | "transparent": false, 127 | "type": "graph", 128 | "xaxis": { 129 | "mode": "time", 130 | "name": null, 131 | "show": true, 132 | "values": [] 133 | }, 134 | "yaxes": [ 135 | { 136 | "format": "bps", 137 | "label": "", 138 | "logBase": 1, 139 | "max": null, 140 | "min": null, 141 | "show": true 142 | }, 143 | { 144 | "format": "none", 145 | "label": "", 146 | "logBase": 1, 147 | "max": null, 148 | "min": null, 149 | "show": true 150 | } 151 | ] 152 | } 153 | ], 154 | "repeat": null, 155 | "repeatIteration": null, 156 | "repeatRowId": null, 157 | "showTitle": false, 158 | "title": "Speed", 159 | "titleSize": "h6" 160 | }, 161 | { 162 | "collapse": false, 163 | "height": 250, 164 | "panels": [ 165 | { 166 | "columns": [], 167 | "datasource": "{{ db_name }}", 168 | "fontSize": "100%", 169 | "id": 2, 170 | "links": [], 171 | "pageSize": null, 172 | "scroll": true, 173 | "showHeader": true, 174 | "sort": { 175 | "col": 0, 176 | "desc": true 177 | }, 178 | "span": 12, 179 | "styles": [ 180 | { 181 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 182 | "pattern": "Time", 183 | "type": "date" 184 | }, 185 | { 186 | "colorMode": null, 187 | "colors": [ 188 | "rgba(245, 54, 54, 0.9)", 189 | "rgba(237, 129, 40, 0.89)", 190 | "rgba(50, 172, 45, 0.97)" 191 | ], 192 | "decimals": 2, 193 | "pattern": "/.*/", 194 | "thresholds": [], 195 | "type": "number", 196 | "unit": "bps" 197 | } 198 | ], 199 | "targets": [ 200 | { 201 | "alias": "$col", 202 | "dsType": "influxdb", 203 | "groupBy": [ 204 | { 205 | "params": [ 206 | "auto" 207 | ], 208 | "type": "time" 209 | }, 210 | { 211 | "params": [ 212 | "server_host" 213 | ], 214 | "type": "tag" 215 | }, 216 | { 217 | "params": [ 218 | "none" 219 | ], 220 | "type": "fill" 221 | } 222 | ], 223 | "hide": false, 224 | "measurement": "speedtest", 225 | "policy": "default", 226 | "refId": "A", 227 | "resultFormat": "table", 228 | "select": [ 229 | [ 230 | { 231 | "params": [ 232 | "download_speed" 233 | ], 234 | "type": "field" 235 | }, 236 | { 237 | "params": [], 238 | "type": "mean" 239 | }, 240 | { 241 | "params": [ 242 | "Download" 243 | ], 244 | "type": "alias" 245 | } 246 | ], 247 | [ 248 | { 249 | "params": [ 250 | "upload_speed" 251 | ], 252 | "type": "field" 253 | }, 254 | { 255 | "params": [], 256 | "type": "mean" 257 | }, 258 | { 259 | "params": [ 260 | "Upload" 261 | ], 262 | "type": "alias" 263 | } 264 | ] 265 | ], 266 | "tags": [] 267 | } 268 | ], 269 | "title": "Speed", 270 | "transform": "table", 271 | "type": "table" 272 | } 273 | ], 274 | "repeat": null, 275 | "repeatIteration": null, 276 | "repeatRowId": null, 277 | "showTitle": false, 278 | "title": "Speed Table", 279 | "titleSize": "h6" 280 | }, 281 | { 282 | "collapse": false, 283 | "height": 250, 284 | "panels": [ 285 | { 286 | "aliasColors": {}, 287 | "bars": false, 288 | "datasource": "{{ db_name }}", 289 | "fill": 2, 290 | "id": 3, 291 | "legend": { 292 | "avg": true, 293 | "current": false, 294 | "max": false, 295 | "min": false, 296 | "show": true, 297 | "total": false, 298 | "values": true 299 | }, 300 | "lines": true, 301 | "linewidth": 1, 302 | "links": [], 303 | "nullPointMode": "connected", 304 | "percentage": false, 305 | "pointradius": 1, 306 | "points": true, 307 | "renderer": "flot", 308 | "seriesOverrides": [], 309 | "span": 12, 310 | "stack": false, 311 | "steppedLine": false, 312 | "targets": [ 313 | { 314 | "alias": "Ping", 315 | "dsType": "influxdb", 316 | "groupBy": [ 317 | { 318 | "params": [ 319 | "auto" 320 | ], 321 | "type": "time" 322 | }, 323 | { 324 | "params": [ 325 | "null" 326 | ], 327 | "type": "fill" 328 | } 329 | ], 330 | "measurement": "speedtest", 331 | "policy": "default", 332 | "refId": "A", 333 | "resultFormat": "time_series", 334 | "select": [ 335 | [ 336 | { 337 | "params": [ 338 | "ping" 339 | ], 340 | "type": "field" 341 | }, 342 | { 343 | "params": [], 344 | "type": "mean" 345 | } 346 | ] 347 | ], 348 | "tags": [ 349 | { 350 | "key": "server_host", 351 | "operator": "=~", 352 | "value": "/^$Server$/" 353 | } 354 | ] 355 | } 356 | ], 357 | "thresholds": [], 358 | "timeFrom": null, 359 | "timeShift": null, 360 | "title": "Ping", 361 | "tooltip": { 362 | "shared": true, 363 | "sort": 0, 364 | "value_type": "individual" 365 | }, 366 | "type": "graph", 367 | "xaxis": { 368 | "mode": "time", 369 | "name": null, 370 | "show": true, 371 | "values": [] 372 | }, 373 | "yaxes": [ 374 | { 375 | "format": "ms", 376 | "label": null, 377 | "logBase": 1, 378 | "max": null, 379 | "min": null, 380 | "show": true 381 | }, 382 | { 383 | "format": "none", 384 | "label": null, 385 | "logBase": 1, 386 | "max": null, 387 | "min": null, 388 | "show": true 389 | } 390 | ] 391 | } 392 | ], 393 | "repeat": null, 394 | "repeatIteration": null, 395 | "repeatRowId": null, 396 | "showTitle": false, 397 | "title": "Ping", 398 | "titleSize": "h6" 399 | }, 400 | { 401 | "collapse": false, 402 | "height": 250, 403 | "panels": [ 404 | { 405 | "columns": [], 406 | "datasource": "{{ db_name }}", 407 | "fontSize": "100%", 408 | "id": 4, 409 | "links": [], 410 | "pageSize": null, 411 | "scroll": true, 412 | "showHeader": true, 413 | "sort": { 414 | "col": 0, 415 | "desc": true 416 | }, 417 | "span": 12, 418 | "styles": [ 419 | { 420 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 421 | "pattern": "Time", 422 | "type": "date" 423 | }, 424 | { 425 | "colorMode": null, 426 | "colors": [ 427 | "rgba(245, 54, 54, 0.9)", 428 | "rgba(237, 129, 40, 0.89)", 429 | "rgba(50, 172, 45, 0.97)" 430 | ], 431 | "decimals": 2, 432 | "pattern": "/.*/", 433 | "sanitize": true, 434 | "thresholds": [], 435 | "type": "string", 436 | "unit": "short" 437 | } 438 | ], 439 | "targets": [ 440 | { 441 | "alias": "message", 442 | "dsType": "influxdb", 443 | "groupBy": [], 444 | "measurement": "errors", 445 | "policy": "default", 446 | "refId": "A", 447 | "resultFormat": "table", 448 | "select": [ 449 | [ 450 | { 451 | "params": [ 452 | "message" 453 | ], 454 | "type": "field" 455 | } 456 | ] 457 | ], 458 | "tags": [] 459 | } 460 | ], 461 | "title": "Errors", 462 | "transform": "timeseries_to_columns", 463 | "type": "table" 464 | } 465 | ], 466 | "repeat": null, 467 | "repeatIteration": null, 468 | "repeatRowId": null, 469 | "showTitle": false, 470 | "title": "Errors", 471 | "titleSize": "h6" 472 | } 473 | ], 474 | "schemaVersion": 14, 475 | "style": "dark", 476 | "tags": [], 477 | "templating": { 478 | "list": [ 479 | { 480 | "allValue": null, 481 | "current": { 482 | "text": "All", 483 | "value": [ 484 | "$__all" 485 | ] 486 | }, 487 | "datasource": "{{ db_name }}", 488 | "hide": 0, 489 | "includeAll": true, 490 | "label": null, 491 | "multi": true, 492 | "name": "Server", 493 | "options": [], 494 | "query": "show tag values from speedtest with key = \"server_host\"", 495 | "refresh": 1, 496 | "regex": "", 497 | "sort": 0, 498 | "tagValuesQuery": "", 499 | "tags": [], 500 | "tagsQuery": "", 501 | "type": "query", 502 | "useTags": false 503 | } 504 | ] 505 | }, 506 | "time": { 507 | "from": "now/w", 508 | "to": "now" 509 | }, 510 | "timepicker": { 511 | "refresh_intervals": [ 512 | "5s", 513 | "10s", 514 | "30s", 515 | "1m", 516 | "5m", 517 | "15m", 518 | "30m", 519 | "1h", 520 | "2h", 521 | "1d" 522 | ], 523 | "time_options": [ 524 | "5m", 525 | "15m", 526 | "1h", 527 | "6h", 528 | "12h", 529 | "24h", 530 | "2d", 531 | "7d", 532 | "30d" 533 | ] 534 | }, 535 | "timezone": "browser", 536 | "title": "ISP Stats", 537 | "version": 0 538 | } 539 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/templates/ispstats.service.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=isp-stats Service 3 | 4 | [Service] 5 | User={{ ispstats_user }} 6 | Group={{ ispstats_user }} 7 | Nice=-20 8 | SyslogIdentifier=isp-stats 9 | ExecStart={{ ispstats_path }}/stats.py 10 | -------------------------------------------------------------------------------- /ansible/roles/ispstats/templates/ispstats.timer.j2: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=isp-stats Timer 3 | 4 | [Timer] 5 | OnCalendar={{ run_interval }} 6 | Persistent=true 7 | 8 | [Install] 9 | WantedBy=basic.target 10 | -------------------------------------------------------------------------------- /ansible/roles/nginx/handlers/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: restart nginx 3 | systemd: name=nginx state=restarted enabled=yes 4 | -------------------------------------------------------------------------------- /ansible/roles/nginx/tasks/main.yml: -------------------------------------------------------------------------------- 1 | --- 2 | - name: Install nginx 3 | apt: name=nginx state=present 4 | 5 | - name: reduce worker processes 6 | replace: 7 | name: /etc/nginx/nginx.conf 8 | regexp: '^worker_processes \d+;$' 9 | replace: "worker_processes {{nginx_processes}};" 10 | backup: yes 11 | notify: restart nginx 12 | 13 | - name: Copy nginx configuration 14 | template: src=default.conf.j2 dest=/etc/nginx/sites-enabled/default 15 | notify: restart nginx 16 | -------------------------------------------------------------------------------- /ansible/roles/nginx/templates/default.conf.j2: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | root /usr/share/nginx/www; 4 | index index.html index.htm; 5 | 6 | location / { 7 | proxy_pass http://localhost:3000/; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /ansible/site.yml: -------------------------------------------------------------------------------- 1 | --- 2 | # This playbook deploys the whole application stack in this site. 3 | 4 | - name: apply common configuration to all nodes 5 | hosts: all 6 | remote_user: root 7 | roles: 8 | - common 9 | 10 | - name: Install and setup InfluxDB 11 | hosts: all 12 | remote_user: root 13 | roles: 14 | - influxdb 15 | 16 | - name: Install and setup Grafana 17 | hosts: all 18 | remote_user: root 19 | roles: 20 | - grafana 21 | 22 | - name: Install and setup NGINX 23 | hosts: all 24 | remote_user: root 25 | roles: 26 | - nginx 27 | 28 | - name: Setup ispstats 29 | hosts: all 30 | remote_user: root 31 | roles: 32 | - ispstats 33 | -------------------------------------------------------------------------------- /images/grafana.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/firefart/isp_stats/0252e416e02b951789d65d8f8ce8cc106212ab00/images/grafana.png -------------------------------------------------------------------------------- /stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import json 4 | from influxdb import InfluxDBClient 5 | import subprocess 6 | 7 | DATABASE = "{{ db_name }}" 8 | DB_USER = "{{ influxdb_username }}" 9 | DB_PW = "{{ influxdb_password }}" 10 | SPEEDTEST = "{{ speedtest_cli_path }}/speedtest.py" 11 | 12 | print("Starting Speedtest...") 13 | p = subprocess.Popen([SPEEDTEST, "--json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 14 | stdout, stderr = p.communicate() 15 | 16 | errors = [] 17 | j = None 18 | if stderr is not None and stderr != "": 19 | print("Got error {}".format(stderr)) 20 | errors.append(stderr) 21 | 22 | try: 23 | j = json.loads(stdout) 24 | except ValueError as e: 25 | print("Got error {}".format(str(e))) 26 | errors.append(str(e)) 27 | 28 | data = [] 29 | 30 | # sample data 31 | # server": {"latency": 17.473, "name": "Vienna", "url": "http://speedtest.nessus.at/speedtest/upload.php", "country": "Austria", "lon": "16.3726", "cc": "AT", "host": "speedtest.nessus.at:8080", "sponsor": "Nessus GmbH (10G Uplink)", "url2": "http://speedtest2.nessus.at/speedtest/upload.php", "lat": "48.2088", "id": "3744", "d": 1.3292411343362884}} 32 | 33 | if j: 34 | data.append({ 35 | "measurement": "speedtest", 36 | "tags": { 37 | "server_id": j["server"]["id"], 38 | "server_name": j["server"]["name"], 39 | "server_host": j["server"]["host"], 40 | "server_sponsor": j["server"]["sponsor"], 41 | "server_latency": j["server"]["latency"], 42 | "server_url": j["server"]["url"], 43 | "server_country": j["server"]["country"], 44 | "server_cc": j["server"]["cc"], 45 | "server_lon": j["server"]["lon"], 46 | "server_lat": j["server"]["lat"], 47 | "server_d": j["server"]["d"] 48 | }, 49 | "fields": { 50 | "download_speed": j["download"], 51 | "upload_speed": j["upload"], 52 | "ping": j["ping"] 53 | } 54 | }) 55 | 56 | for e in errors: 57 | data.append({ 58 | "measurement": "errors", 59 | "fields": { 60 | "message": str(e) 61 | } 62 | }) 63 | 64 | print("Writing to DB:\n{}".format(data)) 65 | 66 | influx = InfluxDBClient("127.0.0.1", 8086, DB_USER, DB_PW, DATABASE) 67 | influx.write_points(data) 68 | --------------------------------------------------------------------------------