├── .gitignore ├── LICENSE ├── README.md ├── docker └── dockerfile ├── examples └── notebooks │ ├── FileLoggerExample.ipynb │ └── InfluxDBLoggerExample.ipynb ├── gpumon ├── __init__.py ├── __version__.py ├── file │ ├── __init__.py │ └── gpu_logger.py ├── gpumon └── influxdb │ ├── __init__.py │ ├── dotenv.py │ ├── gpu_interface.py │ └── gpu_logger.py ├── requirements.txt ├── scripts ├── Makefile ├── docker-compose.yml ├── example.env └── provisioning │ ├── dashboards │ ├── GPUDashboard.json │ └── gpudash.yaml │ └── datasources │ └── influxdb-source.yaml ├── setup.py └── static ├── gpu_dashboard.gif └── influxdb_config.png /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | 28 | # PyInstaller 29 | # Usually these files are written by a python script from a template 30 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 31 | *.manifest 32 | *.spec 33 | 34 | # Installer logs 35 | pip-log.txt 36 | pip-delete-this-directory.txt 37 | 38 | # Unit test / coverage reports 39 | htmlcov/ 40 | .tox/ 41 | .coverage 42 | .coverage.* 43 | .cache 44 | nosetests.xml 45 | coverage.xml 46 | *.cover 47 | .hypothesis/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # intellij 104 | .idea 105 | *.iml 106 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Mat 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GPU Monitor 2 | This is an app for monitoring GPUs on a single machine and across a cluster. You can use it to record various GPU measurements during a specific period using the context based loggers or continuously using the gpumon cli command. The context logger can either record to a file, which can be read back into a dataframe, or to an InfluxDB database. Data from the InfluxDB database can then be accessed using the python InfluxDB client or can be viewed in realtime using dashboards such as Grafana. Examples in Juypyter notebooks can be found [here](examples/notebooks) 3 | 4 | When logging to influxdb the logger uses the Python bindings for the NVIDIA Management Library (NVML) which is a C-based API used for monitoring NVIDIA GPU devices. The performance of NVML is better and more efficient when compared to using nvidia-smi leading to a higher sampling frequency of the measurements. 5 | 6 | Below is an example dashboard using the InfluxDB log context and a Grafana dashboard 7 | 8 |

9 | Grafana GPU Dashboard 10 |

11 | 12 | 13 | ## Installation 14 | 15 | To install simply either clone the repository 16 | 17 | ```bash 18 | git clone https://github.com/msalvaris/gpu_monitor.git 19 | ``` 20 | 21 | Then install it: 22 | ```bash 23 | pip install -e /path/to/repo 24 | ``` 25 | For now I recommend the -e flag since it is in active development and 26 | will be easy to update by pulling the latest changes from the repo. 27 | 28 | 29 | Or just install using pip 30 | 31 | ```bash 32 | pip install git+https://github.com/msalvaris/gpu_monitor.git 33 | ``` 34 | 35 | ## Docker 36 | You can also run it straight from a docker image (masalvar_gpumon). 37 | 38 | ```bash 39 | nvidia-docker run -it masalvar/gpumon gpumon msdlvm.southcentralus.cloudapp.azure.com admin password gpudb 8086 gpuseries 40 | ``` 41 | 42 | ## Usage 43 | ### Running gpu monitor in Jupyter notebook with file based log context 44 | ```python 45 | from gpumon.file import log_context 46 | from bokeh.io import output_notebook, show 47 | 48 | output_notebook()# Without this the plot won't show in Jupyter notebook 49 | 50 | with log_context('log.txt') as log: 51 | # GPU code 52 | 53 | show(log.plot())# Will plot the utilisation during the context 54 | 55 | log()# Will return dataframe with all the logged properties 56 | ``` 57 | [Click here to see the example notebook](examples/notebooks/InfluxDBLoggerExample.ipynb) 58 | 59 | ### Running gpu monitor in Jupyter notebook with InfluxDB based log context 60 | To do this you need to set up and [install InfluxDB](https://docs.influxdata.com/influxdb/v1.5/introduction/installation/) and [Grafana](http://docs.grafana.org/installation/). 61 | There are many ways to install and run InfluxDB and Grafana in this example we will be using Docker containers and docker-compose. 62 | 63 | If you haven't got docker-compose installed see [here for instructions](https://docs.docker.com/compose/install/) 64 | 65 | You must be also be able to execute the docker commands without the requirement of sudo. To do this in Ubuntu execute the following: 66 | ```bash 67 | sudo groupadd docker 68 | sudo usermod -aG docker $USER 69 | ``` 70 | 71 | If you haven't downloaded the whole repo then download the [scripts directory](scripts). In there should be three files 72 | The file *example.env* contains the following variables: 73 | INFLUXDB_DB=gpudb 74 | INFLUXDB_USER=admin 75 | INFLUXDB_USER_PASSWORD=password 76 | INFLUXDB_ADMIN_ENABLED=true 77 | GF_SECURITY_ADMIN_PASSWORD=password 78 | GRAFANA_DATA_LOCATION=/tmp/grafana 79 | INFLUXDB_DATA_LOCATION=/tmp/influxdb 80 | GF_PATHS_PROVISIONING=/grafana-conf 81 | 82 | Please change them to appropriate values. The data location entries (GRAFANA_DATA_LOCATION, INFLUXDB_DATA_LOCATION) will tell Grafana and InfluxDB where to store their data so that when the containers are destroyed the data remains. 83 | Once you have edited it rename *example.env* to *.env*. 84 | 85 | Now inside the folder that contains the file you can run the command below and it will give you the various commands you can execute. 86 | ```bash 87 | make 88 | ``` 89 | 90 | To start InfluxDB and Grafana you run 91 | ```bash 92 | make run 93 | ``` 94 | 95 | Now in your Jupyter notebook simply add these lines 96 | ```python 97 | from gpumon.influxdb import log_context 98 | 99 | with log_context('localhost', 'admin', 'password', 'gpudb', 'gpuseries'): 100 | # GPU Code 101 | 102 | ``` 103 | Make sure you replace the values in the call to the log_context with the appropriate values. 104 | *gpudata* is the name of the database and *gpuseries* is the name we have given to our series, feel free to change these. 105 | If the database name given in the context isn't the same as the one supplied in the .env file a new database will be created. 106 | Have a look at [this notebook](examples/notebooks/InfluxDBLoggerExample.ipynb) for a full example. 107 | 108 | 109 | If you want to use the CLI version run the following command: 110 | ```bash 111 | gpumon localhost admin password gpudb --series_name=gpuseries 112 | ``` 113 | 114 | The above command will connect to the influxdb database running on localhost with 115 | user=admin 116 | password=password 117 | database=gpudb 118 | series_name=gpuseries 119 | 120 | You can also put your parameters in a .env file in the same directory as you are executing the cli logger or the logging context and the necessary information will be pulled from it. You can also pass commands to them and these will override what is in your .env file 121 | 122 | 123 | #### Setting up Grafana dashboard 124 | By default a datasource and dashboard are set up for you if everything went well above GPU metrics should be flowing to your database. 125 | You can log in to Grafana by pointing a browser to the IP of your VM or computer on port 3000. *If you are executing on a VM make sure that port is open*. 126 | Once there log in with the credentials you specified in your .env file. 127 | 128 | #### Manually setting up datasource and dashboard 129 | Below is an example screen-shot of the datasource config 130 | 131 |

132 | Datasource config 133 |

134 | 135 | Once that is set up you will need to also set up your dashboard. The dashboard shown in the gif above can be found [here](scripts/provisioning/dashboards/GPUDashboard.json) and is installed by default. -------------------------------------------------------------------------------- /docker/dockerfile: -------------------------------------------------------------------------------- 1 | FROM nvidia/cuda:9.0-base 2 | 3 | ENV CONDA_DIR /opt/conda 4 | ENV PATH $CONDA_DIR/bin:$PATH 5 | 6 | RUN mkdir -p $CONDA_DIR && \ 7 | echo export PATH=$CONDA_DIR/bin:'$PATH' > /etc/profile.d/conda.sh && \ 8 | apt-get update && \ 9 | apt-get install -y wget git g++ bzip2 && \ 10 | wget --quiet https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh && \ 11 | echo "c59b3dd3cad550ac7596e0d599b91e75d88826db132e4146030ef471bb434e9a *Miniconda3-4.2.12-Linux-x86_64.sh" | sha256sum -c - && \ 12 | /bin/bash /Miniconda3-4.2.12-Linux-x86_64.sh -f -b -p $CONDA_DIR && \ 13 | rm Miniconda3-4.2.12-Linux-x86_64.sh 14 | 15 | # Python 16 | ARG python_version=3.6 17 | 18 | ENV PATH /opt/conda/envs/py$PYTHON_VERSION/bin:$PATH 19 | 20 | RUN mkdir -p /src 21 | 22 | WORKDIR /src 23 | 24 | RUN conda install -y python=${python_version} && \ 25 | pip install --upgrade pip && \ 26 | git clone https://github.com/msalvaris/gpu_monitor.git && \ 27 | pip install -r gpu_monitor/requirements.txt && \ 28 | pip install --no-deps -e gpu_monitor && \ 29 | conda clean -yt 30 | 31 | -------------------------------------------------------------------------------- /examples/notebooks/FileLoggerExample.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# File Logger Example\n", 8 | "\n", 9 | "This notebook is a small demo of how to use gpumon in Jupyter notebooks and some convenience methods for working with GPUs" 10 | ] 11 | }, 12 | { 13 | "cell_type": "code", 14 | "execution_count": 1, 15 | "metadata": {}, 16 | "outputs": [], 17 | "source": [ 18 | "from gpumon.file import log_context" 19 | ] 20 | }, 21 | { 22 | "cell_type": "markdown", 23 | "metadata": {}, 24 | "source": [ 25 | "device_name and device_count will return the name of the GPU found on the system and the number of GPUs" 26 | ] 27 | }, 28 | { 29 | "cell_type": "code", 30 | "execution_count": 2, 31 | "metadata": {}, 32 | "outputs": [], 33 | "source": [ 34 | "from gpumon import device_name, device_count" 35 | ] 36 | }, 37 | { 38 | "cell_type": "code", 39 | "execution_count": 22, 40 | "metadata": {}, 41 | "outputs": [ 42 | { 43 | "data": { 44 | "text/plain": [ 45 | "4" 46 | ] 47 | }, 48 | "execution_count": 22, 49 | "metadata": {}, 50 | "output_type": "execute_result" 51 | } 52 | ], 53 | "source": [ 54 | "device_count() # Returns the number of GPUs available" 55 | ] 56 | }, 57 | { 58 | "cell_type": "code", 59 | "execution_count": 5, 60 | "metadata": {}, 61 | "outputs": [ 62 | { 63 | "data": { 64 | "text/plain": [ 65 | "'Tesla P40'" 66 | ] 67 | }, 68 | "execution_count": 5, 69 | "metadata": {}, 70 | "output_type": "execute_result" 71 | } 72 | ], 73 | "source": [ 74 | "device_name() # Returns the type of GPU available" 75 | ] 76 | }, 77 | { 78 | "cell_type": "code", 79 | "execution_count": 6, 80 | "metadata": {}, 81 | "outputs": [], 82 | "source": [ 83 | "from bokeh.io import output_notebook, show\n", 84 | "import time" 85 | ] 86 | }, 87 | { 88 | "cell_type": "code", 89 | "execution_count": 7, 90 | "metadata": {}, 91 | "outputs": [ 92 | { 93 | "data": { 94 | "text/html": [ 95 | "\n", 96 | "
\n", 97 | " \n", 98 | " Loading BokehJS ...\n", 99 | "
" 100 | ] 101 | }, 102 | "metadata": {}, 103 | "output_type": "display_data" 104 | }, 105 | { 106 | "data": { 107 | "application/javascript": [ 108 | "\n", 109 | "(function(root) {\n", 110 | " function now() {\n", 111 | " return new Date();\n", 112 | " }\n", 113 | "\n", 114 | " var force = true;\n", 115 | "\n", 116 | " if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n", 117 | " root._bokeh_onload_callbacks = [];\n", 118 | " root._bokeh_is_loading = undefined;\n", 119 | " }\n", 120 | "\n", 121 | " var JS_MIME_TYPE = 'application/javascript';\n", 122 | " var HTML_MIME_TYPE = 'text/html';\n", 123 | " var EXEC_MIME_TYPE = 'application/vnd.bokehjs_exec.v0+json';\n", 124 | " var CLASS_NAME = 'output_bokeh rendered_html';\n", 125 | "\n", 126 | " /**\n", 127 | " * Render data to the DOM node\n", 128 | " */\n", 129 | " function render(props, node) {\n", 130 | " var script = document.createElement(\"script\");\n", 131 | " node.appendChild(script);\n", 132 | " }\n", 133 | "\n", 134 | " /**\n", 135 | " * Handle when an output is cleared or removed\n", 136 | " */\n", 137 | " function handleClearOutput(event, handle) {\n", 138 | " var cell = handle.cell;\n", 139 | "\n", 140 | " var id = cell.output_area._bokeh_element_id;\n", 141 | " var server_id = cell.output_area._bokeh_server_id;\n", 142 | " // Clean up Bokeh references\n", 143 | " if (id !== undefined) {\n", 144 | " Bokeh.index[id].model.document.clear();\n", 145 | " delete Bokeh.index[id];\n", 146 | " }\n", 147 | "\n", 148 | " if (server_id !== undefined) {\n", 149 | " // Clean up Bokeh references\n", 150 | " var cmd = \"from bokeh.io.state import curstate; print(curstate().uuid_to_server['\" + server_id + \"'].get_sessions()[0].document.roots[0]._id)\";\n", 151 | " cell.notebook.kernel.execute(cmd, {\n", 152 | " iopub: {\n", 153 | " output: function(msg) {\n", 154 | " var element_id = msg.content.text.trim();\n", 155 | " Bokeh.index[element_id].model.document.clear();\n", 156 | " delete Bokeh.index[element_id];\n", 157 | " }\n", 158 | " }\n", 159 | " });\n", 160 | " // Destroy server and session\n", 161 | " var cmd = \"import bokeh.io.notebook as ion; ion.destroy_server('\" + server_id + \"')\";\n", 162 | " cell.notebook.kernel.execute(cmd);\n", 163 | " }\n", 164 | " }\n", 165 | "\n", 166 | " /**\n", 167 | " * Handle when a new output is added\n", 168 | " */\n", 169 | " function handleAddOutput(event, handle) {\n", 170 | " var output_area = handle.output_area;\n", 171 | " var output = handle.output;\n", 172 | "\n", 173 | " // limit handleAddOutput to display_data with EXEC_MIME_TYPE content only\n", 174 | " if ((output.output_type != \"display_data\") || (!output.data.hasOwnProperty(EXEC_MIME_TYPE))) {\n", 175 | " return\n", 176 | " }\n", 177 | "\n", 178 | " var toinsert = output_area.element.find(\".\" + CLASS_NAME.split(' ')[0]);\n", 179 | "\n", 180 | " if (output.metadata[EXEC_MIME_TYPE][\"id\"] !== undefined) {\n", 181 | " toinsert[0].firstChild.textContent = output.data[JS_MIME_TYPE];\n", 182 | " // store reference to embed id on output_area\n", 183 | " output_area._bokeh_element_id = output.metadata[EXEC_MIME_TYPE][\"id\"];\n", 184 | " }\n", 185 | " if (output.metadata[EXEC_MIME_TYPE][\"server_id\"] !== undefined) {\n", 186 | " var bk_div = document.createElement(\"div\");\n", 187 | " bk_div.innerHTML = output.data[HTML_MIME_TYPE];\n", 188 | " var script_attrs = bk_div.children[0].attributes;\n", 189 | " for (var i = 0; i < script_attrs.length; i++) {\n", 190 | " toinsert[0].firstChild.setAttribute(script_attrs[i].name, script_attrs[i].value);\n", 191 | " }\n", 192 | " // store reference to server id on output_area\n", 193 | " output_area._bokeh_server_id = output.metadata[EXEC_MIME_TYPE][\"server_id\"];\n", 194 | " }\n", 195 | " }\n", 196 | "\n", 197 | " function register_renderer(events, OutputArea) {\n", 198 | "\n", 199 | " function append_mime(data, metadata, element) {\n", 200 | " // create a DOM node to render to\n", 201 | " var toinsert = this.create_output_subarea(\n", 202 | " metadata,\n", 203 | " CLASS_NAME,\n", 204 | " EXEC_MIME_TYPE\n", 205 | " );\n", 206 | " this.keyboard_manager.register_events(toinsert);\n", 207 | " // Render to node\n", 208 | " var props = {data: data, metadata: metadata[EXEC_MIME_TYPE]};\n", 209 | " render(props, toinsert[0]);\n", 210 | " element.append(toinsert);\n", 211 | " return toinsert\n", 212 | " }\n", 213 | "\n", 214 | " /* Handle when an output is cleared or removed */\n", 215 | " events.on('clear_output.CodeCell', handleClearOutput);\n", 216 | " events.on('delete.Cell', handleClearOutput);\n", 217 | "\n", 218 | " /* Handle when a new output is added */\n", 219 | " events.on('output_added.OutputArea', handleAddOutput);\n", 220 | "\n", 221 | " /**\n", 222 | " * Register the mime type and append_mime function with output_area\n", 223 | " */\n", 224 | " OutputArea.prototype.register_mime_type(EXEC_MIME_TYPE, append_mime, {\n", 225 | " /* Is output safe? */\n", 226 | " safe: true,\n", 227 | " /* Index of renderer in `output_area.display_order` */\n", 228 | " index: 0\n", 229 | " });\n", 230 | " }\n", 231 | "\n", 232 | " // register the mime type if in Jupyter Notebook environment and previously unregistered\n", 233 | " if (root.Jupyter !== undefined) {\n", 234 | " var events = require('base/js/events');\n", 235 | " var OutputArea = require('notebook/js/outputarea').OutputArea;\n", 236 | "\n", 237 | " if (OutputArea.prototype.mime_types().indexOf(EXEC_MIME_TYPE) == -1) {\n", 238 | " register_renderer(events, OutputArea);\n", 239 | " }\n", 240 | " }\n", 241 | "\n", 242 | " \n", 243 | " if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n", 244 | " root._bokeh_timeout = Date.now() + 5000;\n", 245 | " root._bokeh_failed_load = false;\n", 246 | " }\n", 247 | "\n", 248 | " var NB_LOAD_WARNING = {'data': {'text/html':\n", 249 | " \"
\\n\"+\n", 250 | " \"

\\n\"+\n", 251 | " \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n", 252 | " \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n", 253 | " \"

\\n\"+\n", 254 | " \"\\n\"+\n", 258 | " \"\\n\"+\n", 259 | " \"from bokeh.resources import INLINE\\n\"+\n", 260 | " \"output_notebook(resources=INLINE)\\n\"+\n", 261 | " \"\\n\"+\n", 262 | " \"
\"}};\n", 263 | "\n", 264 | " function display_loaded() {\n", 265 | " var el = document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\");\n", 266 | " if (el != null) {\n", 267 | " el.textContent = \"BokehJS is loading...\";\n", 268 | " }\n", 269 | " if (root.Bokeh !== undefined) {\n", 270 | " if (el != null) {\n", 271 | " el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n", 272 | " }\n", 273 | " } else if (Date.now() < root._bokeh_timeout) {\n", 274 | " setTimeout(display_loaded, 100)\n", 275 | " }\n", 276 | " }\n", 277 | "\n", 278 | "\n", 279 | " function run_callbacks() {\n", 280 | " try {\n", 281 | " root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n", 282 | " }\n", 283 | " finally {\n", 284 | " delete root._bokeh_onload_callbacks\n", 285 | " }\n", 286 | " console.info(\"Bokeh: all callbacks have finished\");\n", 287 | " }\n", 288 | "\n", 289 | " function load_libs(js_urls, callback) {\n", 290 | " root._bokeh_onload_callbacks.push(callback);\n", 291 | " if (root._bokeh_is_loading > 0) {\n", 292 | " console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n", 293 | " return null;\n", 294 | " }\n", 295 | " if (js_urls == null || js_urls.length === 0) {\n", 296 | " run_callbacks();\n", 297 | " return null;\n", 298 | " }\n", 299 | " console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n", 300 | " root._bokeh_is_loading = js_urls.length;\n", 301 | " for (var i = 0; i < js_urls.length; i++) {\n", 302 | " var url = js_urls[i];\n", 303 | " var s = document.createElement('script');\n", 304 | " s.src = url;\n", 305 | " s.async = false;\n", 306 | " s.onreadystatechange = s.onload = function() {\n", 307 | " root._bokeh_is_loading--;\n", 308 | " if (root._bokeh_is_loading === 0) {\n", 309 | " console.log(\"Bokeh: all BokehJS libraries loaded\");\n", 310 | " run_callbacks()\n", 311 | " }\n", 312 | " };\n", 313 | " s.onerror = function() {\n", 314 | " console.warn(\"failed to load library \" + url);\n", 315 | " };\n", 316 | " console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n", 317 | " document.getElementsByTagName(\"head\")[0].appendChild(s);\n", 318 | " }\n", 319 | " };var element = document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\");\n", 320 | " if (element == null) {\n", 321 | " console.log(\"Bokeh: ERROR: autoload.js configured with elementid '7aa1426e-d8d2-4dbd-bdd7-c030fbb17518' but no matching script tag was found. \")\n", 322 | " return false;\n", 323 | " }\n", 324 | "\n", 325 | " var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n", 326 | "\n", 327 | " var inline_js = [\n", 328 | " function(Bokeh) {\n", 329 | " Bokeh.set_log_level(\"info\");\n", 330 | " },\n", 331 | " \n", 332 | " function(Bokeh) {\n", 333 | " \n", 334 | " },\n", 335 | " function(Bokeh) {\n", 336 | " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n", 337 | " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n", 338 | " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n", 339 | " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n", 340 | " console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n", 341 | " Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n", 342 | " }\n", 343 | " ];\n", 344 | "\n", 345 | " function run_inline_js() {\n", 346 | " \n", 347 | " if ((root.Bokeh !== undefined) || (force === true)) {\n", 348 | " for (var i = 0; i < inline_js.length; i++) {\n", 349 | " inline_js[i].call(root, root.Bokeh);\n", 350 | " }if (force === true) {\n", 351 | " display_loaded();\n", 352 | " }} else if (Date.now() < root._bokeh_timeout) {\n", 353 | " setTimeout(run_inline_js, 100);\n", 354 | " } else if (!root._bokeh_failed_load) {\n", 355 | " console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n", 356 | " root._bokeh_failed_load = true;\n", 357 | " } else if (force !== true) {\n", 358 | " var cell = $(document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\")).parents('.cell').data().cell;\n", 359 | " cell.output_area.append_execute_result(NB_LOAD_WARNING)\n", 360 | " }\n", 361 | "\n", 362 | " }\n", 363 | "\n", 364 | " if (root._bokeh_is_loading === 0) {\n", 365 | " console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n", 366 | " run_inline_js();\n", 367 | " } else {\n", 368 | " load_libs(js_urls, function() {\n", 369 | " console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n", 370 | " run_inline_js();\n", 371 | " });\n", 372 | " }\n", 373 | "}(window));" 374 | ], 375 | "application/vnd.bokehjs_load.v0+json": "\n(function(root) {\n function now() {\n return new Date();\n }\n\n var force = true;\n\n if (typeof (root._bokeh_onload_callbacks) === \"undefined\" || force === true) {\n root._bokeh_onload_callbacks = [];\n root._bokeh_is_loading = undefined;\n }\n\n \n\n \n if (typeof (root._bokeh_timeout) === \"undefined\" || force === true) {\n root._bokeh_timeout = Date.now() + 5000;\n root._bokeh_failed_load = false;\n }\n\n var NB_LOAD_WARNING = {'data': {'text/html':\n \"
\\n\"+\n \"

\\n\"+\n \"BokehJS does not appear to have successfully loaded. If loading BokehJS from CDN, this \\n\"+\n \"may be due to a slow or bad network connection. Possible fixes:\\n\"+\n \"

\\n\"+\n \"\\n\"+\n \"\\n\"+\n \"from bokeh.resources import INLINE\\n\"+\n \"output_notebook(resources=INLINE)\\n\"+\n \"\\n\"+\n \"
\"}};\n\n function display_loaded() {\n var el = document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\");\n if (el != null) {\n el.textContent = \"BokehJS is loading...\";\n }\n if (root.Bokeh !== undefined) {\n if (el != null) {\n el.textContent = \"BokehJS \" + root.Bokeh.version + \" successfully loaded.\";\n }\n } else if (Date.now() < root._bokeh_timeout) {\n setTimeout(display_loaded, 100)\n }\n }\n\n\n function run_callbacks() {\n try {\n root._bokeh_onload_callbacks.forEach(function(callback) { callback() });\n }\n finally {\n delete root._bokeh_onload_callbacks\n }\n console.info(\"Bokeh: all callbacks have finished\");\n }\n\n function load_libs(js_urls, callback) {\n root._bokeh_onload_callbacks.push(callback);\n if (root._bokeh_is_loading > 0) {\n console.log(\"Bokeh: BokehJS is being loaded, scheduling callback at\", now());\n return null;\n }\n if (js_urls == null || js_urls.length === 0) {\n run_callbacks();\n return null;\n }\n console.log(\"Bokeh: BokehJS not loaded, scheduling load and callback at\", now());\n root._bokeh_is_loading = js_urls.length;\n for (var i = 0; i < js_urls.length; i++) {\n var url = js_urls[i];\n var s = document.createElement('script');\n s.src = url;\n s.async = false;\n s.onreadystatechange = s.onload = function() {\n root._bokeh_is_loading--;\n if (root._bokeh_is_loading === 0) {\n console.log(\"Bokeh: all BokehJS libraries loaded\");\n run_callbacks()\n }\n };\n s.onerror = function() {\n console.warn(\"failed to load library \" + url);\n };\n console.log(\"Bokeh: injecting script tag for BokehJS library: \", url);\n document.getElementsByTagName(\"head\")[0].appendChild(s);\n }\n };var element = document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\");\n if (element == null) {\n console.log(\"Bokeh: ERROR: autoload.js configured with elementid '7aa1426e-d8d2-4dbd-bdd7-c030fbb17518' but no matching script tag was found. \")\n return false;\n }\n\n var js_urls = [\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.js\", \"https://cdn.pydata.org/bokeh/release/bokeh-gl-0.12.13.min.js\"];\n\n var inline_js = [\n function(Bokeh) {\n Bokeh.set_log_level(\"info\");\n },\n \n function(Bokeh) {\n \n },\n function(Bokeh) {\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-0.12.13.min.css\");\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-widgets-0.12.13.min.css\");\n console.log(\"Bokeh: injecting CSS: https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n Bokeh.embed.inject_css(\"https://cdn.pydata.org/bokeh/release/bokeh-tables-0.12.13.min.css\");\n }\n ];\n\n function run_inline_js() {\n \n if ((root.Bokeh !== undefined) || (force === true)) {\n for (var i = 0; i < inline_js.length; i++) {\n inline_js[i].call(root, root.Bokeh);\n }if (force === true) {\n display_loaded();\n }} else if (Date.now() < root._bokeh_timeout) {\n setTimeout(run_inline_js, 100);\n } else if (!root._bokeh_failed_load) {\n console.log(\"Bokeh: BokehJS failed to load within specified timeout.\");\n root._bokeh_failed_load = true;\n } else if (force !== true) {\n var cell = $(document.getElementById(\"7aa1426e-d8d2-4dbd-bdd7-c030fbb17518\")).parents('.cell').data().cell;\n cell.output_area.append_execute_result(NB_LOAD_WARNING)\n }\n\n }\n\n if (root._bokeh_is_loading === 0) {\n console.log(\"Bokeh: BokehJS loaded, going straight to plotting\");\n run_inline_js();\n } else {\n load_libs(js_urls, function() {\n console.log(\"Bokeh: BokehJS plotting callback run at\", now());\n run_inline_js();\n });\n }\n}(window));" 376 | }, 377 | "metadata": {}, 378 | "output_type": "display_data" 379 | } 380 | ], 381 | "source": [ 382 | "output_notebook()" 383 | ] 384 | }, 385 | { 386 | "cell_type": "markdown", 387 | "metadata": {}, 388 | "source": [ 389 | "Here we are simply going to tell the log context to record GPU measurements to the file test_gpu.txt" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": 8, 395 | "metadata": {}, 396 | "outputs": [], 397 | "source": [ 398 | "with log_context('test_gpu.txt') as log:\n", 399 | " time.sleep(10)" 400 | ] 401 | }, 402 | { 403 | "cell_type": "markdown", 404 | "metadata": {}, 405 | "source": [ 406 | "We can then cat the file to see what we recorded in the 10 seconds" 407 | ] 408 | }, 409 | { 410 | "cell_type": "code", 411 | "execution_count": 9, 412 | "metadata": {}, 413 | "outputs": [ 414 | { 415 | "name": "stdout", 416 | "output_type": "stream", 417 | "text": [ 418 | "#Date Time gpu pwr temp sm mem enc dec mclk pclk\r\n", 419 | "#YYYYMMDD HH:MM:SS Idx W C % % % % MHz MHz\r\n", 420 | " 20180330 16:21:28 0 10 21 0 0 0 0 405 544\r\n", 421 | " 20180330 16:21:28 1 9 26 0 0 0 0 405 544\r\n", 422 | " 20180330 16:21:28 2 9 21 0 0 0 0 405 544\r\n", 423 | " 20180330 16:21:28 3 9 21 0 0 0 0 405 544\r\n", 424 | " 20180330 16:21:29 0 9 21 0 0 0 0 405 544\r\n", 425 | " 20180330 16:21:29 1 9 26 0 0 0 0 405 544\r\n", 426 | " 20180330 16:21:29 2 9 21 0 0 0 0 405 544\r\n", 427 | " 20180330 16:21:29 3 9 21 0 0 0 0 405 544\r\n", 428 | " 20180330 16:21:31 0 9 21 0 0 0 0 405 544\r\n", 429 | " 20180330 16:21:31 1 9 26 0 0 0 0 405 544\r\n", 430 | " 20180330 16:21:31 2 9 21 0 0 0 0 405 544\r\n", 431 | " 20180330 16:21:31 3 9 21 0 0 0 0 405 544\r\n", 432 | " 20180330 16:21:32 0 9 21 0 0 0 0 405 544\r\n", 433 | " 20180330 16:21:32 1 9 26 0 0 0 0 405 544\r\n", 434 | " 20180330 16:21:32 2 9 21 0 0 0 0 405 544\r\n", 435 | " 20180330 16:21:32 3 9 21 0 0 0 0 405 544\r\n", 436 | " 20180330 16:21:33 0 9 21 0 0 0 0 405 544\r\n", 437 | " 20180330 16:21:33 1 9 26 0 0 0 0 405 544\r\n", 438 | " 20180330 16:21:33 2 9 21 0 0 0 0 405 544\r\n", 439 | " 20180330 16:21:33 3 9 21 0 0 0 0 405 544\r\n", 440 | " 20180330 16:21:35 0 9 21 0 0 0 0 405 544\r\n", 441 | " 20180330 16:21:35 1 9 26 0 0 0 0 405 544\r\n", 442 | " 20180330 16:21:35 2 9 21 0 0 0 0 405 544\r\n", 443 | " 20180330 16:21:35 3 9 21 0 0 0 0 405 544\r\n", 444 | " 20180330 16:21:36 0 10 21 0 0 0 0 405 544\r\n", 445 | " 20180330 16:21:36 1 10 26 0 0 0 0 405 544\r\n", 446 | " 20180330 16:21:36 2 9 21 0 0 0 0 405 544\r\n", 447 | " 20180330 16:21:36 3 9 21 0 0 0 0 405 544\r\n" 448 | ] 449 | } 450 | ], 451 | "source": [ 452 | "!cat test_gpu.txt" 453 | ] 454 | }, 455 | { 456 | "cell_type": "markdown", 457 | "metadata": {}, 458 | "source": [ 459 | "By calling the log object we get all the data returned to us in a dataframe" 460 | ] 461 | }, 462 | { 463 | "cell_type": "code", 464 | "execution_count": 10, 465 | "metadata": {}, 466 | "outputs": [], 467 | "source": [ 468 | "df = log()" 469 | ] 470 | }, 471 | { 472 | "cell_type": "code", 473 | "execution_count": 11, 474 | "metadata": {}, 475 | "outputs": [ 476 | { 477 | "data": { 478 | "text/html": [ 479 | "
\n", 480 | "\n", 493 | "\n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | "
timestampgpupwrtempsmmemencdecmclkpclk
02018-03-30 16:21:28010210000405544
12018-03-30 16:21:2819260000405544
22018-03-30 16:21:2829210000405544
32018-03-30 16:21:2839210000405544
42018-03-30 16:21:2909210000405544
52018-03-30 16:21:2919260000405544
62018-03-30 16:21:2929210000405544
72018-03-30 16:21:2939210000405544
82018-03-30 16:21:3109210000405544
92018-03-30 16:21:3119260000405544
102018-03-30 16:21:3129210000405544
112018-03-30 16:21:3139210000405544
122018-03-30 16:21:3209210000405544
132018-03-30 16:21:3219260000405544
142018-03-30 16:21:3229210000405544
152018-03-30 16:21:3239210000405544
162018-03-30 16:21:3309210000405544
172018-03-30 16:21:3319260000405544
182018-03-30 16:21:3329210000405544
192018-03-30 16:21:3339210000405544
202018-03-30 16:21:3509210000405544
212018-03-30 16:21:3519260000405544
222018-03-30 16:21:3529210000405544
232018-03-30 16:21:3539210000405544
242018-03-30 16:21:36010210000405544
252018-03-30 16:21:36110260000405544
262018-03-30 16:21:3629210000405544
272018-03-30 16:21:3639210000405544
\n", 876 | "
" 877 | ], 878 | "text/plain": [ 879 | " timestamp gpu pwr temp sm mem enc dec mclk pclk\n", 880 | "0 2018-03-30 16:21:28 0 10 21 0 0 0 0 405 544\n", 881 | "1 2018-03-30 16:21:28 1 9 26 0 0 0 0 405 544\n", 882 | "2 2018-03-30 16:21:28 2 9 21 0 0 0 0 405 544\n", 883 | "3 2018-03-30 16:21:28 3 9 21 0 0 0 0 405 544\n", 884 | "4 2018-03-30 16:21:29 0 9 21 0 0 0 0 405 544\n", 885 | "5 2018-03-30 16:21:29 1 9 26 0 0 0 0 405 544\n", 886 | "6 2018-03-30 16:21:29 2 9 21 0 0 0 0 405 544\n", 887 | "7 2018-03-30 16:21:29 3 9 21 0 0 0 0 405 544\n", 888 | "8 2018-03-30 16:21:31 0 9 21 0 0 0 0 405 544\n", 889 | "9 2018-03-30 16:21:31 1 9 26 0 0 0 0 405 544\n", 890 | "10 2018-03-30 16:21:31 2 9 21 0 0 0 0 405 544\n", 891 | "11 2018-03-30 16:21:31 3 9 21 0 0 0 0 405 544\n", 892 | "12 2018-03-30 16:21:32 0 9 21 0 0 0 0 405 544\n", 893 | "13 2018-03-30 16:21:32 1 9 26 0 0 0 0 405 544\n", 894 | "14 2018-03-30 16:21:32 2 9 21 0 0 0 0 405 544\n", 895 | "15 2018-03-30 16:21:32 3 9 21 0 0 0 0 405 544\n", 896 | "16 2018-03-30 16:21:33 0 9 21 0 0 0 0 405 544\n", 897 | "17 2018-03-30 16:21:33 1 9 26 0 0 0 0 405 544\n", 898 | "18 2018-03-30 16:21:33 2 9 21 0 0 0 0 405 544\n", 899 | "19 2018-03-30 16:21:33 3 9 21 0 0 0 0 405 544\n", 900 | "20 2018-03-30 16:21:35 0 9 21 0 0 0 0 405 544\n", 901 | "21 2018-03-30 16:21:35 1 9 26 0 0 0 0 405 544\n", 902 | "22 2018-03-30 16:21:35 2 9 21 0 0 0 0 405 544\n", 903 | "23 2018-03-30 16:21:35 3 9 21 0 0 0 0 405 544\n", 904 | "24 2018-03-30 16:21:36 0 10 21 0 0 0 0 405 544\n", 905 | "25 2018-03-30 16:21:36 1 10 26 0 0 0 0 405 544\n", 906 | "26 2018-03-30 16:21:36 2 9 21 0 0 0 0 405 544\n", 907 | "27 2018-03-30 16:21:36 3 9 21 0 0 0 0 405 544" 908 | ] 909 | }, 910 | "execution_count": 11, 911 | "metadata": {}, 912 | "output_type": "execute_result" 913 | } 914 | ], 915 | "source": [ 916 | "df" 917 | ] 918 | }, 919 | { 920 | "cell_type": "markdown", 921 | "metadata": {}, 922 | "source": [ 923 | "We can also call plot on the log object to plot the measurements we want" 924 | ] 925 | }, 926 | { 927 | "cell_type": "code", 928 | "execution_count": 21, 929 | "metadata": {}, 930 | "outputs": [], 931 | "source": [ 932 | "p = log.plot(gpu_measurement='pwr', num_gpus=4)" 933 | ] 934 | }, 935 | { 936 | "cell_type": "code", 937 | "execution_count": 20, 938 | "metadata": {}, 939 | "outputs": [ 940 | { 941 | "data": { 942 | "text/html": [ 943 | "\n", 944 | "
\n", 945 | "
\n", 946 | "
" 947 | ] 948 | }, 949 | "metadata": {}, 950 | "output_type": "display_data" 951 | }, 952 | { 953 | "data": { 954 | "application/javascript": [ 955 | "(function(root) {\n", 956 | " function embed_document(root) {\n", 957 | " \n", 958 | " var docs_json = {\"07ae0cf9-d00b-42a8-b203-9965751e5a7d\":{\"roots\":{\"references\":[{\"attributes\":{\"callback\":null},\"id\":\"c96db59a-c74c-4c6e-a3df-b9067adf87a8\",\"type\":\"DataRange1d\"},{\"attributes\":{\"dimension\":1,\"plot\":{\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"30c8c6c9-42a7-4581-8bcb-f35c88c48aa8\",\"type\":\"BasicTicker\"}},\"id\":\"21bc9bc1-1f28-4e0d-b865-18eeec765024\",\"type\":\"Grid\"},{\"attributes\":{\"line_color\":\"#1f78b4\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 1\"}},\"id\":\"258f39b8-17b5-4bef-bc44-abb0b15ea855\",\"type\":\"Line\"},{\"attributes\":{\"months\":[0,2,4,6,8,10]},\"id\":\"454ed3cf-31f4-481b-97bd-2b91108c2678\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"label\":{\"value\":\"GPU 0\"},\"renderers\":[{\"id\":\"b64f4ef7-ae2d-4faf-aea9-d7d2fd0aa1c5\",\"type\":\"GlyphRenderer\"}]},\"id\":\"e71ee2f0-485c-4314-b4d4-6dbddf5955cf\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"e1c74eff-aa1e-4526-8cc9-1e13cc79fcfd\",\"type\":\"SaveTool\"},{\"attributes\":{\"num_minor_ticks\":5,\"tickers\":[{\"id\":\"aaf7a149-cd35-4fc0-a790-f30055bef2a8\",\"type\":\"AdaptiveTicker\"},{\"id\":\"ece6293d-fd8b-480e-ada2-785bdf2a37c9\",\"type\":\"AdaptiveTicker\"},{\"id\":\"883e7b62-721d-474d-a87b-4c74026d5ab2\",\"type\":\"AdaptiveTicker\"},{\"id\":\"71ea7253-502c-47da-85e8-7fd754f18d6b\",\"type\":\"DaysTicker\"},{\"id\":\"840175eb-bf79-4f40-9a38-093c7797a58c\",\"type\":\"DaysTicker\"},{\"id\":\"8a68faea-6955-417f-b0a1-c0ba5d4b798a\",\"type\":\"DaysTicker\"},{\"id\":\"fdfde3c6-3d24-4d3c-962c-3bff55a1e6ca\",\"type\":\"DaysTicker\"},{\"id\":\"ec904a2d-97fb-42da-9890-a9f59fe6f93a\",\"type\":\"MonthsTicker\"},{\"id\":\"454ed3cf-31f4-481b-97bd-2b91108c2678\",\"type\":\"MonthsTicker\"},{\"id\":\"d251c3e7-a93f-4890-af85-2768b464afd1\",\"type\":\"MonthsTicker\"},{\"id\":\"a06ec688-cd17-41d6-ad3b-57d7db427b2a\",\"type\":\"MonthsTicker\"},{\"id\":\"a9be8fca-4f3f-4c79-a0c6-cead8c0ab431\",\"type\":\"YearsTicker\"}]},\"id\":\"93d14b30-c552-4d6a-9858-110b1e265388\",\"type\":\"DatetimeTicker\"},{\"attributes\":{\"source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"}},\"id\":\"22626edc-ec11-4b9a-9fc0-e75d8f67d9b2\",\"type\":\"CDSView\"},{\"attributes\":{\"overlay\":{\"id\":\"2a547a47-1beb-45f8-b50c-bed2a2b85147\",\"type\":\"BoxAnnotation\"}},\"id\":\"87529c7e-2017-4253-9f1f-b0867c088435\",\"type\":\"BoxZoomTool\"},{\"attributes\":{\"days\":[1,15]},\"id\":\"fdfde3c6-3d24-4d3c-962c-3bff55a1e6ca\",\"type\":\"DaysTicker\"},{\"attributes\":{\"bottom_units\":\"screen\",\"fill_alpha\":{\"value\":0.5},\"fill_color\":{\"value\":\"lightgrey\"},\"left_units\":\"screen\",\"level\":\"overlay\",\"line_alpha\":{\"value\":1.0},\"line_color\":{\"value\":\"black\"},\"line_dash\":[4,4],\"line_width\":{\"value\":2},\"plot\":null,\"render_mode\":\"css\",\"right_units\":\"screen\",\"top_units\":\"screen\"},\"id\":\"2a547a47-1beb-45f8-b50c-bed2a2b85147\",\"type\":\"BoxAnnotation\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 1\"}},\"id\":\"d4f02dfe-f8cf-4507-bf15-40ba5b9aa430\",\"type\":\"Line\"},{\"attributes\":{\"below\":[{\"id\":\"924b8312-c942-4f45-a9f8-48a518b81263\",\"type\":\"DatetimeAxis\"}],\"left\":[{\"id\":\"e130cdc7-4e63-41f0-8737-5aafb45dda62\",\"type\":\"LinearAxis\"}],\"plot_height\":400,\"renderers\":[{\"id\":\"924b8312-c942-4f45-a9f8-48a518b81263\",\"type\":\"DatetimeAxis\"},{\"id\":\"70ccb8a9-2d91-4014-b5da-201bd25be729\",\"type\":\"Grid\"},{\"id\":\"e130cdc7-4e63-41f0-8737-5aafb45dda62\",\"type\":\"LinearAxis\"},{\"id\":\"21bc9bc1-1f28-4e0d-b865-18eeec765024\",\"type\":\"Grid\"},{\"id\":\"2a547a47-1beb-45f8-b50c-bed2a2b85147\",\"type\":\"BoxAnnotation\"},{\"id\":\"178f3d16-09e1-4989-b5a2-6ad359084074\",\"type\":\"Legend\"},{\"id\":\"b64f4ef7-ae2d-4faf-aea9-d7d2fd0aa1c5\",\"type\":\"GlyphRenderer\"},{\"id\":\"86db11d6-de99-471f-8557-e2479d4b55ab\",\"type\":\"GlyphRenderer\"},{\"id\":\"673a851a-d5e9-4e1d-b413-285e41805617\",\"type\":\"GlyphRenderer\"},{\"id\":\"cb69546b-70b8-4556-8705-8a3c5168a209\",\"type\":\"GlyphRenderer\"}],\"title\":{\"id\":\"5a6acbac-696b-480b-bda6-dcdb0f1754ca\",\"type\":\"Title\"},\"toolbar\":{\"id\":\"765ee3c1-6195-45d3-9ed4-4677b898db55\",\"type\":\"Toolbar\"},\"x_range\":{\"id\":\"c96db59a-c74c-4c6e-a3df-b9067adf87a8\",\"type\":\"DataRange1d\"},\"x_scale\":{\"id\":\"9bd3254b-c41b-413a-96dd-473a6c7138fe\",\"type\":\"LinearScale\"},\"y_range\":{\"id\":\"6900f29c-e1db-4a17-9e15-b9ca3a2a3782\",\"type\":\"Range1d\"},\"y_scale\":{\"id\":\"1bd755da-fffb-4224-8381-183f1b286eaa\",\"type\":\"LinearScale\"}},\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},{\"attributes\":{\"data_source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"258f39b8-17b5-4bef-bc44-abb0b15ea855\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"d4f02dfe-f8cf-4507-bf15-40ba5b9aa430\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"22626edc-ec11-4b9a-9fc0-e75d8f67d9b2\",\"type\":\"CDSView\"}},\"id\":\"86db11d6-de99-471f-8557-e2479d4b55ab\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"label\":{\"value\":\"GPU 1\"},\"renderers\":[{\"id\":\"86db11d6-de99-471f-8557-e2479d4b55ab\",\"type\":\"GlyphRenderer\"}]},\"id\":\"dc4d3f18-76a9-4279-bb56-4f9d8c6068a6\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"a9be8fca-4f3f-4c79-a0c6-cead8c0ab431\",\"type\":\"YearsTicker\"},{\"attributes\":{\"days\":[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]},\"id\":\"71ea7253-502c-47da-85e8-7fd754f18d6b\",\"type\":\"DaysTicker\"},{\"attributes\":{\"mantissas\":[1,2,5],\"max_interval\":500.0,\"num_minor_ticks\":0},\"id\":\"aaf7a149-cd35-4fc0-a790-f30055bef2a8\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 0\"}},\"id\":\"be6a16f1-28e1-4bf1-95f7-e1d2b2f0d93a\",\"type\":\"Line\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 2\"}},\"id\":\"bff0a1a0-9580-413e-8726-e42027059f77\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"f0a72514-4766-47cc-b01c-ccb00683c5c8\",\"type\":\"HelpTool\"},{\"attributes\":{\"line_color\":\"#33a02c\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 3\"}},\"id\":\"f8532390-fa29-4af1-9e4f-1d887dbbc45c\",\"type\":\"Line\"},{\"attributes\":{},\"id\":\"9bd3254b-c41b-413a-96dd-473a6c7138fe\",\"type\":\"LinearScale\"},{\"attributes\":{\"callback\":null,\"column_names\":[\"gpu 0\",\"gpu 1\",\"gpu 2\",\"gpu 3\",\"timestamp\"],\"data\":{\"gpu 0\":[21,21,21,21,21,21,21],\"gpu 1\":[26,26,26,26,26,26,26],\"gpu 2\":[21,21,21,21,21,21,21],\"gpu 3\":[21,21,21,21,21,21,21],\"timestamp\":{\"__ndarray__\":\"AAD0a3sndkIAgDJseyd2QgCAr2x7J3ZCAADubHsndkIAgCxteyd2QgCAqW17J3ZCAADobXsndkI=\",\"dtype\":\"float64\",\"shape\":[7]}}},\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"},{\"attributes\":{\"months\":[0,1,2,3,4,5,6,7,8,9,10,11]},\"id\":\"ec904a2d-97fb-42da-9890-a9f59fe6f93a\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"line_color\":\"#b2df8a\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 2\"}},\"id\":\"16776a4a-58c8-4e2e-a6be-56da102cbf2b\",\"type\":\"Line\"},{\"attributes\":{\"source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"}},\"id\":\"415b87e7-cbaf-4478-b359-d12f21db7b3b\",\"type\":\"CDSView\"},{\"attributes\":{\"source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"}},\"id\":\"4819cda0-954e-470f-971b-af26f9708806\",\"type\":\"CDSView\"},{\"attributes\":{\"data_source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"16776a4a-58c8-4e2e-a6be-56da102cbf2b\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"bff0a1a0-9580-413e-8726-e42027059f77\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"4819cda0-954e-470f-971b-af26f9708806\",\"type\":\"CDSView\"}},\"id\":\"673a851a-d5e9-4e1d-b413-285e41805617\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"formatter\":{\"id\":\"da26cbc4-421c-433a-9813-b488368a4ff1\",\"type\":\"BasicTickFormatter\"},\"plot\":{\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"30c8c6c9-42a7-4581-8bcb-f35c88c48aa8\",\"type\":\"BasicTicker\"}},\"id\":\"e130cdc7-4e63-41f0-8737-5aafb45dda62\",\"type\":\"LinearAxis\"},{\"attributes\":{\"label\":{\"value\":\"GPU 2\"},\"renderers\":[{\"id\":\"673a851a-d5e9-4e1d-b413-285e41805617\",\"type\":\"GlyphRenderer\"}]},\"id\":\"e93d094b-aa3e-4862-b248-7bea04418350\",\"type\":\"LegendItem\"},{\"attributes\":{\"base\":24,\"mantissas\":[1,2,4,6,8,12],\"max_interval\":43200000.0,\"min_interval\":3600000.0,\"num_minor_ticks\":0},\"id\":\"883e7b62-721d-474d-a87b-4c74026d5ab2\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"days\":[1,4,7,10,13,16,19,22,25,28]},\"id\":\"840175eb-bf79-4f40-9a38-093c7797a58c\",\"type\":\"DaysTicker\"},{\"attributes\":{},\"id\":\"381e56e7-f60c-4d9f-b124-8c19b0b9aabd\",\"type\":\"WheelZoomTool\"},{\"attributes\":{\"plot\":null,\"text\":\"\"},\"id\":\"5a6acbac-696b-480b-bda6-dcdb0f1754ca\",\"type\":\"Title\"},{\"attributes\":{},\"id\":\"5178e6b8-caad-4ed5-a255-a61e60f25512\",\"type\":\"PanTool\"},{\"attributes\":{},\"id\":\"da26cbc4-421c-433a-9813-b488368a4ff1\",\"type\":\"BasicTickFormatter\"},{\"attributes\":{\"source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"}},\"id\":\"6afba6ad-d5e8-49dd-965e-500b104305f4\",\"type\":\"CDSView\"},{\"attributes\":{\"base\":60,\"mantissas\":[1,2,5,10,15,20,30],\"max_interval\":1800000.0,\"min_interval\":1000.0,\"num_minor_ticks\":0},\"id\":\"ece6293d-fd8b-480e-ada2-785bdf2a37c9\",\"type\":\"AdaptiveTicker\"},{\"attributes\":{\"formatter\":{\"id\":\"5003b373-7b10-48ad-a5d1-9825529e66d5\",\"type\":\"DatetimeTickFormatter\"},\"plot\":{\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"93d14b30-c552-4d6a-9858-110b1e265388\",\"type\":\"DatetimeTicker\"}},\"id\":\"924b8312-c942-4f45-a9f8-48a518b81263\",\"type\":\"DatetimeAxis\"},{\"attributes\":{\"line_alpha\":0.1,\"line_color\":\"#1f77b4\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 3\"}},\"id\":\"18099ea4-9ed0-4980-bc52-6d6dd145e8fb\",\"type\":\"Line\"},{\"attributes\":{\"data_source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"f8532390-fa29-4af1-9e4f-1d887dbbc45c\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"18099ea4-9ed0-4980-bc52-6d6dd145e8fb\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"6afba6ad-d5e8-49dd-965e-500b104305f4\",\"type\":\"CDSView\"}},\"id\":\"cb69546b-70b8-4556-8705-8a3c5168a209\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"months\":[0,4,8]},\"id\":\"d251c3e7-a93f-4890-af85-2768b464afd1\",\"type\":\"MonthsTicker\"},{\"attributes\":{\"label\":{\"value\":\"GPU 3\"},\"renderers\":[{\"id\":\"cb69546b-70b8-4556-8705-8a3c5168a209\",\"type\":\"GlyphRenderer\"}]},\"id\":\"bf071b89-79d1-428b-9362-3621543d975e\",\"type\":\"LegendItem\"},{\"attributes\":{},\"id\":\"b82e16c8-fb17-4c84-b134-124f08873e7e\",\"type\":\"ResetTool\"},{\"attributes\":{\"data_source\":{\"id\":\"29553d7c-3be5-46ff-95ed-ba136df679c9\",\"type\":\"ColumnDataSource\"},\"glyph\":{\"id\":\"60223a1a-2a27-4012-b7ed-cbc8773fb248\",\"type\":\"Line\"},\"hover_glyph\":null,\"muted_glyph\":null,\"nonselection_glyph\":{\"id\":\"be6a16f1-28e1-4bf1-95f7-e1d2b2f0d93a\",\"type\":\"Line\"},\"selection_glyph\":null,\"view\":{\"id\":\"415b87e7-cbaf-4478-b359-d12f21db7b3b\",\"type\":\"CDSView\"}},\"id\":\"b64f4ef7-ae2d-4faf-aea9-d7d2fd0aa1c5\",\"type\":\"GlyphRenderer\"},{\"attributes\":{\"items\":[{\"id\":\"e71ee2f0-485c-4314-b4d4-6dbddf5955cf\",\"type\":\"LegendItem\"},{\"id\":\"dc4d3f18-76a9-4279-bb56-4f9d8c6068a6\",\"type\":\"LegendItem\"},{\"id\":\"e93d094b-aa3e-4862-b248-7bea04418350\",\"type\":\"LegendItem\"},{\"id\":\"bf071b89-79d1-428b-9362-3621543d975e\",\"type\":\"LegendItem\"}],\"plot\":{\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"}},\"id\":\"178f3d16-09e1-4989-b5a2-6ad359084074\",\"type\":\"Legend\"},{\"attributes\":{\"plot\":{\"id\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\",\"subtype\":\"Figure\",\"type\":\"Plot\"},\"ticker\":{\"id\":\"93d14b30-c552-4d6a-9858-110b1e265388\",\"type\":\"DatetimeTicker\"}},\"id\":\"70ccb8a9-2d91-4014-b5da-201bd25be729\",\"type\":\"Grid\"},{\"attributes\":{\"active_drag\":\"auto\",\"active_inspect\":\"auto\",\"active_scroll\":\"auto\",\"active_tap\":\"auto\",\"tools\":[{\"id\":\"5178e6b8-caad-4ed5-a255-a61e60f25512\",\"type\":\"PanTool\"},{\"id\":\"381e56e7-f60c-4d9f-b124-8c19b0b9aabd\",\"type\":\"WheelZoomTool\"},{\"id\":\"87529c7e-2017-4253-9f1f-b0867c088435\",\"type\":\"BoxZoomTool\"},{\"id\":\"e1c74eff-aa1e-4526-8cc9-1e13cc79fcfd\",\"type\":\"SaveTool\"},{\"id\":\"b82e16c8-fb17-4c84-b134-124f08873e7e\",\"type\":\"ResetTool\"},{\"id\":\"f0a72514-4766-47cc-b01c-ccb00683c5c8\",\"type\":\"HelpTool\"}]},\"id\":\"765ee3c1-6195-45d3-9ed4-4677b898db55\",\"type\":\"Toolbar\"},{\"attributes\":{},\"id\":\"1bd755da-fffb-4224-8381-183f1b286eaa\",\"type\":\"LinearScale\"},{\"attributes\":{},\"id\":\"5003b373-7b10-48ad-a5d1-9825529e66d5\",\"type\":\"DatetimeTickFormatter\"},{\"attributes\":{\"months\":[0,6]},\"id\":\"a06ec688-cd17-41d6-ad3b-57d7db427b2a\",\"type\":\"MonthsTicker\"},{\"attributes\":{},\"id\":\"30c8c6c9-42a7-4581-8bcb-f35c88c48aa8\",\"type\":\"BasicTicker\"},{\"attributes\":{\"callback\":null,\"end\":110},\"id\":\"6900f29c-e1db-4a17-9e15-b9ca3a2a3782\",\"type\":\"Range1d\"},{\"attributes\":{\"line_color\":\"#a6cee3\",\"line_width\":4,\"x\":{\"field\":\"timestamp\"},\"y\":{\"field\":\"gpu 0\"}},\"id\":\"60223a1a-2a27-4012-b7ed-cbc8773fb248\",\"type\":\"Line\"},{\"attributes\":{\"days\":[1,8,15,22]},\"id\":\"8a68faea-6955-417f-b0a1-c0ba5d4b798a\",\"type\":\"DaysTicker\"}],\"root_ids\":[\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\"]},\"title\":\"Bokeh Application\",\"version\":\"0.12.13\"}};\n", 959 | " var render_items = [{\"docid\":\"07ae0cf9-d00b-42a8-b203-9965751e5a7d\",\"elementid\":\"f5fb9fbc-b6fd-4d1f-8def-c09e2b5ba6c4\",\"modelid\":\"85e8b8fd-db36-4b93-ad90-ef52da32dde0\"}];\n", 960 | " root.Bokeh.embed.embed_items_notebook(docs_json, render_items);\n", 961 | "\n", 962 | " }\n", 963 | " if (root.Bokeh !== undefined) {\n", 964 | " embed_document(root);\n", 965 | " } else {\n", 966 | " var attempts = 0;\n", 967 | " var timer = setInterval(function(root) {\n", 968 | " if (root.Bokeh !== undefined) {\n", 969 | " embed_document(root);\n", 970 | " clearInterval(timer);\n", 971 | " }\n", 972 | " attempts++;\n", 973 | " if (attempts > 100) {\n", 974 | " console.log(\"Bokeh: ERROR: Unable to run BokehJS code because BokehJS library is missing\")\n", 975 | " clearInterval(timer);\n", 976 | " }\n", 977 | " }, 10, root)\n", 978 | " }\n", 979 | "})(window);" 980 | ], 981 | "application/vnd.bokehjs_exec.v0+json": "" 982 | }, 983 | "metadata": { 984 | "application/vnd.bokehjs_exec.v0+json": { 985 | "id": "85e8b8fd-db36-4b93-ad90-ef52da32dde0" 986 | } 987 | }, 988 | "output_type": "display_data" 989 | } 990 | ], 991 | "source": [ 992 | "show(p)" 993 | ] 994 | } 995 | ], 996 | "metadata": { 997 | "kernelspec": { 998 | "display_name": "Python 3", 999 | "language": "python", 1000 | "name": "python3" 1001 | }, 1002 | "language_info": { 1003 | "codemirror_mode": { 1004 | "name": "ipython", 1005 | "version": 3 1006 | }, 1007 | "file_extension": ".py", 1008 | "mimetype": "text/x-python", 1009 | "name": "python", 1010 | "nbconvert_exporter": "python", 1011 | "pygments_lexer": "ipython3", 1012 | "version": "3.6.4" 1013 | } 1014 | }, 1015 | "nbformat": 4, 1016 | "nbformat_minor": 2 1017 | } 1018 | -------------------------------------------------------------------------------- /examples/notebooks/InfluxDBLoggerExample.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "markdown", 5 | "metadata": {}, 6 | "source": [ 7 | "# InfluxDB Logger Example\n", 8 | "\n", 9 | "This notebook is a small demo of how to use gpumon in Jupyter notebooks and some convenience methods for working with GPUs\n", 10 | "You will need to have PyTorch and Torchvision installed to run this as well as the python InfluxDB client\n", 11 | "\n", 12 | "To install Pytorch and associated requiremetns run the following:\n", 13 | "```bash\n", 14 | "cuda install pytorch torchvision cuda80 -c python\n", 15 | "```\n", 16 | "\n", 17 | "To install python InfluxDB client\n", 18 | "```bash\n", 19 | "pip install influxdb\n", 20 | "```\n", 21 | "see [here](https://github.com/influxdata/influxdb-python) for more details on the InfluxDB client" 22 | ] 23 | }, 24 | { 25 | "cell_type": "code", 26 | "execution_count": 37, 27 | "metadata": {}, 28 | "outputs": [], 29 | "source": [ 30 | "from gpumon import device_count, device_name" 31 | ] 32 | }, 33 | { 34 | "cell_type": "code", 35 | "execution_count": 38, 36 | "metadata": {}, 37 | "outputs": [ 38 | { 39 | "data": { 40 | "text/plain": [ 41 | "4" 42 | ] 43 | }, 44 | "execution_count": 38, 45 | "metadata": {}, 46 | "output_type": "execute_result" 47 | } 48 | ], 49 | "source": [ 50 | "device_count() # Returns the number of GPUs available" 51 | ] 52 | }, 53 | { 54 | "cell_type": "code", 55 | "execution_count": 39, 56 | "metadata": {}, 57 | "outputs": [ 58 | { 59 | "data": { 60 | "text/plain": [ 61 | "'Tesla P40'" 62 | ] 63 | }, 64 | "execution_count": 39, 65 | "metadata": {}, 66 | "output_type": "execute_result" 67 | } 68 | ], 69 | "source": [ 70 | "device_name() # Returns the type of GPU available" 71 | ] 72 | }, 73 | { 74 | "cell_type": "markdown", 75 | "metadata": {}, 76 | "source": [ 77 | "Let's create a simple CNN and run the CIFAR dataset against it to see the load on our GPU" 78 | ] 79 | }, 80 | { 81 | "cell_type": "code", 82 | "execution_count": 1, 83 | "metadata": {}, 84 | "outputs": [], 85 | "source": [ 86 | "import torch\n", 87 | "import torchvision\n", 88 | "import torchvision.transforms as transforms" 89 | ] 90 | }, 91 | { 92 | "cell_type": "code", 93 | "execution_count": 2, 94 | "metadata": {}, 95 | "outputs": [ 96 | { 97 | "name": "stdout", 98 | "output_type": "stream", 99 | "text": [ 100 | "Files already downloaded and verified\n" 101 | ] 102 | } 103 | ], 104 | "source": [ 105 | "transform = transforms.Compose(\n", 106 | " [transforms.ToTensor(),\n", 107 | " transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])\n", 108 | "\n", 109 | "trainset = torchvision.datasets.CIFAR10(root='./data', train=True,\n", 110 | " download=True, transform=transform)\n", 111 | "trainloader = torch.utils.data.DataLoader(trainset, batch_size=64,\n", 112 | " shuffle=True, num_workers=4)\n", 113 | "\n", 114 | "classes = ('plane', 'car', 'bird', 'cat',\n", 115 | "'deer', 'dog', 'frog', 'horse', 'ship', 'truck')" 116 | ] 117 | }, 118 | { 119 | "cell_type": "code", 120 | "execution_count": 3, 121 | "metadata": {}, 122 | "outputs": [], 123 | "source": [ 124 | "from torch.autograd import Variable\n", 125 | "import torch.nn as nn\n", 126 | "import torch.nn.functional as F\n", 127 | "\n", 128 | "\n", 129 | "class Net(nn.Module):\n", 130 | " def __init__(self):\n", 131 | " super(Net, self).__init__()\n", 132 | " self.conv1 = nn.Conv2d(3, 6, 5)\n", 133 | " self.pool = nn.MaxPool2d(2, 2)\n", 134 | " self.conv2 = nn.Conv2d(6, 16, 5)\n", 135 | " self.fc1 = nn.Linear(16 * 5 * 5, 120)\n", 136 | " self.fc2 = nn.Linear(120, 84)\n", 137 | " self.fc3 = nn.Linear(84, 10)\n", 138 | "\n", 139 | " def forward(self, x):\n", 140 | " x = self.pool(F.relu(self.conv1(x)))\n", 141 | " x = self.pool(F.relu(self.conv2(x)))\n", 142 | " x = x.view(-1, 16 * 5 * 5)\n", 143 | " x = F.relu(self.fc1(x))\n", 144 | " x = F.relu(self.fc2(x))\n", 145 | " x = self.fc3(x)\n", 146 | " return x\n", 147 | "\n", 148 | "\n", 149 | "net = Net()" 150 | ] 151 | }, 152 | { 153 | "cell_type": "code", 154 | "execution_count": 4, 155 | "metadata": {}, 156 | "outputs": [ 157 | { 158 | "data": { 159 | "text/plain": [ 160 | "Net(\n", 161 | " (conv1): Conv2d(3, 6, kernel_size=(5, 5), stride=(1, 1))\n", 162 | " (pool): MaxPool2d(kernel_size=(2, 2), stride=(2, 2), dilation=(1, 1), ceil_mode=False)\n", 163 | " (conv2): Conv2d(6, 16, kernel_size=(5, 5), stride=(1, 1))\n", 164 | " (fc1): Linear(in_features=400, out_features=120, bias=True)\n", 165 | " (fc2): Linear(in_features=120, out_features=84, bias=True)\n", 166 | " (fc3): Linear(in_features=84, out_features=10, bias=True)\n", 167 | ")" 168 | ] 169 | }, 170 | "execution_count": 4, 171 | "metadata": {}, 172 | "output_type": "execute_result" 173 | } 174 | ], 175 | "source": [ 176 | "net.cuda()" 177 | ] 178 | }, 179 | { 180 | "cell_type": "code", 181 | "execution_count": 5, 182 | "metadata": {}, 183 | "outputs": [], 184 | "source": [ 185 | "import torch.optim as optim\n", 186 | "\n", 187 | "criterion = nn.CrossEntropyLoss()\n", 188 | "optimizer = optim.SGD(net.parameters(), lr=0.001, momentum=0.9)" 189 | ] 190 | }, 191 | { 192 | "cell_type": "code", 193 | "execution_count": 6, 194 | "metadata": {}, 195 | "outputs": [], 196 | "source": [ 197 | "from gpumon.influxdb import log_context" 198 | ] 199 | }, 200 | { 201 | "cell_type": "code", 202 | "execution_count": 7, 203 | "metadata": {}, 204 | "outputs": [], 205 | "source": [ 206 | "display_every_minibatches=100" 207 | ] 208 | }, 209 | { 210 | "cell_type": "markdown", 211 | "metadata": {}, 212 | "source": [ 213 | "Be carefull that you specify the correct host and credentials in the context below" 214 | ] 215 | }, 216 | { 217 | "cell_type": "code", 218 | "execution_count": 8, 219 | "metadata": { 220 | "scrolled": false 221 | }, 222 | "outputs": [ 223 | { 224 | "name": "stdout", 225 | "output_type": "stream", 226 | "text": [ 227 | "[1] loss: 2.301\n", 228 | "[2] loss: 2.241\n", 229 | "[3] loss: 1.954\n", 230 | "[4] loss: 1.778\n", 231 | "[5] loss: 1.670\n", 232 | "[6] loss: 1.593\n", 233 | "[7] loss: 1.532\n", 234 | "[8] loss: 1.475\n", 235 | "[9] loss: 1.422\n", 236 | "[10] loss: 1.370\n", 237 | "[11] loss: 1.330\n", 238 | "[12] loss: 1.289\n", 239 | "[13] loss: 1.254\n", 240 | "[14] loss: 1.221\n", 241 | "[15] loss: 1.194\n", 242 | "[16] loss: 1.161\n", 243 | "[17] loss: 1.137\n", 244 | "[18] loss: 1.111\n", 245 | "[19] loss: 1.086\n", 246 | "[20] loss: 1.069\n", 247 | "Finished Training\n" 248 | ] 249 | } 250 | ], 251 | "source": [ 252 | "with log_context('localhost', 'admin', 'password', 'gpudb', 'gpuseries'):\n", 253 | " for epoch in range(20): # loop over the dataset multiple times\n", 254 | "\n", 255 | " running_loss = 0.0\n", 256 | " for i, data in enumerate(trainloader, 0):\n", 257 | " # get the inputs\n", 258 | " inputs, labels = data\n", 259 | "\n", 260 | " # wrap them in Variable\n", 261 | " inputs, labels = Variable(inputs.cuda()), Variable(labels.cuda())\n", 262 | "\n", 263 | " # zero the parameter gradients\n", 264 | " optimizer.zero_grad()\n", 265 | "\n", 266 | " # forward + backward + optimize\n", 267 | " outputs = net(inputs)\n", 268 | " loss = criterion(outputs, labels)\n", 269 | " loss.backward()\n", 270 | " optimizer.step()\n", 271 | "\n", 272 | " # print statistics\n", 273 | " running_loss += loss.data[0]\n", 274 | " print('[%d] loss: %.3f' %\n", 275 | " (epoch + 1, running_loss / (i+1)))\n", 276 | "\n", 277 | " print('Finished Training')" 278 | ] 279 | }, 280 | { 281 | "cell_type": "markdown", 282 | "metadata": {}, 283 | "source": [ 284 | "If you had your Grafana dashboard running you should have seen the measurements there. You can also pull the data from the database using the InfluxDB python client" 285 | ] 286 | }, 287 | { 288 | "cell_type": "code", 289 | "execution_count": 22, 290 | "metadata": {}, 291 | "outputs": [], 292 | "source": [ 293 | "from influxdb import InfluxDBClient, DataFrameClient" 294 | ] 295 | }, 296 | { 297 | "cell_type": "code", 298 | "execution_count": 14, 299 | "metadata": {}, 300 | "outputs": [], 301 | "source": [ 302 | "client = InfluxDBClient(host='localhost', username='admnin', password='password', database='gpudb')" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 15, 308 | "metadata": {}, 309 | "outputs": [ 310 | { 311 | "data": { 312 | "text/plain": [ 313 | "[{'name': 'gpuseries'}]" 314 | ] 315 | }, 316 | "execution_count": 15, 317 | "metadata": {}, 318 | "output_type": "execute_result" 319 | } 320 | ], 321 | "source": [ 322 | "client.get_list_measurements()" 323 | ] 324 | }, 325 | { 326 | "cell_type": "code", 327 | "execution_count": 30, 328 | "metadata": {}, 329 | "outputs": [], 330 | "source": [ 331 | "data = client.query('select * from gpuseries limit 10;')" 332 | ] 333 | }, 334 | { 335 | "cell_type": "code", 336 | "execution_count": 31, 337 | "metadata": {}, 338 | "outputs": [ 339 | { 340 | "data": { 341 | "text/plain": [ 342 | "influxdb.resultset.ResultSet" 343 | ] 344 | }, 345 | "execution_count": 31, 346 | "metadata": {}, 347 | "output_type": "execute_result" 348 | } 349 | ], 350 | "source": [ 351 | "type(data)" 352 | ] 353 | }, 354 | { 355 | "cell_type": "code", 356 | "execution_count": 32, 357 | "metadata": {}, 358 | "outputs": [ 359 | { 360 | "data": { 361 | "text/plain": [ 362 | "ResultSet({'('gpuseries', None)': [{'time': '2018-04-02T15:00:00.452204032Z', 'GPU': '0', 'Memory Used': 8685355008, 'Memory Used Percent': 36.14988257180032, 'Memory Utilization': 70, 'Power': 94728, 'Temperature': 39, 'Utilization': 90, 'timestamp': '2018-04-02 15:00:00.452204'}, {'time': '2018-04-02T15:00:00.499869184Z', 'GPU': '1', 'Memory Used': 6447693824, 'Memory Used Percent': 26.83636700881325, 'Memory Utilization': 56, 'Power': 57664, 'Temperature': 42, 'Utilization': 73, 'timestamp': '2018-04-02 15:00:00.499869'}, {'time': '2018-04-02T15:00:00.54717312Z', 'GPU': '2', 'Memory Used': 8507097088, 'Memory Used Percent': 35.40794365628589, 'Memory Utilization': 55, 'Power': 57827, 'Temperature': 38, 'Utilization': 68, 'timestamp': '2018-04-02 15:00:00.547173'}, {'time': '2018-04-02T15:00:00.595969024Z', 'GPU': '3', 'Memory Used': 2947547136, 'Memory Used Percent': 12.268178185359254, 'Memory Utilization': 38, 'Power': 62068, 'Temperature': 36, 'Utilization': 73, 'timestamp': '2018-04-02 15:00:00.595969'}, {'time': '2018-04-02T15:00:02.519879168Z', 'GPU': '0', 'Memory Used': 9931063296, 'Memory Used Percent': 41.334726287277654, 'Memory Utilization': 0, 'Power': 57226, 'Temperature': 36, 'Utilization': 0, 'timestamp': '2018-04-02 15:00:02.519879'}, {'time': '2018-04-02T15:00:02.565956096Z', 'GPU': '1', 'Memory Used': 7693402112, 'Memory Used Percent': 32.02121072429059, 'Memory Utilization': 0, 'Power': 56027, 'Temperature': 40, 'Utilization': 0, 'timestamp': '2018-04-02 15:00:02.565956'}, {'time': '2018-04-02T15:00:02.615364096Z', 'GPU': '2', 'Memory Used': 9752805376, 'Memory Used Percent': 40.59278737176322, 'Memory Utilization': 0, 'Power': 56572, 'Temperature': 36, 'Utilization': 0, 'timestamp': '2018-04-02 15:00:02.615364'}, {'time': '2018-04-02T15:00:02.66162304Z', 'GPU': '3', 'Memory Used': 3641704448, 'Memory Used Percent': 15.157375609303696, 'Memory Utilization': 1, 'Power': 185043, 'Temperature': 35, 'Utilization': 7, 'timestamp': '2018-04-02 15:00:02.661623'}, {'time': '2018-04-02T15:00:04.063194112Z', 'GPU': '0', 'Memory Used': 3687841792, 'Memory Used Percent': 22.428376981345146, 'Memory Utilization': 0, 'Power': 69760, 'Temperature': 36, 'Utilization': 0, 'timestamp': '2018-04-02 15:00:04.063194'}, {'time': '2018-04-02T15:00:04.11728512Z', 'GPU': '1', 'Memory Used': 4067426304, 'Memory Used Percent': 16.929300313414636, 'Memory Utilization': 0, 'Power': 57086, 'Temperature': 39, 'Utilization': 0, 'timestamp': '2018-04-02 15:00:04.117285'}]})" 363 | ] 364 | }, 365 | "execution_count": 32, 366 | "metadata": {}, 367 | "output_type": "execute_result" 368 | } 369 | ], 370 | "source": [ 371 | "data" 372 | ] 373 | }, 374 | { 375 | "cell_type": "code", 376 | "execution_count": 23, 377 | "metadata": {}, 378 | "outputs": [], 379 | "source": [ 380 | "df_client = DataFrameClient(host='localhost', username='admnin', password='password', database='gpudb')" 381 | ] 382 | }, 383 | { 384 | "cell_type": "code", 385 | "execution_count": 33, 386 | "metadata": {}, 387 | "outputs": [], 388 | "source": [ 389 | "df = df_client.query('select * from gpuseries limit 100;')['gpuseries']" 390 | ] 391 | }, 392 | { 393 | "cell_type": "code", 394 | "execution_count": 36, 395 | "metadata": { 396 | "scrolled": false 397 | }, 398 | "outputs": [ 399 | { 400 | "data": { 401 | "text/html": [ 402 | "
\n", 403 | "\n", 416 | "\n", 417 | " \n", 418 | " \n", 419 | " \n", 420 | " \n", 421 | " \n", 422 | " \n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | " \n", 548 | " \n", 549 | " \n", 550 | " \n", 551 | " \n", 552 | " \n", 553 | " \n", 554 | " \n", 555 | " \n", 556 | " \n", 557 | " \n", 558 | " \n", 559 | " \n", 560 | " \n", 561 | " \n", 562 | " \n", 563 | " \n", 564 | " \n", 565 | " \n", 566 | " \n", 567 | " \n", 568 | " \n", 569 | " \n", 570 | " \n", 571 | " \n", 572 | " \n", 573 | " \n", 574 | " \n", 575 | " \n", 576 | " \n", 577 | " \n", 578 | " \n", 579 | " \n", 580 | " \n", 581 | " \n", 582 | " \n", 583 | " \n", 584 | " \n", 585 | " \n", 586 | " \n", 587 | " \n", 588 | " \n", 589 | " \n", 590 | " \n", 591 | " \n", 592 | " \n", 593 | " \n", 594 | " \n", 595 | " \n", 596 | " \n", 597 | " \n", 598 | " \n", 599 | " \n", 600 | " \n", 601 | " \n", 602 | " \n", 603 | " \n", 604 | " \n", 605 | " \n", 606 | " \n", 607 | " \n", 608 | " \n", 609 | " \n", 610 | " \n", 611 | " \n", 612 | " \n", 613 | " \n", 614 | " \n", 615 | " \n", 616 | " \n", 617 | " \n", 618 | " \n", 619 | " \n", 620 | " \n", 621 | " \n", 622 | " \n", 623 | " \n", 624 | " \n", 625 | " \n", 626 | " \n", 627 | " \n", 628 | " \n", 629 | " \n", 630 | " \n", 631 | " \n", 632 | " \n", 633 | " \n", 634 | " \n", 635 | " \n", 636 | " \n", 637 | " \n", 638 | " \n", 639 | " \n", 640 | " \n", 641 | " \n", 642 | " \n", 643 | " \n", 644 | " \n", 645 | " \n", 646 | " \n", 647 | " \n", 648 | " \n", 649 | " \n", 650 | " \n", 651 | " \n", 652 | " \n", 653 | " \n", 654 | " \n", 655 | " \n", 656 | " \n", 657 | " \n", 658 | " \n", 659 | " \n", 660 | " \n", 661 | " \n", 662 | " \n", 663 | " \n", 664 | " \n", 665 | " \n", 666 | " \n", 667 | " \n", 668 | " \n", 669 | " \n", 670 | " \n", 671 | " \n", 672 | " \n", 673 | " \n", 674 | " \n", 675 | " \n", 676 | " \n", 677 | " \n", 678 | " \n", 679 | " \n", 680 | " \n", 681 | " \n", 682 | " \n", 683 | " \n", 684 | " \n", 685 | " \n", 686 | " \n", 687 | " \n", 688 | " \n", 689 | " \n", 690 | " \n", 691 | " \n", 692 | " \n", 693 | " \n", 694 | " \n", 695 | " \n", 696 | " \n", 697 | " \n", 698 | " \n", 699 | " \n", 700 | " \n", 701 | " \n", 702 | " \n", 703 | " \n", 704 | " \n", 705 | " \n", 706 | " \n", 707 | " \n", 708 | " \n", 709 | " \n", 710 | " \n", 711 | " \n", 712 | " \n", 713 | " \n", 714 | " \n", 715 | " \n", 716 | " \n", 717 | " \n", 718 | " \n", 719 | " \n", 720 | " \n", 721 | " \n", 722 | " \n", 723 | " \n", 724 | " \n", 725 | " \n", 726 | " \n", 727 | " \n", 728 | " \n", 729 | " \n", 730 | " \n", 731 | " \n", 732 | " \n", 733 | " \n", 734 | " \n", 735 | " \n", 736 | " \n", 737 | " \n", 738 | " \n", 739 | " \n", 740 | " \n", 741 | " \n", 742 | " \n", 743 | " \n", 744 | " \n", 745 | " \n", 746 | " \n", 747 | " \n", 748 | " \n", 749 | " \n", 750 | " \n", 751 | " \n", 752 | " \n", 753 | " \n", 754 | " \n", 755 | " \n", 756 | " \n", 757 | " \n", 758 | " \n", 759 | " \n", 760 | " \n", 761 | " \n", 762 | " \n", 763 | " \n", 764 | " \n", 765 | " \n", 766 | " \n", 767 | " \n", 768 | " \n", 769 | " \n", 770 | " \n", 771 | " \n", 772 | " \n", 773 | " \n", 774 | " \n", 775 | " \n", 776 | " \n", 777 | " \n", 778 | " \n", 779 | " \n", 780 | " \n", 781 | " \n", 782 | " \n", 783 | " \n", 784 | " \n", 785 | " \n", 786 | " \n", 787 | " \n", 788 | " \n", 789 | " \n", 790 | " \n", 791 | " \n", 792 | " \n", 793 | " \n", 794 | " \n", 795 | " \n", 796 | " \n", 797 | " \n", 798 | " \n", 799 | " \n", 800 | " \n", 801 | " \n", 802 | " \n", 803 | " \n", 804 | " \n", 805 | " \n", 806 | " \n", 807 | " \n", 808 | " \n", 809 | " \n", 810 | " \n", 811 | " \n", 812 | " \n", 813 | " \n", 814 | " \n", 815 | " \n", 816 | " \n", 817 | " \n", 818 | " \n", 819 | " \n", 820 | " \n", 821 | " \n", 822 | " \n", 823 | " \n", 824 | " \n", 825 | " \n", 826 | " \n", 827 | " \n", 828 | " \n", 829 | " \n", 830 | " \n", 831 | " \n", 832 | " \n", 833 | " \n", 834 | " \n", 835 | " \n", 836 | " \n", 837 | " \n", 838 | " \n", 839 | " \n", 840 | " \n", 841 | " \n", 842 | " \n", 843 | " \n", 844 | " \n", 845 | " \n", 846 | " \n", 847 | " \n", 848 | " \n", 849 | " \n", 850 | " \n", 851 | " \n", 852 | " \n", 853 | " \n", 854 | " \n", 855 | " \n", 856 | " \n", 857 | " \n", 858 | " \n", 859 | " \n", 860 | " \n", 861 | " \n", 862 | " \n", 863 | " \n", 864 | " \n", 865 | " \n", 866 | " \n", 867 | " \n", 868 | " \n", 869 | " \n", 870 | " \n", 871 | " \n", 872 | " \n", 873 | " \n", 874 | " \n", 875 | " \n", 876 | " \n", 877 | " \n", 878 | " \n", 879 | " \n", 880 | " \n", 881 | " \n", 882 | " \n", 883 | " \n", 884 | " \n", 885 | " \n", 886 | " \n", 887 | " \n", 888 | " \n", 889 | " \n", 890 | " \n", 891 | " \n", 892 | " \n", 893 | " \n", 894 | " \n", 895 | " \n", 896 | " \n", 897 | " \n", 898 | " \n", 899 | " \n", 900 | " \n", 901 | " \n", 902 | " \n", 903 | " \n", 904 | " \n", 905 | " \n", 906 | " \n", 907 | " \n", 908 | " \n", 909 | " \n", 910 | " \n", 911 | " \n", 912 | " \n", 913 | " \n", 914 | " \n", 915 | " \n", 916 | " \n", 917 | " \n", 918 | " \n", 919 | " \n", 920 | " \n", 921 | " \n", 922 | " \n", 923 | " \n", 924 | " \n", 925 | " \n", 926 | " \n", 927 | " \n", 928 | " \n", 929 | " \n", 930 | " \n", 931 | " \n", 932 | " \n", 933 | " \n", 934 | " \n", 935 | " \n", 936 | " \n", 937 | " \n", 938 | " \n", 939 | " \n", 940 | " \n", 941 | " \n", 942 | " \n", 943 | " \n", 944 | " \n", 945 | " \n", 946 | " \n", 947 | " \n", 948 | " \n", 949 | " \n", 950 | " \n", 951 | " \n", 952 | " \n", 953 | " \n", 954 | " \n", 955 | " \n", 956 | " \n", 957 | " \n", 958 | " \n", 959 | " \n", 960 | " \n", 961 | " \n", 962 | " \n", 963 | " \n", 964 | " \n", 965 | " \n", 966 | " \n", 967 | " \n", 968 | " \n", 969 | " \n", 970 | " \n", 971 | " \n", 972 | " \n", 973 | " \n", 974 | " \n", 975 | " \n", 976 | " \n", 977 | " \n", 978 | " \n", 979 | " \n", 980 | " \n", 981 | " \n", 982 | " \n", 983 | " \n", 984 | " \n", 985 | " \n", 986 | " \n", 987 | " \n", 988 | " \n", 989 | " \n", 990 | " \n", 991 | " \n", 992 | " \n", 993 | " \n", 994 | " \n", 995 | " \n", 996 | " \n", 997 | " \n", 998 | " \n", 999 | " \n", 1000 | " \n", 1001 | " \n", 1002 | " \n", 1003 | " \n", 1004 | " \n", 1005 | " \n", 1006 | " \n", 1007 | " \n", 1008 | " \n", 1009 | " \n", 1010 | " \n", 1011 | " \n", 1012 | " \n", 1013 | " \n", 1014 | " \n", 1015 | " \n", 1016 | " \n", 1017 | " \n", 1018 | " \n", 1019 | " \n", 1020 | " \n", 1021 | " \n", 1022 | " \n", 1023 | " \n", 1024 | " \n", 1025 | " \n", 1026 | " \n", 1027 | " \n", 1028 | " \n", 1029 | " \n", 1030 | " \n", 1031 | " \n", 1032 | " \n", 1033 | " \n", 1034 | " \n", 1035 | " \n", 1036 | " \n", 1037 | " \n", 1038 | " \n", 1039 | " \n", 1040 | " \n", 1041 | " \n", 1042 | " \n", 1043 | " \n", 1044 | " \n", 1045 | " \n", 1046 | " \n", 1047 | " \n", 1048 | " \n", 1049 | " \n", 1050 | " \n", 1051 | " \n", 1052 | " \n", 1053 | " \n", 1054 | " \n", 1055 | " \n", 1056 | " \n", 1057 | " \n", 1058 | " \n", 1059 | " \n", 1060 | " \n", 1061 | " \n", 1062 | " \n", 1063 | " \n", 1064 | " \n", 1065 | " \n", 1066 | " \n", 1067 | " \n", 1068 | " \n", 1069 | " \n", 1070 | " \n", 1071 | " \n", 1072 | " \n", 1073 | " \n", 1074 | " \n", 1075 | " \n", 1076 | " \n", 1077 | " \n", 1078 | " \n", 1079 | " \n", 1080 | " \n", 1081 | " \n", 1082 | " \n", 1083 | " \n", 1084 | " \n", 1085 | " \n", 1086 | " \n", 1087 | " \n", 1088 | " \n", 1089 | " \n", 1090 | " \n", 1091 | " \n", 1092 | " \n", 1093 | " \n", 1094 | " \n", 1095 | " \n", 1096 | " \n", 1097 | " \n", 1098 | " \n", 1099 | " \n", 1100 | " \n", 1101 | " \n", 1102 | " \n", 1103 | "
GPUMemory UsedMemory Used PercentMemory UtilizationPowerTemperatureUtilizationtimestamp
2018-04-02 15:00:00.452204032+00:000868535500836.149883709472839902018-04-02 15:00:00.452204
2018-04-02 15:00:00.499869184+00:001644769382426.836367565766442732018-04-02 15:00:00.499869
2018-04-02 15:00:00.547173120+00:002850709708835.407944555782738682018-04-02 15:00:00.547173
2018-04-02 15:00:00.595969024+00:003294754713612.268178386206836732018-04-02 15:00:00.595969
2018-04-02 15:00:02.519879168+00:000993106329641.3347260572263602018-04-02 15:00:02.519879
2018-04-02 15:00:02.565956096+00:001769340211232.0212110560274002018-04-02 15:00:02.565956
2018-04-02 15:00:02.615364096+00:002975280537640.5927870565723602018-04-02 15:00:02.615364
2018-04-02 15:00:02.661623040+00:003364170444815.15737611850433572018-04-02 15:00:02.661623
2018-04-02 15:00:04.063194112+00:000368784179222.4283770697603602018-04-02 15:00:04.063194
2018-04-02 15:00:04.117285120+00:001406742630416.9293000570863902018-04-02 15:00:04.117285
2018-04-02 15:00:04.170159104+00:002453928550418.89325631628723542018-04-02 15:00:04.170159
2018-04-02 15:00:04.219676928+00:003369832755215.3930500590183502018-04-02 15:00:04.219677
2018-04-02 15:00:05.273978880+00:000423729561617.636324605809137942018-04-02 15:00:05.273979
2018-04-02 15:00:05.316706048+00:00111901337604.953533515631640912018-04-02 15:00:05.316706
2018-04-02 15:00:05.358547968+00:00211943280644.97099165708636412018-04-02 15:00:05.358548
2018-04-02 15:00:05.401321984+00:00312006195204.9971770566523502018-04-02 15:00:05.401322
2018-04-02 15:00:06.460380160+00:0001289014476853.6509125414917938922018-04-02 15:00:06.460380
2018-04-02 15:00:06.509203968+00:0011065248358444.3373977911511341892018-04-02 15:00:06.509204
2018-04-02 15:00:06.561969152+00:0021065667788844.354854846016837892018-04-02 15:00:06.561969
2018-04-02 15:00:06.611894016+00:0031066296934444.381040585979336622018-04-02 15:00:06.611894
2018-04-02 15:00:07.817122048+00:000766823628831.91646685767436242018-04-02 15:00:07.817122
2018-04-02 15:00:07.872345856+00:001401709465616.71981266282540442018-04-02 15:00:07.872346
2018-04-02 15:00:07.927056128+00:002543476940822.6204081457944361002018-04-02 15:00:07.927056
2018-04-02 15:00:07.973288960+00:0031066296934444.38104040138665361002018-04-02 15:00:07.973289
2018-04-02 15:00:09.074198016+00:0001488663347261.9606282419522037582018-04-02 15:00:09.074198
2018-04-02 15:00:09.132369152+00:001684405555228.4860902414618340262018-04-02 15:00:09.132369
2018-04-02 15:00:10.191634944+00:0021406664704058.547709225593136212018-04-02 15:00:10.191635
2018-04-02 15:00:10.257611008+00:003862244044835.8880220564553402018-04-02 15:00:10.257611
2018-04-02 15:00:11.995235072+00:000978845696040.7411750574153502018-04-02 15:00:11.995235
2018-04-02 15:00:12.071012096+00:001861195468835.844378145660439152018-04-02 15:00:12.071012
...........................
2018-04-02 15:00:28.550659840+00:002387239116816.1175321588203622018-04-02 15:00:28.550660
2018-04-02 15:00:28.607621888+00:003367525888015.2970350765063502018-04-02 15:00:28.607622
2018-04-02 15:00:29.683577088+00:000359976140814.982802435808937862018-04-02 15:00:29.683577
2018-04-02 15:00:29.725943040+00:00111901337604.953533405621940922018-04-02 15:00:29.725943
2018-04-02 15:00:29.766936064+00:00211943280644.97099145727837242018-04-02 15:00:29.766936
2018-04-02 15:00:29.810834944+00:00312006195204.99717705684236902018-04-02 15:00:29.810835
2018-04-02 15:00:30.872054016+00:000359976140814.9828020575113502018-04-02 15:00:30.872054
2018-04-02 15:00:30.914840064+00:00111901337604.9535330561233902018-04-02 15:00:30.914840
2018-04-02 15:00:30.957829888+00:00211943280644.9709910569903502018-04-02 15:00:30.957830
2018-04-02 15:00:31.000922880+00:00312006195204.9971770565483402018-04-02 15:00:31.000923
2018-04-02 15:00:32.136954112+00:000359976140814.9828020574153502018-04-02 15:00:32.136954
2018-04-02 15:00:32.231229184+00:00111901337604.9535330558973902018-04-02 15:00:32.231229
2018-04-02 15:00:33.769680896+00:00211943280644.9709910514983402018-04-02 15:00:33.769681
2018-04-02 15:00:33.813342208+00:00312006195204.9971770510573302018-04-02 15:00:33.813342
2018-04-02 15:00:34.890827776+00:000359976140814.98280201039973402018-04-02 15:00:34.890828
2018-04-02 15:00:34.940800+00:00112425625605.1717510515653852018-04-02 15:00:34.940800
2018-04-02 15:00:35.006163968+00:00211943280644.9709910567593402018-04-02 15:00:35.006164
2018-04-02 15:00:35.070588160+00:00312006195204.9971771512193312018-04-02 15:00:35.070588
2018-04-02 15:00:36.154573056+00:000933547212838.8557771520213472018-04-02 15:00:36.154573
2018-04-02 15:00:36.199578880+00:001743964672030.9650393506333832018-04-02 15:00:36.199579
2018-04-02 15:00:36.249982208+00:002878392115236.56013195159534102018-04-02 15:00:36.249982
2018-04-02 15:00:36.296148992+00:003902719078437.5726602512553342018-04-02 15:00:36.296149
2018-04-02 15:00:39.521494016+00:0001011351552042.0941234615425336552018-04-02 15:00:39.521494
2018-04-02 15:00:39.565041152+00:001988912025641.1601522910198940382018-04-02 15:00:39.565041
2018-04-02 15:00:39.608422144+00:002988492595241.1426952515360936342018-04-02 15:00:39.608422
2018-04-02 15:00:39.650899968+00:003988492595241.1426955515918636622018-04-02 15:00:39.650900
2018-04-02 15:00:40.716825856+00:0001011351552042.0941231615673037212018-04-02 15:00:40.716826
2018-04-02 15:00:40.765632+00:001988912025641.16015201294664192018-04-02 15:00:40.765632
2018-04-02 15:00:40.807444992+00:002988492595241.1426957515707837872018-04-02 15:00:40.807445
2018-04-02 15:00:40.849585920+00:003988492595241.1426957714747536922018-04-02 15:00:40.849586
\n", 1104 | "

100 rows × 8 columns

\n", 1105 | "
" 1106 | ], 1107 | "text/plain": [ 1108 | " GPU Memory Used Memory Used Percent \\\n", 1109 | "2018-04-02 15:00:00.452204032+00:00 0 8685355008 36.149883 \n", 1110 | "2018-04-02 15:00:00.499869184+00:00 1 6447693824 26.836367 \n", 1111 | "2018-04-02 15:00:00.547173120+00:00 2 8507097088 35.407944 \n", 1112 | "2018-04-02 15:00:00.595969024+00:00 3 2947547136 12.268178 \n", 1113 | "2018-04-02 15:00:02.519879168+00:00 0 9931063296 41.334726 \n", 1114 | "2018-04-02 15:00:02.565956096+00:00 1 7693402112 32.021211 \n", 1115 | "2018-04-02 15:00:02.615364096+00:00 2 9752805376 40.592787 \n", 1116 | "2018-04-02 15:00:02.661623040+00:00 3 3641704448 15.157376 \n", 1117 | "2018-04-02 15:00:04.063194112+00:00 0 3687841792 22.428377 \n", 1118 | "2018-04-02 15:00:04.117285120+00:00 1 4067426304 16.929300 \n", 1119 | "2018-04-02 15:00:04.170159104+00:00 2 4539285504 18.893256 \n", 1120 | "2018-04-02 15:00:04.219676928+00:00 3 3698327552 15.393050 \n", 1121 | "2018-04-02 15:00:05.273978880+00:00 0 4237295616 17.636324 \n", 1122 | "2018-04-02 15:00:05.316706048+00:00 1 1190133760 4.953533 \n", 1123 | "2018-04-02 15:00:05.358547968+00:00 2 1194328064 4.970991 \n", 1124 | "2018-04-02 15:00:05.401321984+00:00 3 1200619520 4.997177 \n", 1125 | "2018-04-02 15:00:06.460380160+00:00 0 12890144768 53.650912 \n", 1126 | "2018-04-02 15:00:06.509203968+00:00 1 10652483584 44.337397 \n", 1127 | "2018-04-02 15:00:06.561969152+00:00 2 10656677888 44.354854 \n", 1128 | "2018-04-02 15:00:06.611894016+00:00 3 10662969344 44.381040 \n", 1129 | "2018-04-02 15:00:07.817122048+00:00 0 7668236288 31.916466 \n", 1130 | "2018-04-02 15:00:07.872345856+00:00 1 4017094656 16.719812 \n", 1131 | "2018-04-02 15:00:07.927056128+00:00 2 5434769408 22.620408 \n", 1132 | "2018-04-02 15:00:07.973288960+00:00 3 10662969344 44.381040 \n", 1133 | "2018-04-02 15:00:09.074198016+00:00 0 14886633472 61.960628 \n", 1134 | "2018-04-02 15:00:09.132369152+00:00 1 6844055552 28.486090 \n", 1135 | "2018-04-02 15:00:10.191634944+00:00 2 14066647040 58.547709 \n", 1136 | "2018-04-02 15:00:10.257611008+00:00 3 8622440448 35.888022 \n", 1137 | "2018-04-02 15:00:11.995235072+00:00 0 9788456960 40.741175 \n", 1138 | "2018-04-02 15:00:12.071012096+00:00 1 8611954688 35.844378 \n", 1139 | "... .. ... ... \n", 1140 | "2018-04-02 15:00:28.550659840+00:00 2 3872391168 16.117532 \n", 1141 | "2018-04-02 15:00:28.607621888+00:00 3 3675258880 15.297035 \n", 1142 | "2018-04-02 15:00:29.683577088+00:00 0 3599761408 14.982802 \n", 1143 | "2018-04-02 15:00:29.725943040+00:00 1 1190133760 4.953533 \n", 1144 | "2018-04-02 15:00:29.766936064+00:00 2 1194328064 4.970991 \n", 1145 | "2018-04-02 15:00:29.810834944+00:00 3 1200619520 4.997177 \n", 1146 | "2018-04-02 15:00:30.872054016+00:00 0 3599761408 14.982802 \n", 1147 | "2018-04-02 15:00:30.914840064+00:00 1 1190133760 4.953533 \n", 1148 | "2018-04-02 15:00:30.957829888+00:00 2 1194328064 4.970991 \n", 1149 | "2018-04-02 15:00:31.000922880+00:00 3 1200619520 4.997177 \n", 1150 | "2018-04-02 15:00:32.136954112+00:00 0 3599761408 14.982802 \n", 1151 | "2018-04-02 15:00:32.231229184+00:00 1 1190133760 4.953533 \n", 1152 | "2018-04-02 15:00:33.769680896+00:00 2 1194328064 4.970991 \n", 1153 | "2018-04-02 15:00:33.813342208+00:00 3 1200619520 4.997177 \n", 1154 | "2018-04-02 15:00:34.890827776+00:00 0 3599761408 14.982802 \n", 1155 | "2018-04-02 15:00:34.940800+00:00 1 1242562560 5.171751 \n", 1156 | "2018-04-02 15:00:35.006163968+00:00 2 1194328064 4.970991 \n", 1157 | "2018-04-02 15:00:35.070588160+00:00 3 1200619520 4.997177 \n", 1158 | "2018-04-02 15:00:36.154573056+00:00 0 9335472128 38.855777 \n", 1159 | "2018-04-02 15:00:36.199578880+00:00 1 7439646720 30.965039 \n", 1160 | "2018-04-02 15:00:36.249982208+00:00 2 8783921152 36.560131 \n", 1161 | "2018-04-02 15:00:36.296148992+00:00 3 9027190784 37.572660 \n", 1162 | "2018-04-02 15:00:39.521494016+00:00 0 10113515520 42.094123 \n", 1163 | "2018-04-02 15:00:39.565041152+00:00 1 9889120256 41.160152 \n", 1164 | "2018-04-02 15:00:39.608422144+00:00 2 9884925952 41.142695 \n", 1165 | "2018-04-02 15:00:39.650899968+00:00 3 9884925952 41.142695 \n", 1166 | "2018-04-02 15:00:40.716825856+00:00 0 10113515520 42.094123 \n", 1167 | "2018-04-02 15:00:40.765632+00:00 1 9889120256 41.160152 \n", 1168 | "2018-04-02 15:00:40.807444992+00:00 2 9884925952 41.142695 \n", 1169 | "2018-04-02 15:00:40.849585920+00:00 3 9884925952 41.142695 \n", 1170 | "\n", 1171 | " Memory Utilization Power Temperature \\\n", 1172 | "2018-04-02 15:00:00.452204032+00:00 70 94728 39 \n", 1173 | "2018-04-02 15:00:00.499869184+00:00 56 57664 42 \n", 1174 | "2018-04-02 15:00:00.547173120+00:00 55 57827 38 \n", 1175 | "2018-04-02 15:00:00.595969024+00:00 38 62068 36 \n", 1176 | "2018-04-02 15:00:02.519879168+00:00 0 57226 36 \n", 1177 | "2018-04-02 15:00:02.565956096+00:00 0 56027 40 \n", 1178 | "2018-04-02 15:00:02.615364096+00:00 0 56572 36 \n", 1179 | "2018-04-02 15:00:02.661623040+00:00 1 185043 35 \n", 1180 | "2018-04-02 15:00:04.063194112+00:00 0 69760 36 \n", 1181 | "2018-04-02 15:00:04.117285120+00:00 0 57086 39 \n", 1182 | "2018-04-02 15:00:04.170159104+00:00 3 162872 35 \n", 1183 | "2018-04-02 15:00:04.219676928+00:00 0 59018 35 \n", 1184 | "2018-04-02 15:00:05.273978880+00:00 60 58091 37 \n", 1185 | "2018-04-02 15:00:05.316706048+00:00 51 56316 40 \n", 1186 | "2018-04-02 15:00:05.358547968+00:00 6 57086 36 \n", 1187 | "2018-04-02 15:00:05.401321984+00:00 0 56652 35 \n", 1188 | "2018-04-02 15:00:06.460380160+00:00 54 149179 38 \n", 1189 | "2018-04-02 15:00:06.509203968+00:00 79 115113 41 \n", 1190 | "2018-04-02 15:00:06.561969152+00:00 84 60168 37 \n", 1191 | "2018-04-02 15:00:06.611894016+00:00 58 59793 36 \n", 1192 | "2018-04-02 15:00:07.817122048+00:00 8 57674 36 \n", 1193 | "2018-04-02 15:00:07.872345856+00:00 6 62825 40 \n", 1194 | "2018-04-02 15:00:07.927056128+00:00 14 57944 36 \n", 1195 | "2018-04-02 15:00:07.973288960+00:00 40 138665 36 \n", 1196 | "2018-04-02 15:00:09.074198016+00:00 24 195220 37 \n", 1197 | "2018-04-02 15:00:09.132369152+00:00 24 146183 40 \n", 1198 | "2018-04-02 15:00:10.191634944+00:00 22 55931 36 \n", 1199 | "2018-04-02 15:00:10.257611008+00:00 0 56455 34 \n", 1200 | "2018-04-02 15:00:11.995235072+00:00 0 57415 35 \n", 1201 | "2018-04-02 15:00:12.071012096+00:00 14 56604 39 \n", 1202 | "... ... ... ... \n", 1203 | "2018-04-02 15:00:28.550659840+00:00 1 58820 36 \n", 1204 | "2018-04-02 15:00:28.607621888+00:00 0 76506 35 \n", 1205 | "2018-04-02 15:00:29.683577088+00:00 43 58089 37 \n", 1206 | "2018-04-02 15:00:29.725943040+00:00 40 56219 40 \n", 1207 | "2018-04-02 15:00:29.766936064+00:00 4 57278 37 \n", 1208 | "2018-04-02 15:00:29.810834944+00:00 0 56842 36 \n", 1209 | "2018-04-02 15:00:30.872054016+00:00 0 57511 35 \n", 1210 | "2018-04-02 15:00:30.914840064+00:00 0 56123 39 \n", 1211 | "2018-04-02 15:00:30.957829888+00:00 0 56990 35 \n", 1212 | "2018-04-02 15:00:31.000922880+00:00 0 56548 34 \n", 1213 | "2018-04-02 15:00:32.136954112+00:00 0 57415 35 \n", 1214 | "2018-04-02 15:00:32.231229184+00:00 0 55897 39 \n", 1215 | "2018-04-02 15:00:33.769680896+00:00 0 51498 34 \n", 1216 | "2018-04-02 15:00:33.813342208+00:00 0 51057 33 \n", 1217 | "2018-04-02 15:00:34.890827776+00:00 0 103997 34 \n", 1218 | "2018-04-02 15:00:34.940800+00:00 0 51565 38 \n", 1219 | "2018-04-02 15:00:35.006163968+00:00 0 56759 34 \n", 1220 | "2018-04-02 15:00:35.070588160+00:00 1 51219 33 \n", 1221 | "2018-04-02 15:00:36.154573056+00:00 1 52021 34 \n", 1222 | "2018-04-02 15:00:36.199578880+00:00 3 50633 38 \n", 1223 | "2018-04-02 15:00:36.249982208+00:00 9 51595 34 \n", 1224 | "2018-04-02 15:00:36.296148992+00:00 2 51255 33 \n", 1225 | "2018-04-02 15:00:39.521494016+00:00 46 154253 36 \n", 1226 | "2018-04-02 15:00:39.565041152+00:00 29 101989 40 \n", 1227 | "2018-04-02 15:00:39.608422144+00:00 25 153609 36 \n", 1228 | "2018-04-02 15:00:39.650899968+00:00 55 159186 36 \n", 1229 | "2018-04-02 15:00:40.716825856+00:00 16 156730 37 \n", 1230 | "2018-04-02 15:00:40.765632+00:00 0 129466 41 \n", 1231 | "2018-04-02 15:00:40.807444992+00:00 75 157078 37 \n", 1232 | "2018-04-02 15:00:40.849585920+00:00 77 147475 36 \n", 1233 | "\n", 1234 | " Utilization timestamp \n", 1235 | "2018-04-02 15:00:00.452204032+00:00 90 2018-04-02 15:00:00.452204 \n", 1236 | "2018-04-02 15:00:00.499869184+00:00 73 2018-04-02 15:00:00.499869 \n", 1237 | "2018-04-02 15:00:00.547173120+00:00 68 2018-04-02 15:00:00.547173 \n", 1238 | "2018-04-02 15:00:00.595969024+00:00 73 2018-04-02 15:00:00.595969 \n", 1239 | "2018-04-02 15:00:02.519879168+00:00 0 2018-04-02 15:00:02.519879 \n", 1240 | "2018-04-02 15:00:02.565956096+00:00 0 2018-04-02 15:00:02.565956 \n", 1241 | "2018-04-02 15:00:02.615364096+00:00 0 2018-04-02 15:00:02.615364 \n", 1242 | "2018-04-02 15:00:02.661623040+00:00 7 2018-04-02 15:00:02.661623 \n", 1243 | "2018-04-02 15:00:04.063194112+00:00 0 2018-04-02 15:00:04.063194 \n", 1244 | "2018-04-02 15:00:04.117285120+00:00 0 2018-04-02 15:00:04.117285 \n", 1245 | "2018-04-02 15:00:04.170159104+00:00 4 2018-04-02 15:00:04.170159 \n", 1246 | "2018-04-02 15:00:04.219676928+00:00 0 2018-04-02 15:00:04.219677 \n", 1247 | "2018-04-02 15:00:05.273978880+00:00 94 2018-04-02 15:00:05.273979 \n", 1248 | "2018-04-02 15:00:05.316706048+00:00 91 2018-04-02 15:00:05.316706 \n", 1249 | "2018-04-02 15:00:05.358547968+00:00 41 2018-04-02 15:00:05.358548 \n", 1250 | "2018-04-02 15:00:05.401321984+00:00 0 2018-04-02 15:00:05.401322 \n", 1251 | "2018-04-02 15:00:06.460380160+00:00 92 2018-04-02 15:00:06.460380 \n", 1252 | "2018-04-02 15:00:06.509203968+00:00 89 2018-04-02 15:00:06.509204 \n", 1253 | "2018-04-02 15:00:06.561969152+00:00 89 2018-04-02 15:00:06.561969 \n", 1254 | "2018-04-02 15:00:06.611894016+00:00 62 2018-04-02 15:00:06.611894 \n", 1255 | "2018-04-02 15:00:07.817122048+00:00 24 2018-04-02 15:00:07.817122 \n", 1256 | "2018-04-02 15:00:07.872345856+00:00 44 2018-04-02 15:00:07.872346 \n", 1257 | "2018-04-02 15:00:07.927056128+00:00 100 2018-04-02 15:00:07.927056 \n", 1258 | "2018-04-02 15:00:07.973288960+00:00 100 2018-04-02 15:00:07.973289 \n", 1259 | "2018-04-02 15:00:09.074198016+00:00 58 2018-04-02 15:00:09.074198 \n", 1260 | "2018-04-02 15:00:09.132369152+00:00 26 2018-04-02 15:00:09.132369 \n", 1261 | "2018-04-02 15:00:10.191634944+00:00 21 2018-04-02 15:00:10.191635 \n", 1262 | "2018-04-02 15:00:10.257611008+00:00 0 2018-04-02 15:00:10.257611 \n", 1263 | "2018-04-02 15:00:11.995235072+00:00 0 2018-04-02 15:00:11.995235 \n", 1264 | "2018-04-02 15:00:12.071012096+00:00 15 2018-04-02 15:00:12.071012 \n", 1265 | "... ... ... \n", 1266 | "2018-04-02 15:00:28.550659840+00:00 2 2018-04-02 15:00:28.550660 \n", 1267 | "2018-04-02 15:00:28.607621888+00:00 0 2018-04-02 15:00:28.607622 \n", 1268 | "2018-04-02 15:00:29.683577088+00:00 86 2018-04-02 15:00:29.683577 \n", 1269 | "2018-04-02 15:00:29.725943040+00:00 92 2018-04-02 15:00:29.725943 \n", 1270 | "2018-04-02 15:00:29.766936064+00:00 24 2018-04-02 15:00:29.766936 \n", 1271 | "2018-04-02 15:00:29.810834944+00:00 90 2018-04-02 15:00:29.810835 \n", 1272 | "2018-04-02 15:00:30.872054016+00:00 0 2018-04-02 15:00:30.872054 \n", 1273 | "2018-04-02 15:00:30.914840064+00:00 0 2018-04-02 15:00:30.914840 \n", 1274 | "2018-04-02 15:00:30.957829888+00:00 0 2018-04-02 15:00:30.957830 \n", 1275 | "2018-04-02 15:00:31.000922880+00:00 0 2018-04-02 15:00:31.000923 \n", 1276 | "2018-04-02 15:00:32.136954112+00:00 0 2018-04-02 15:00:32.136954 \n", 1277 | "2018-04-02 15:00:32.231229184+00:00 0 2018-04-02 15:00:32.231229 \n", 1278 | "2018-04-02 15:00:33.769680896+00:00 0 2018-04-02 15:00:33.769681 \n", 1279 | "2018-04-02 15:00:33.813342208+00:00 0 2018-04-02 15:00:33.813342 \n", 1280 | "2018-04-02 15:00:34.890827776+00:00 0 2018-04-02 15:00:34.890828 \n", 1281 | "2018-04-02 15:00:34.940800+00:00 5 2018-04-02 15:00:34.940800 \n", 1282 | "2018-04-02 15:00:35.006163968+00:00 0 2018-04-02 15:00:35.006164 \n", 1283 | "2018-04-02 15:00:35.070588160+00:00 1 2018-04-02 15:00:35.070588 \n", 1284 | "2018-04-02 15:00:36.154573056+00:00 7 2018-04-02 15:00:36.154573 \n", 1285 | "2018-04-02 15:00:36.199578880+00:00 3 2018-04-02 15:00:36.199579 \n", 1286 | "2018-04-02 15:00:36.249982208+00:00 10 2018-04-02 15:00:36.249982 \n", 1287 | "2018-04-02 15:00:36.296148992+00:00 4 2018-04-02 15:00:36.296149 \n", 1288 | "2018-04-02 15:00:39.521494016+00:00 55 2018-04-02 15:00:39.521494 \n", 1289 | "2018-04-02 15:00:39.565041152+00:00 38 2018-04-02 15:00:39.565041 \n", 1290 | "2018-04-02 15:00:39.608422144+00:00 34 2018-04-02 15:00:39.608422 \n", 1291 | "2018-04-02 15:00:39.650899968+00:00 62 2018-04-02 15:00:39.650900 \n", 1292 | "2018-04-02 15:00:40.716825856+00:00 21 2018-04-02 15:00:40.716826 \n", 1293 | "2018-04-02 15:00:40.765632+00:00 9 2018-04-02 15:00:40.765632 \n", 1294 | "2018-04-02 15:00:40.807444992+00:00 87 2018-04-02 15:00:40.807445 \n", 1295 | "2018-04-02 15:00:40.849585920+00:00 92 2018-04-02 15:00:40.849586 \n", 1296 | "\n", 1297 | "[100 rows x 8 columns]" 1298 | ] 1299 | }, 1300 | "execution_count": 36, 1301 | "metadata": {}, 1302 | "output_type": "execute_result" 1303 | } 1304 | ], 1305 | "source": [ 1306 | "df.head(100)" 1307 | ] 1308 | }, 1309 | { 1310 | "cell_type": "code", 1311 | "execution_count": null, 1312 | "metadata": {}, 1313 | "outputs": [], 1314 | "source": [] 1315 | } 1316 | ], 1317 | "metadata": { 1318 | "kernelspec": { 1319 | "display_name": "Python 3", 1320 | "language": "python", 1321 | "name": "python3" 1322 | }, 1323 | "language_info": { 1324 | "codemirror_mode": { 1325 | "name": "ipython", 1326 | "version": 3 1327 | }, 1328 | "file_extension": ".py", 1329 | "mimetype": "text/x-python", 1330 | "name": "python", 1331 | "nbconvert_exporter": "python", 1332 | "pygments_lexer": "ipython3", 1333 | "version": "3.6.4" 1334 | } 1335 | }, 1336 | "nbformat": 4, 1337 | "nbformat_minor": 2 1338 | } 1339 | -------------------------------------------------------------------------------- /gpumon/__init__.py: -------------------------------------------------------------------------------- 1 | from gpumon.influxdb.gpu_interface import device_name, device_count 2 | -------------------------------------------------------------------------------- /gpumon/__version__.py: -------------------------------------------------------------------------------- 1 | __version__ = '0.0.2' -------------------------------------------------------------------------------- /gpumon/file/__init__.py: -------------------------------------------------------------------------------- 1 | from gpumon.file.gpu_logger import log_context 2 | -------------------------------------------------------------------------------- /gpumon/file/gpu_logger.py: -------------------------------------------------------------------------------- 1 | import subprocess 2 | from contextlib import contextmanager 3 | from datetime import datetime 4 | 5 | try: 6 | from itertools import filterfalse 7 | except ImportError: 8 | from itertools import ifilterfalse as filterfalse 9 | 10 | import pandas as pd 11 | from bokeh.models import ColumnDataSource 12 | from bokeh.palettes import Paired 13 | from bokeh.plotting import figure 14 | from toolz import curry, pipe 15 | import logging 16 | 17 | 18 | logger = logging.getLogger(__name__) 19 | 20 | 21 | convert_datetime = lambda x: datetime.strptime(x, '%Y%m%d') 22 | convert_time = lambda x: datetime.strptime(x, '%H:%M:%S').time() 23 | conversion_funcs = convert_datetime, convert_time, int, int, int, int, int, int, int, int, int 24 | gpu_headers = 'timestamp', 'gpu', 'pwr', 'temp', 'sm', 'mem', 'enc', 'dec', 'mclk', 'pclk' 25 | header_dict = dict(zip(range(len(gpu_headers)), gpu_headers)) 26 | 27 | 28 | def parse_line(line_string): 29 | parsed_list = list((func(val) for func, val in zip(conversion_funcs, line_string.split()))) 30 | logger.debug(parsed_list) 31 | return [datetime.combine(*parsed_list[:2])] + parsed_list[2:] 32 | 33 | 34 | def convert_to_df(msg_list): 35 | return (pd.DataFrame(msg_list) 36 | .rename(columns=header_dict) 37 | .dropna(how='any')) 38 | 39 | 40 | def parse_lines(msg_list): 41 | msg_list = filterfalse(lambda x: '#' in x, msg_list) 42 | new_list = [] 43 | for line in msg_list: 44 | try: 45 | new_list.append(parse_line(line)) 46 | except (ValueError, TypeError): 47 | logger.debug('Error parsing {}'.format(line)) 48 | return convert_to_df(new_list) 49 | 50 | 51 | def parse_log(filename): 52 | with open(filename) as f: 53 | return parse_lines(list(f)) 54 | 55 | 56 | @curry 57 | def extract(gpu_property, df): 58 | return (df.groupby(['timestamp', 'gpu'])[gpu_property] 59 | .first() 60 | .unstack(level=1) 61 | .ffill() 62 | .bfill() 63 | .rename(columns={i:'gpu {}'.format(i) for i in range(4)})) 64 | 65 | 66 | def plot(df, num_gpus=1, plot_width=600, plot_height=400, y_range=(0, 110)): 67 | """ 68 | """ 69 | data = ColumnDataSource(data=df) 70 | p = figure(plot_width=plot_width, plot_height=plot_height, y_range=y_range, x_axis_type="datetime") 71 | for gpu, color in zip(range(num_gpus), Paired[12]): 72 | p.line('timestamp', 73 | 'gpu {}'.format(gpu), 74 | line_width=4, 75 | source=data, 76 | color=color, 77 | legend="GPU {}".format(gpu)) 78 | return p 79 | 80 | 81 | class Logger(object): 82 | def __init__(self, log_file): 83 | self._log_file = log_file 84 | 85 | def __call__(self): 86 | return parse_log(self._log_file) 87 | 88 | def plot(self, gpu_measurement='sm', num_gpus=1, plot_width=600, plot_height=400, y_range=(0, 110)): 89 | """ Plot the specified GPU measurement 90 | 91 | Parameters 92 | ---------- 93 | gpu_measurement: GPU measurement to plot possible values 94 | num_gpus: Number of GPUs to plot ['pwr', 'temp', 'sm', 'mem', 'enc', 'dec', 'mclk', 'pclk'] 95 | plot_width: 96 | plot_height: 97 | y_range: 98 | 99 | Returns 100 | ------- 101 | Bokeh Figure 102 | """ 103 | df = pipe(self._log_file, 104 | parse_log, 105 | extract(gpu_measurement)) 106 | return plot(df, 107 | num_gpus=num_gpus, 108 | plot_width=plot_width, 109 | plot_height=plot_height, 110 | y_range=y_range) 111 | 112 | 113 | @contextmanager 114 | def log_context(log_file, interval_seconds=1): 115 | logger.info('Logging GPU in {}'.format(log_file)) 116 | process_args = ["nvidia-smi", 117 | "dmon", 118 | "-d", str(interval_seconds), 119 | "-o", "DT", 120 | "-f", log_file] 121 | 122 | with subprocess.Popen(process_args, stdout=subprocess.PIPE) as proc: 123 | yield Logger(log_file) 124 | proc.terminate() 125 | 126 | -------------------------------------------------------------------------------- /gpumon/gpumon: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import logging 4 | logging.basicConfig(level=logging.INFO) 5 | 6 | import fire 7 | 8 | from gpumon.influxdb.dotenv import populate_args_from_dotenv 9 | from gpumon.influxdb.gpu_logger import start_logger, MEASUREMENTS_RETENTION_DURATION, MetricsRecordingFailed 10 | 11 | 12 | def _logger(): 13 | return logging.getLogger(__name__) 14 | 15 | 16 | def _log_process(ip_or_url, 17 | username, 18 | password, 19 | database, 20 | port=8086, 21 | series_name='gpu_measurements', 22 | polling_interval=1, 23 | debug=False, 24 | retention_duration=MEASUREMENTS_RETENTION_DURATION, 25 | **tags): 26 | """ Starts GPU logger 27 | 28 | Logs GPU measurements to an influxdb database 29 | 30 | 31 | Parameters 32 | ---------- 33 | ip_or_url: ip or url of influxdb 34 | username: Username to log into influxdb database 35 | password: Password to log into influxdb database 36 | database: Name of database to log data to. It will create the database if one doesn't exist 37 | port: A number indicating the port on which influxdb is listening. default:8086 38 | series_name: Name of series/table to log data to 39 | polling_interval: polling interval for measurements in seconds. default:1 40 | debug: whether to escalate logging to debug 41 | retention_duration: the duration to retain the measurements for valid values are 1h, 90m, 12h, 7d, and 4w. default:1d 42 | tags: One or more tags to apply to the data. These can then be used to group or select timeseries 43 | Example: --machine my_machine --cluster kerb01 44 | 45 | Example 46 | ------- 47 | gpumon localhost 8086 username password gpudata --machine=my_gpu_machine 48 | 49 | """ 50 | _logger().info(debug) 51 | if bool(debug): 52 | logging.basicConfig(level=logging.DEBUG) 53 | _logger().debug('Debug logging | ON') 54 | 55 | stop_logging=None 56 | try: 57 | t, stop_logging = start_logger(ip_or_url, 58 | username, 59 | password, 60 | database, 61 | port=port, 62 | series_name=series_name, 63 | polling_interval=polling_interval, 64 | retention_duration=retention_duration, 65 | **tags) 66 | t.join() 67 | except KeyboardInterrupt: 68 | logger =_logger() 69 | logger.info("Cancelling") 70 | logger.info("Waiting for GPU call to finish....") 71 | except MetricsRecordingFailed: 72 | logger = _logger() 73 | logger.warning("Failed to start logging due to issue connecting to InfluxDB") 74 | logger.info("Waiting for GPU call to finish....") 75 | finally: 76 | if stop_logging is not None: 77 | stop_logging() 78 | 79 | if __name__=="__main__": 80 | fire.Fire(populate_args_from_dotenv(_log_process)) -------------------------------------------------------------------------------- /gpumon/influxdb/__init__.py: -------------------------------------------------------------------------------- 1 | from gpumon.influxdb.gpu_logger import log_context 2 | -------------------------------------------------------------------------------- /gpumon/influxdb/dotenv.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from functools import partial 3 | 4 | from dotenv import find_dotenv, dotenv_values 5 | 6 | 7 | def _logger(): 8 | return logging.getLogger(__name__) 9 | 10 | def populate_args_from_dotenv(func): 11 | logger = _logger() 12 | try: 13 | dotenv_path = find_dotenv(raise_error_if_not_found=True) 14 | logger.info('Found .evn, loading variables') 15 | env_dict = dotenv_values(dotenv_path=dotenv_path) 16 | par_func = partial(func, **env_dict) 17 | par_func.__doc__ = func.__doc__ 18 | return par_func 19 | except IOError: 20 | logger.info('Didn\'t find .env') 21 | return func -------------------------------------------------------------------------------- /gpumon/influxdb/gpu_interface.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import logging 3 | from concurrent.futures import CancelledError 4 | from contextlib import contextmanager 5 | from datetime import datetime 6 | from threading import Thread 7 | 8 | import pynvml 9 | from toolz.functoolz import compose 10 | 11 | 12 | def _logger(): 13 | return logging.getLogger(__name__) 14 | 15 | 16 | def nativestr(s): 17 | if isinstance(s, str): 18 | return s 19 | return s.decode('utf-8', 'replace') 20 | 21 | 22 | @contextmanager 23 | def pynvml_context(): 24 | pynvml.nvmlInit() 25 | yield 26 | pynvml.nvmlShutdown() 27 | 28 | 29 | def device_count(): 30 | with pynvml_context(): 31 | deviceCount = device_count_for() 32 | return deviceCount 33 | 34 | 35 | def device_name(): 36 | with pynvml_context(): 37 | device_name = device_name_for(pynvml.nvmlDeviceGetHandleByIndex(0)) 38 | return device_name 39 | 40 | 41 | def device_count_for(): 42 | try: 43 | return pynvml.nvmlDeviceGetCount() 44 | except pynvml.NVMlError: 45 | return None 46 | 47 | 48 | def device_name_for(device_handle): 49 | """Get GPU device name""" 50 | try: 51 | return nativestr(pynvml.nvmlDeviceGetName(device_handle)) 52 | except pynvml.NVMlError: 53 | return "NVIDIA" 54 | 55 | 56 | def mem_used_for(device_handle): 57 | """Get GPU device memory consumption in percent""" 58 | try: 59 | return pynvml.nvmlDeviceGetMemoryInfo(device_handle).used 60 | except pynvml.NVMLError: 61 | return None 62 | 63 | 64 | def mem_used_percent_for(device_handle): 65 | """Get GPU device memory consumption in percent""" 66 | try: 67 | memory_info = pynvml.nvmlDeviceGetMemoryInfo(device_handle) 68 | return memory_info.used * 100.0 / memory_info.total 69 | except pynvml.NVMLError: 70 | return None 71 | 72 | 73 | def utilization_for(device_handle): 74 | """Get GPU device consumption in percent 75 | Percent of time over the past sample period during which one or more kernels was executing on the GPU. 76 | """ 77 | try: 78 | return pynvml.nvmlDeviceGetUtilizationRates(device_handle).gpu 79 | except pynvml.NVMLError: 80 | return None 81 | 82 | 83 | def mem_utilization_for(device_handle): 84 | """ 85 | Percent of time over the past sample period during which global (device) memory was being read or written. 86 | """ 87 | try: 88 | return pynvml.nvmlDeviceGetUtilizationRates(device_handle).memory 89 | except pynvml.NVMLError: 90 | return None 91 | 92 | 93 | def power_for(device_handle): 94 | try: 95 | return pynvml.nvmlDeviceGetPowerUsage(device_handle) 96 | except pynvml.NVMLError: 97 | return None 98 | 99 | 100 | def temperature_for(device_handle): 101 | try: 102 | return pynvml.nvmlDeviceGetTemperature(device_handle, 0) 103 | except pynvml.NVMLError: 104 | return None 105 | 106 | 107 | _MEASUREMENTS_FUNCS = { 108 | "Memory Used": mem_used_for, 109 | "Memory Used Percent": mem_used_percent_for, 110 | "Utilization": utilization_for, 111 | "Memory Utilization": mem_utilization_for, 112 | "Power": power_for, 113 | "Temperature": temperature_for, 114 | } 115 | 116 | 117 | def measurements_for(gpu_handle): 118 | mes_dict = {k: func(gpu_handle) for k, func in _MEASUREMENTS_FUNCS.items()} 119 | mes_dict['timestamp'] = str(datetime.now()) 120 | return mes_dict 121 | 122 | 123 | async def aggregate_measurements(device_count): 124 | measures_for_device = compose(measurements_for, 125 | pynvml.nvmlDeviceGetHandleByIndex) 126 | return {i: measures_for_device(i) for i in range(device_count)} 127 | 128 | 129 | async def record_measurements_to(async_reporting_func, polling_interval=1): 130 | try: 131 | deviceCount = pynvml.nvmlDeviceGetCount() 132 | while True: 133 | measurement = await aggregate_measurements(deviceCount) 134 | await async_reporting_func(measurement) 135 | await asyncio.sleep(polling_interval) 136 | except CancelledError: 137 | _logger().info("Logging cancelled") 138 | 139 | 140 | def async_function_from(output_function): 141 | async def async_output_function(measurement): 142 | output_function(measurement) 143 | 144 | return async_output_function 145 | 146 | 147 | def run_logging_loop(async_task, async_loop): 148 | asyncio.set_event_loop(async_loop) 149 | pynvml.nvmlInit() 150 | logger = _logger() 151 | logger.info("Driver Version: {}".format(nativestr(pynvml.nvmlSystemGetDriverVersion()))) 152 | async_loop.run_until_complete(async_task) 153 | logger.info("Shutting down driver") 154 | pynvml.nvmlShutdown() 155 | 156 | 157 | def start_pushing_measurements_to(output_function, polling_interval=1): 158 | new_loop = asyncio.new_event_loop() 159 | task = new_loop.create_task(record_measurements_to(async_function_from(output_function), 160 | polling_interval=polling_interval)) 161 | t = Thread(target=run_logging_loop, args=(task, new_loop)) 162 | t.start() 163 | 164 | def stop_logging(): 165 | task.cancel() 166 | 167 | return t, stop_logging 168 | -------------------------------------------------------------------------------- /gpumon/influxdb/gpu_logger.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from contextlib import contextmanager 3 | from multiprocessing import Process 4 | from time import sleep 5 | 6 | from influxdb import InfluxDBClient 7 | from influxdb.exceptions import InfluxDBClientError 8 | from requests.exceptions import ConnectionError 9 | from toolz import curry, compose 10 | 11 | from gpumon.influxdb.dotenv import populate_args_from_dotenv 12 | from gpumon.influxdb.gpu_interface import start_pushing_measurements_to 13 | 14 | MEASUREMENTS_RETENTION_DURATION = '1d' 15 | 16 | 17 | def _logger(): 18 | return logging.getLogger(__name__) 19 | 20 | 21 | def _switch_to_database(influxdb_client, database_name): 22 | dbs = influxdb_client.get_list_database() 23 | if database_name in [db['name'] for db in dbs]: 24 | _logger().info('Database {} exists'.format(database_name)) 25 | else: 26 | _logger().info('Creating Database {}'.format(database_name)) 27 | influxdb_client.create_database(database_name) 28 | influxdb_client.switch_database(database_name) 29 | 30 | 31 | def _compose_measurement_dict(gpu_num, gpu_dict, series_name): 32 | return {"measurement": series_name, 33 | "tags": {'GPU': gpu_num}, 34 | "time": gpu_dict['timestamp'], 35 | "fields": gpu_dict} 36 | 37 | 38 | @curry 39 | def _gpu_to_influxdb_format(series_name, gpu_dict): 40 | return [_compose_measurement_dict(gpu, gpu_dict[gpu], series_name) for gpu in gpu_dict] 41 | 42 | 43 | def _create_influxdb_writer(influxdb_client, tags): 44 | """ Returns function which writes to influxdb 45 | 46 | Parameters 47 | ---------- 48 | influxdb_client: 49 | """ 50 | 51 | def to_influxdf(data_list, retries=5, pause=5): 52 | logger = _logger() 53 | logger.debug(data_list) 54 | for i in range(retries): 55 | try: 56 | if influxdb_client.write_points(data_list, tags=tags): 57 | logger.debug("Success") 58 | break 59 | else: 60 | sleep(pause) 61 | except InfluxDBClientError: 62 | logger.debug('Failed {} out of {}'.format(i, retries)) 63 | else: 64 | logger.warning("Failed to write to Database") 65 | 66 | return to_influxdf 67 | 68 | 69 | def _set_retention_policy(influxdb_client, database, retention_duration, policy_name='standard'): 70 | retention_policies = influxdb_client.get_list_retention_policies(database=database) 71 | 72 | for policy in retention_policies: 73 | if policy['name'] == policy_name: 74 | _logger().debug('Found policy {}:{}'.format(policy_name, str(policy))) 75 | influxdb_client.alter_retention_policy(policy_name, 76 | database=database, 77 | duration=retention_duration, 78 | default=True) 79 | break 80 | else: 81 | _logger().debug('Creating policy {}'.format(policy_name)) 82 | influxdb_client.create_retention_policy(policy_name, 83 | retention_duration, 84 | 1, 85 | database=database, 86 | default=True) 87 | 88 | 89 | class MetricsRecordingFailed(Exception): 90 | pass 91 | 92 | 93 | def start_logger(ip_or_url, 94 | username, 95 | password, 96 | database, 97 | port=8086, 98 | series_name='gpu_measurements', 99 | polling_interval=1, 100 | retention_duration=MEASUREMENTS_RETENTION_DURATION, 101 | **tags): 102 | """ Starts GPU logger 103 | 104 | Logs GPU measurements to an influxdb database 105 | 106 | Parameters 107 | ---------- 108 | ip_or_url: ip or url of influxdb 109 | username: Username to log into influxdb database 110 | password: Password to log into influxdb database 111 | database: Name of database to log data to. It will create the database if one doesn't exist 112 | port: A number indicating the port on which influxdb is listening 113 | series_name: Name of series/table to log data to 114 | polling_interval: polling interval for measurements in seconds [default:1] 115 | retention_duration: the duration to retain the measurements for valid values are 1h, 90m, 12h, 7d, and 4w. default:1d 116 | tags: One or more tags to apply to the data. These can then be used to group or select timeseries 117 | Example: --machine my_machine --cluster kerb01 118 | 119 | """ 120 | 121 | logger = _logger() 122 | logger.info('Trying to connect to {} on port {} as {}'.format(ip_or_url, port, username)) 123 | try: 124 | client = InfluxDBClient(ip_or_url, port, username, password) 125 | response = client.ping() 126 | except ConnectionError: 127 | logger.warning('Could not connect to InfluxDB. GPU metrics NOT being recorded') 128 | raise MetricsRecordingFailed() 129 | 130 | logger.info('Connected | version {}'.format(response)) 131 | _switch_to_database(client, database) 132 | 133 | logger.info('Measurement retention duration {}'.format(retention_duration)) 134 | _set_retention_policy(client, database, retention_duration) 135 | 136 | to_db = compose(_create_influxdb_writer(client, tags=tags), 137 | _gpu_to_influxdb_format(series_name)) 138 | logger.info('Starting logging...') 139 | return start_pushing_measurements_to(to_db, polling_interval=polling_interval) 140 | 141 | 142 | def _start_logger_process(ip_or_url, 143 | port, 144 | username, 145 | password, 146 | database, 147 | series_name='gpu_measurements', 148 | polling_interval=1, 149 | retention_duration=MEASUREMENTS_RETENTION_DURATION, 150 | **tags): 151 | try: 152 | t, stop_logging = start_logger(ip_or_url, 153 | username, 154 | password, 155 | database, 156 | port=port, 157 | series_name=series_name, 158 | polling_interval=polling_interval, 159 | retention_duration=retention_duration, 160 | **tags) 161 | t.join() 162 | except MetricsRecordingFailed: 163 | return None 164 | 165 | 166 | @contextmanager 167 | def _log_process(ip_or_url, 168 | username, 169 | password, 170 | database, 171 | series_name, 172 | port=8086, 173 | polling_interval=1, 174 | retention_duration=MEASUREMENTS_RETENTION_DURATION, 175 | **tags): 176 | """ GPU logging context 177 | 178 | Logs GPU measurements to an influxdb database 179 | 180 | Parameters 181 | ---------- 182 | ip_or_url: ip or url of influxdb 183 | username: Username to log into influxdb database 184 | password: Password to log into influxdb database 185 | database: Name of database to log data to. It will create the database if one doesn't exist 186 | series_name: Name of series/table to log data to 187 | port: A number indicating the port on which influxdb is listening 188 | polling_interval: polling interval for measurements in seconds [default:1] 189 | retention_duration: the duration to retain the measurements for valid values are 1h, 90m, 12h, 7d, and 4w. default:1d 190 | tags: One or more tags to apply to the data. These can then be used to group or select timeseries 191 | Example: machine=my_machine cluster=kerb01 192 | """ 193 | logger = _logger() 194 | logger.info('Logging GPU to Database {}'.format(ip_or_url)) 195 | 196 | kwargs = {'series_name': series_name, 197 | 'polling_interval': polling_interval, 198 | 'retention_duration': retention_duration} 199 | kwargs.update(tags) 200 | p = Process(target=_start_logger_process, 201 | args=(ip_or_url, 202 | port, 203 | username, 204 | password, 205 | database), 206 | kwargs=kwargs) 207 | p.start() 208 | yield p 209 | p.terminate() 210 | p.join() 211 | 212 | 213 | log_context = populate_args_from_dotenv(_log_process) 214 | 215 | 216 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | bokeh>=0.12.6 2 | pandas>=0.19.1 3 | toolz>=0.8.2 4 | pip>=8.1.2 5 | setuptools 6 | pypandoc>=1.4 7 | influxdb==5.0.0 8 | requests>=2.17.0 9 | python-dotenv[cli]==0.8.2 10 | nvidia-ml-py3==7.352.0 11 | fire 12 | -------------------------------------------------------------------------------- /scripts/Makefile: -------------------------------------------------------------------------------- 1 | .ONESHELL: 2 | SHELL := /bin/bash 3 | 4 | define PROJECT_HELP_MSG 5 | Usage: 6 | make help show this message 7 | make run start influx container and grafana container 8 | make stop stop all containers 9 | make remove remove all containers 10 | make clean delete grafana and influxdb data 11 | endef 12 | export PROJECT_HELP_MSG 13 | 14 | help: 15 | @echo "$$PROJECT_HELP_MSG" | less 16 | 17 | run: 18 | @echo "Starting InfluxDB and Grafana" 19 | $(eval grafana:=$(shell dotenv get GRAFANA_DATA_LOCATION| cut -d '=' -f 2)) 20 | $(eval influxdb:=$(shell dotenv get INFLUXDB_DATA_LOCATION| cut -d '=' -f 2)) 21 | @mkdir $(grafana) 22 | @chmod 777 $(grafana) 23 | @mkdir $(influxdb) 24 | @chmod 777 $(influxdb) 25 | @docker-compose up -d 26 | 27 | stop: 28 | docker-compose down 29 | 30 | remove: 31 | docker-compose down --rmi all 32 | 33 | clean: 34 | $(eval grafana:=$(shell dotenv get GRAFANA_DATA_LOCATION| cut -d '=' -f 2)) 35 | $(eval influxdb:=$(shell dotenv get INFLUXDB_DATA_LOCATION| cut -d '=' -f 2)) 36 | @echo "Deleting $(grafana)" 37 | @sudo rm -rf $(grafana) 38 | @echo "Deleting $(influxdb)" 39 | @sudo rm -rf $(influxdb) 40 | 41 | .PHONY: help run stop remove clean 42 | -------------------------------------------------------------------------------- /scripts/docker-compose.yml: -------------------------------------------------------------------------------- 1 | influxdb: 2 | image: influxdb:latest 3 | container_name: influxdb 4 | ports: 5 | - "8083:8083" 6 | - "8086:8086" 7 | - "8090:8090" 8 | environment: 9 | INFLUXDB_ADMIN_ENABLED: "true" 10 | volumes: 11 | - ${INFLUXDB_DATA_LOCATION}:/var/lib/influxdb 12 | env_file: 13 | - .env 14 | 15 | grafana: 16 | image: grafana/grafana:latest 17 | container_name: grafana 18 | ports: 19 | - "3000:3000" 20 | links: 21 | - influxdb 22 | volumes: 23 | - ${GRAFANA_DATA_LOCATION}:/var/lib/grafana 24 | - ${PWD}/provisioning:/grafana-conf 25 | env_file: 26 | - .env 27 | user: "472" 28 | -------------------------------------------------------------------------------- /scripts/example.env: -------------------------------------------------------------------------------- 1 | # Example .env file please modify and DO NOT USE AS IS 2 | INFLUXDB_DB=gpudb 3 | INFLUXDB_USER=admin 4 | INFLUXDB_USER_PASSWORD=password 5 | INFLUXDB_ADMIN_ENABLED=true 6 | GF_SECURITY_ADMIN_PASSWORD=password 7 | GRAFANA_DATA_LOCATION=/tmp/grafana 8 | INFLUXDB_DATA_LOCATION=/tmp/influxdb 9 | GF_PATHS_PROVISIONING=/grafana-conf -------------------------------------------------------------------------------- /scripts/provisioning/dashboards/GPUDashboard.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "links": [], 19 | "panels": [ 20 | { 21 | "aliasColors": {}, 22 | "bars": false, 23 | "dashLength": 10, 24 | "dashes": false, 25 | "datasource": "${DS_GPU}", 26 | "fill": 2, 27 | "gridPos": { 28 | "h": 13, 29 | "w": 24, 30 | "x": 0, 31 | "y": 0 32 | }, 33 | "id": 2, 34 | "legend": { 35 | "avg": false, 36 | "current": false, 37 | "max": true, 38 | "min": false, 39 | "show": true, 40 | "total": false, 41 | "values": true 42 | }, 43 | "lines": true, 44 | "linewidth": 1, 45 | "links": [], 46 | "nullPointMode": "null", 47 | "percentage": false, 48 | "pointradius": 5, 49 | "points": false, 50 | "renderer": "flot", 51 | "seriesOverrides": [], 52 | "spaceLength": 10, 53 | "stack": false, 54 | "steppedLine": false, 55 | "targets": [ 56 | { 57 | "groupBy": [ 58 | { 59 | "params": [ 60 | "$__interval" 61 | ], 62 | "type": "time" 63 | }, 64 | { 65 | "params": [ 66 | "GPU" 67 | ], 68 | "type": "tag" 69 | }, 70 | { 71 | "params": [ 72 | "none" 73 | ], 74 | "type": "fill" 75 | } 76 | ], 77 | "measurement": "gpuseries", 78 | "orderByTime": "ASC", 79 | "policy": "standard", 80 | "refId": "A", 81 | "resultFormat": "time_series", 82 | "select": [ 83 | [ 84 | { 85 | "params": [ 86 | "Utilization" 87 | ], 88 | "type": "field" 89 | }, 90 | { 91 | "params": [], 92 | "type": "max" 93 | } 94 | ] 95 | ], 96 | "tags": [] 97 | } 98 | ], 99 | "thresholds": [], 100 | "timeFrom": null, 101 | "timeShift": null, 102 | "title": "GPU Utilization", 103 | "tooltip": { 104 | "shared": true, 105 | "sort": 0, 106 | "value_type": "individual" 107 | }, 108 | "type": "graph", 109 | "xaxis": { 110 | "buckets": null, 111 | "mode": "time", 112 | "name": null, 113 | "show": true, 114 | "values": [] 115 | }, 116 | "yaxes": [ 117 | { 118 | "format": "short", 119 | "label": null, 120 | "logBase": 1, 121 | "max": null, 122 | "min": null, 123 | "show": true 124 | }, 125 | { 126 | "format": "short", 127 | "label": null, 128 | "logBase": 1, 129 | "max": null, 130 | "min": null, 131 | "show": true 132 | } 133 | ], 134 | "yaxis": { 135 | "align": false, 136 | "alignLevel": null 137 | } 138 | }, 139 | { 140 | "cacheTimeout": null, 141 | "colorBackground": false, 142 | "colorValue": true, 143 | "colors": [ 144 | "#299c46", 145 | "rgba(237, 129, 40, 0.89)", 146 | "#d44a3a" 147 | ], 148 | "datasource": "${DS_GPU}", 149 | "format": "celsius", 150 | "gauge": { 151 | "maxValue": 100, 152 | "minValue": 0, 153 | "show": true, 154 | "thresholdLabels": false, 155 | "thresholdMarkers": true 156 | }, 157 | "gridPos": { 158 | "h": 8, 159 | "w": 4, 160 | "x": 0, 161 | "y": 13 162 | }, 163 | "hideTimeOverride": true, 164 | "id": 4, 165 | "interval": null, 166 | "links": [], 167 | "mappingType": 1, 168 | "mappingTypes": [ 169 | { 170 | "name": "value to text", 171 | "value": 1 172 | }, 173 | { 174 | "name": "range to text", 175 | "value": 2 176 | } 177 | ], 178 | "maxDataPoints": 100, 179 | "nullPointMode": "connected", 180 | "nullText": null, 181 | "postfix": "", 182 | "postfixFontSize": "50%", 183 | "prefix": "", 184 | "prefixFontSize": "50%", 185 | "rangeMaps": [ 186 | { 187 | "from": "null", 188 | "text": "N/A", 189 | "to": "null" 190 | } 191 | ], 192 | "sparkline": { 193 | "fillColor": "rgba(31, 118, 189, 0.18)", 194 | "full": false, 195 | "lineColor": "rgb(31, 120, 193)", 196 | "show": false 197 | }, 198 | "tableColumn": "", 199 | "targets": [ 200 | { 201 | "groupBy": [ 202 | { 203 | "params": [ 204 | "$__interval" 205 | ], 206 | "type": "time" 207 | }, 208 | { 209 | "params": [ 210 | "null" 211 | ], 212 | "type": "fill" 213 | } 214 | ], 215 | "measurement": "gpuseries", 216 | "orderByTime": "ASC", 217 | "policy": "default", 218 | "refId": "A", 219 | "resultFormat": "time_series", 220 | "select": [ 221 | [ 222 | { 223 | "params": [ 224 | "Temperature" 225 | ], 226 | "type": "field" 227 | }, 228 | { 229 | "params": [], 230 | "type": "mean" 231 | } 232 | ] 233 | ], 234 | "tags": [ 235 | { 236 | "key": "GPU", 237 | "operator": "=", 238 | "value": "0" 239 | } 240 | ] 241 | } 242 | ], 243 | "thresholds": "40,80", 244 | "timeFrom": "10s", 245 | "title": "GPU 0", 246 | "type": "singlestat", 247 | "valueFontSize": "80%", 248 | "valueMaps": [ 249 | { 250 | "op": "=", 251 | "text": "N/A", 252 | "value": "null" 253 | } 254 | ], 255 | "valueName": "current" 256 | }, 257 | { 258 | "cacheTimeout": null, 259 | "colorBackground": false, 260 | "colorValue": true, 261 | "colors": [ 262 | "#299c46", 263 | "rgba(237, 129, 40, 0.89)", 264 | "#d44a3a" 265 | ], 266 | "datasource": "${DS_GPU}", 267 | "format": "celsius", 268 | "gauge": { 269 | "maxValue": 100, 270 | "minValue": 0, 271 | "show": true, 272 | "thresholdLabels": false, 273 | "thresholdMarkers": true 274 | }, 275 | "gridPos": { 276 | "h": 8, 277 | "w": 4, 278 | "x": 4, 279 | "y": 13 280 | }, 281 | "hideTimeOverride": true, 282 | "id": 5, 283 | "interval": null, 284 | "links": [], 285 | "mappingType": 1, 286 | "mappingTypes": [ 287 | { 288 | "name": "value to text", 289 | "value": 1 290 | }, 291 | { 292 | "name": "range to text", 293 | "value": 2 294 | } 295 | ], 296 | "maxDataPoints": 100, 297 | "nullPointMode": "connected", 298 | "nullText": null, 299 | "postfix": "", 300 | "postfixFontSize": "50%", 301 | "prefix": "", 302 | "prefixFontSize": "50%", 303 | "rangeMaps": [ 304 | { 305 | "from": "null", 306 | "text": "N/A", 307 | "to": "null" 308 | } 309 | ], 310 | "sparkline": { 311 | "fillColor": "rgba(31, 118, 189, 0.18)", 312 | "full": false, 313 | "lineColor": "rgb(31, 120, 193)", 314 | "show": false 315 | }, 316 | "tableColumn": "", 317 | "targets": [ 318 | { 319 | "groupBy": [ 320 | { 321 | "params": [ 322 | "$__interval" 323 | ], 324 | "type": "time" 325 | }, 326 | { 327 | "params": [ 328 | "null" 329 | ], 330 | "type": "fill" 331 | } 332 | ], 333 | "measurement": "gpuseries", 334 | "orderByTime": "ASC", 335 | "policy": "default", 336 | "refId": "A", 337 | "resultFormat": "time_series", 338 | "select": [ 339 | [ 340 | { 341 | "params": [ 342 | "Temperature" 343 | ], 344 | "type": "field" 345 | }, 346 | { 347 | "params": [], 348 | "type": "mean" 349 | } 350 | ] 351 | ], 352 | "tags": [ 353 | { 354 | "key": "GPU", 355 | "operator": "=", 356 | "value": "1" 357 | } 358 | ] 359 | } 360 | ], 361 | "thresholds": "40,80", 362 | "timeFrom": "10s", 363 | "title": "GPU 1", 364 | "type": "singlestat", 365 | "valueFontSize": "80%", 366 | "valueMaps": [ 367 | { 368 | "op": "=", 369 | "text": "N/A", 370 | "value": "null" 371 | } 372 | ], 373 | "valueName": "current" 374 | }, 375 | { 376 | "cacheTimeout": null, 377 | "colorBackground": false, 378 | "colorValue": true, 379 | "colors": [ 380 | "#299c46", 381 | "rgba(237, 129, 40, 0.89)", 382 | "#d44a3a" 383 | ], 384 | "datasource": "${DS_GPU}", 385 | "format": "celsius", 386 | "gauge": { 387 | "maxValue": 100, 388 | "minValue": 0, 389 | "show": true, 390 | "thresholdLabels": false, 391 | "thresholdMarkers": true 392 | }, 393 | "gridPos": { 394 | "h": 8, 395 | "w": 4, 396 | "x": 8, 397 | "y": 13 398 | }, 399 | "hideTimeOverride": true, 400 | "id": 7, 401 | "interval": null, 402 | "links": [], 403 | "mappingType": 1, 404 | "mappingTypes": [ 405 | { 406 | "name": "value to text", 407 | "value": 1 408 | }, 409 | { 410 | "name": "range to text", 411 | "value": 2 412 | } 413 | ], 414 | "maxDataPoints": 100, 415 | "nullPointMode": "connected", 416 | "nullText": null, 417 | "postfix": "", 418 | "postfixFontSize": "50%", 419 | "prefix": "", 420 | "prefixFontSize": "50%", 421 | "rangeMaps": [ 422 | { 423 | "from": "null", 424 | "text": "N/A", 425 | "to": "null" 426 | } 427 | ], 428 | "sparkline": { 429 | "fillColor": "rgba(31, 118, 189, 0.18)", 430 | "full": false, 431 | "lineColor": "rgb(31, 120, 193)", 432 | "show": false 433 | }, 434 | "tableColumn": "", 435 | "targets": [ 436 | { 437 | "groupBy": [ 438 | { 439 | "params": [ 440 | "$__interval" 441 | ], 442 | "type": "time" 443 | }, 444 | { 445 | "params": [ 446 | "null" 447 | ], 448 | "type": "fill" 449 | } 450 | ], 451 | "measurement": "gpuseries", 452 | "orderByTime": "ASC", 453 | "policy": "default", 454 | "refId": "A", 455 | "resultFormat": "time_series", 456 | "select": [ 457 | [ 458 | { 459 | "params": [ 460 | "Temperature" 461 | ], 462 | "type": "field" 463 | }, 464 | { 465 | "params": [], 466 | "type": "mean" 467 | } 468 | ] 469 | ], 470 | "tags": [ 471 | { 472 | "key": "GPU", 473 | "operator": "=", 474 | "value": "3" 475 | } 476 | ] 477 | } 478 | ], 479 | "thresholds": "40,80", 480 | "timeFrom": "10s", 481 | "title": "GPU 3", 482 | "type": "singlestat", 483 | "valueFontSize": "80%", 484 | "valueMaps": [ 485 | { 486 | "op": "=", 487 | "text": "N/A", 488 | "value": "null" 489 | } 490 | ], 491 | "valueName": "current" 492 | }, 493 | { 494 | "cacheTimeout": null, 495 | "colorBackground": false, 496 | "colorValue": true, 497 | "colors": [ 498 | "#299c46", 499 | "rgba(237, 129, 40, 0.89)", 500 | "#d44a3a" 501 | ], 502 | "datasource": "${DS_GPU}", 503 | "format": "celsius", 504 | "gauge": { 505 | "maxValue": 100, 506 | "minValue": 0, 507 | "show": true, 508 | "thresholdLabels": false, 509 | "thresholdMarkers": true 510 | }, 511 | "gridPos": { 512 | "h": 8, 513 | "w": 4, 514 | "x": 12, 515 | "y": 13 516 | }, 517 | "hideTimeOverride": true, 518 | "id": 6, 519 | "interval": null, 520 | "links": [], 521 | "mappingType": 1, 522 | "mappingTypes": [ 523 | { 524 | "name": "value to text", 525 | "value": 1 526 | }, 527 | { 528 | "name": "range to text", 529 | "value": 2 530 | } 531 | ], 532 | "maxDataPoints": 100, 533 | "nullPointMode": "connected", 534 | "nullText": null, 535 | "postfix": "", 536 | "postfixFontSize": "50%", 537 | "prefix": "", 538 | "prefixFontSize": "50%", 539 | "rangeMaps": [ 540 | { 541 | "from": "null", 542 | "text": "N/A", 543 | "to": "null" 544 | } 545 | ], 546 | "sparkline": { 547 | "fillColor": "rgba(31, 118, 189, 0.18)", 548 | "full": false, 549 | "lineColor": "rgb(31, 120, 193)", 550 | "show": false 551 | }, 552 | "tableColumn": "", 553 | "targets": [ 554 | { 555 | "groupBy": [ 556 | { 557 | "params": [ 558 | "$__interval" 559 | ], 560 | "type": "time" 561 | }, 562 | { 563 | "params": [ 564 | "null" 565 | ], 566 | "type": "fill" 567 | } 568 | ], 569 | "measurement": "gpuseries", 570 | "orderByTime": "ASC", 571 | "policy": "default", 572 | "refId": "A", 573 | "resultFormat": "time_series", 574 | "select": [ 575 | [ 576 | { 577 | "params": [ 578 | "Temperature" 579 | ], 580 | "type": "field" 581 | }, 582 | { 583 | "params": [], 584 | "type": "mean" 585 | } 586 | ] 587 | ], 588 | "tags": [ 589 | { 590 | "key": "GPU", 591 | "operator": "=", 592 | "value": "2" 593 | } 594 | ] 595 | } 596 | ], 597 | "thresholds": "40,80", 598 | "timeFrom": "10s", 599 | "title": "GPU 2", 600 | "type": "singlestat", 601 | "valueFontSize": "80%", 602 | "valueMaps": [ 603 | { 604 | "op": "=", 605 | "text": "N/A", 606 | "value": "null" 607 | } 608 | ], 609 | "valueName": "current" 610 | }, 611 | { 612 | "aliasColors": {}, 613 | "bars": false, 614 | "dashLength": 10, 615 | "dashes": false, 616 | "datasource": "${DS_GPU}", 617 | "fill": 1, 618 | "gridPos": { 619 | "h": 8, 620 | "w": 8, 621 | "x": 16, 622 | "y": 13 623 | }, 624 | "id": 11, 625 | "legend": { 626 | "avg": false, 627 | "current": false, 628 | "max": true, 629 | "min": false, 630 | "show": true, 631 | "total": false, 632 | "values": true 633 | }, 634 | "lines": true, 635 | "linewidth": 1, 636 | "links": [], 637 | "nullPointMode": "null", 638 | "percentage": false, 639 | "pointradius": 5, 640 | "points": false, 641 | "renderer": "flot", 642 | "seriesOverrides": [], 643 | "spaceLength": 10, 644 | "stack": false, 645 | "steppedLine": false, 646 | "targets": [ 647 | { 648 | "groupBy": [ 649 | { 650 | "params": [ 651 | "$__interval" 652 | ], 653 | "type": "time" 654 | }, 655 | { 656 | "params": [ 657 | "GPU" 658 | ], 659 | "type": "tag" 660 | }, 661 | { 662 | "params": [ 663 | "none" 664 | ], 665 | "type": "fill" 666 | } 667 | ], 668 | "measurement": "gpuseries", 669 | "orderByTime": "ASC", 670 | "policy": "standard", 671 | "refId": "A", 672 | "resultFormat": "time_series", 673 | "select": [ 674 | [ 675 | { 676 | "params": [ 677 | "Memory Used Percent" 678 | ], 679 | "type": "field" 680 | }, 681 | { 682 | "params": [], 683 | "type": "max" 684 | } 685 | ] 686 | ], 687 | "tags": [] 688 | } 689 | ], 690 | "thresholds": [], 691 | "timeFrom": null, 692 | "timeShift": null, 693 | "title": "Memory Used (%)", 694 | "tooltip": { 695 | "shared": true, 696 | "sort": 0, 697 | "value_type": "individual" 698 | }, 699 | "type": "graph", 700 | "xaxis": { 701 | "buckets": null, 702 | "mode": "time", 703 | "name": null, 704 | "show": true, 705 | "values": [] 706 | }, 707 | "yaxes": [ 708 | { 709 | "decimals": null, 710 | "format": "short", 711 | "label": "Memory Used (%)", 712 | "logBase": 1, 713 | "max": null, 714 | "min": "0", 715 | "show": true 716 | }, 717 | { 718 | "format": "short", 719 | "label": null, 720 | "logBase": 1, 721 | "max": null, 722 | "min": null, 723 | "show": false 724 | } 725 | ], 726 | "yaxis": { 727 | "align": false, 728 | "alignLevel": null 729 | } 730 | }, 731 | { 732 | "cacheTimeout": null, 733 | "colorBackground": false, 734 | "colorValue": true, 735 | "colors": [ 736 | "#299c46", 737 | "rgba(237, 129, 40, 0.89)", 738 | "#d44a3a" 739 | ], 740 | "datasource": "${DS_GPU}", 741 | "decimals": 2, 742 | "format": "decgbytes", 743 | "gauge": { 744 | "maxValue": 100, 745 | "minValue": 0, 746 | "show": false, 747 | "thresholdLabels": false, 748 | "thresholdMarkers": true 749 | }, 750 | "gridPos": { 751 | "h": 3, 752 | "w": 4, 753 | "x": 0, 754 | "y": 21 755 | }, 756 | "hideTimeOverride": true, 757 | "id": 14, 758 | "interval": null, 759 | "links": [], 760 | "mappingType": 1, 761 | "mappingTypes": [ 762 | { 763 | "name": "value to text", 764 | "value": 1 765 | }, 766 | { 767 | "name": "range to text", 768 | "value": 2 769 | } 770 | ], 771 | "maxDataPoints": 100, 772 | "nullPointMode": "connected", 773 | "nullText": null, 774 | "postfix": "", 775 | "postfixFontSize": "50%", 776 | "prefix": "", 777 | "prefixFontSize": "50%", 778 | "rangeMaps": [ 779 | { 780 | "from": "null", 781 | "text": "N/A", 782 | "to": "null" 783 | } 784 | ], 785 | "sparkline": { 786 | "fillColor": "rgba(31, 118, 189, 0.18)", 787 | "full": false, 788 | "lineColor": "rgb(31, 120, 193)", 789 | "show": false 790 | }, 791 | "tableColumn": "", 792 | "targets": [ 793 | { 794 | "groupBy": [ 795 | { 796 | "params": [ 797 | "$__interval" 798 | ], 799 | "type": "time" 800 | }, 801 | { 802 | "params": [ 803 | "null" 804 | ], 805 | "type": "fill" 806 | } 807 | ], 808 | "measurement": "gpuseries", 809 | "orderByTime": "ASC", 810 | "policy": "standard", 811 | "refId": "A", 812 | "resultFormat": "time_series", 813 | "select": [ 814 | [ 815 | { 816 | "params": [ 817 | "Memory Used" 818 | ], 819 | "type": "field" 820 | }, 821 | { 822 | "params": [], 823 | "type": "max" 824 | }, 825 | { 826 | "params": [ 827 | " / 1000000000" 828 | ], 829 | "type": "math" 830 | } 831 | ] 832 | ], 833 | "tags": [ 834 | { 835 | "key": "GPU", 836 | "operator": "=", 837 | "value": "0" 838 | } 839 | ] 840 | } 841 | ], 842 | "thresholds": "10,19", 843 | "timeFrom": "10s", 844 | "title": "GPU 0 Memory Used", 845 | "type": "singlestat", 846 | "valueFontSize": "120%", 847 | "valueMaps": [ 848 | { 849 | "op": "=", 850 | "text": "N/A", 851 | "value": "null" 852 | } 853 | ], 854 | "valueName": "current" 855 | }, 856 | { 857 | "cacheTimeout": null, 858 | "colorBackground": false, 859 | "colorValue": true, 860 | "colors": [ 861 | "#299c46", 862 | "rgba(237, 129, 40, 0.89)", 863 | "#d44a3a" 864 | ], 865 | "datasource": "${DS_GPU}", 866 | "decimals": 2, 867 | "format": "decgbytes", 868 | "gauge": { 869 | "maxValue": 100, 870 | "minValue": 0, 871 | "show": false, 872 | "thresholdLabels": false, 873 | "thresholdMarkers": true 874 | }, 875 | "gridPos": { 876 | "h": 3, 877 | "w": 4, 878 | "x": 4, 879 | "y": 21 880 | }, 881 | "hideTimeOverride": true, 882 | "id": 15, 883 | "interval": null, 884 | "links": [], 885 | "mappingType": 1, 886 | "mappingTypes": [ 887 | { 888 | "name": "value to text", 889 | "value": 1 890 | }, 891 | { 892 | "name": "range to text", 893 | "value": 2 894 | } 895 | ], 896 | "maxDataPoints": 100, 897 | "nullPointMode": "connected", 898 | "nullText": null, 899 | "postfix": "", 900 | "postfixFontSize": "50%", 901 | "prefix": "", 902 | "prefixFontSize": "50%", 903 | "rangeMaps": [ 904 | { 905 | "from": "null", 906 | "text": "N/A", 907 | "to": "null" 908 | } 909 | ], 910 | "sparkline": { 911 | "fillColor": "rgba(31, 118, 189, 0.18)", 912 | "full": false, 913 | "lineColor": "rgb(31, 120, 193)", 914 | "show": false 915 | }, 916 | "tableColumn": "", 917 | "targets": [ 918 | { 919 | "groupBy": [ 920 | { 921 | "params": [ 922 | "$__interval" 923 | ], 924 | "type": "time" 925 | }, 926 | { 927 | "params": [ 928 | "null" 929 | ], 930 | "type": "fill" 931 | } 932 | ], 933 | "measurement": "gpuseries", 934 | "orderByTime": "ASC", 935 | "policy": "standard", 936 | "refId": "A", 937 | "resultFormat": "time_series", 938 | "select": [ 939 | [ 940 | { 941 | "params": [ 942 | "Memory Used" 943 | ], 944 | "type": "field" 945 | }, 946 | { 947 | "params": [], 948 | "type": "max" 949 | }, 950 | { 951 | "params": [ 952 | " / 1000000000" 953 | ], 954 | "type": "math" 955 | } 956 | ] 957 | ], 958 | "tags": [ 959 | { 960 | "key": "GPU", 961 | "operator": "=", 962 | "value": "1" 963 | } 964 | ] 965 | } 966 | ], 967 | "thresholds": "10,19", 968 | "timeFrom": "10s", 969 | "title": "GPU 1 Memory Used", 970 | "type": "singlestat", 971 | "valueFontSize": "120%", 972 | "valueMaps": [ 973 | { 974 | "op": "=", 975 | "text": "N/A", 976 | "value": "null" 977 | } 978 | ], 979 | "valueName": "current" 980 | }, 981 | { 982 | "aliasColors": {}, 983 | "bars": false, 984 | "dashLength": 10, 985 | "dashes": false, 986 | "datasource": "${DS_GPU}", 987 | "fill": 1, 988 | "gridPos": { 989 | "h": 6, 990 | "w": 16, 991 | "x": 8, 992 | "y": 21 993 | }, 994 | "id": 9, 995 | "legend": { 996 | "alignAsTable": false, 997 | "avg": false, 998 | "current": false, 999 | "max": true, 1000 | "min": false, 1001 | "show": true, 1002 | "total": false, 1003 | "values": true 1004 | }, 1005 | "lines": true, 1006 | "linewidth": 1, 1007 | "links": [], 1008 | "nullPointMode": "null", 1009 | "percentage": false, 1010 | "pointradius": 5, 1011 | "points": false, 1012 | "renderer": "flot", 1013 | "seriesOverrides": [], 1014 | "spaceLength": 10, 1015 | "stack": false, 1016 | "steppedLine": false, 1017 | "targets": [ 1018 | { 1019 | "groupBy": [ 1020 | { 1021 | "params": [ 1022 | "$__interval" 1023 | ], 1024 | "type": "time" 1025 | }, 1026 | { 1027 | "params": [ 1028 | "GPU" 1029 | ], 1030 | "type": "tag" 1031 | }, 1032 | { 1033 | "params": [ 1034 | "none" 1035 | ], 1036 | "type": "fill" 1037 | } 1038 | ], 1039 | "measurement": "gpuseries", 1040 | "orderByTime": "ASC", 1041 | "policy": "standard", 1042 | "refId": "A", 1043 | "resultFormat": "time_series", 1044 | "select": [ 1045 | [ 1046 | { 1047 | "params": [ 1048 | "Memory Utilization" 1049 | ], 1050 | "type": "field" 1051 | }, 1052 | { 1053 | "params": [], 1054 | "type": "max" 1055 | } 1056 | ] 1057 | ], 1058 | "tags": [] 1059 | } 1060 | ], 1061 | "thresholds": [], 1062 | "timeFrom": null, 1063 | "timeShift": null, 1064 | "title": "Memory Utilization", 1065 | "tooltip": { 1066 | "shared": true, 1067 | "sort": 0, 1068 | "value_type": "individual" 1069 | }, 1070 | "type": "graph", 1071 | "xaxis": { 1072 | "buckets": null, 1073 | "mode": "time", 1074 | "name": null, 1075 | "show": true, 1076 | "values": [] 1077 | }, 1078 | "yaxes": [ 1079 | { 1080 | "format": "short", 1081 | "label": null, 1082 | "logBase": 1, 1083 | "max": null, 1084 | "min": null, 1085 | "show": true 1086 | }, 1087 | { 1088 | "format": "short", 1089 | "label": null, 1090 | "logBase": 1, 1091 | "max": null, 1092 | "min": null, 1093 | "show": true 1094 | } 1095 | ], 1096 | "yaxis": { 1097 | "align": false, 1098 | "alignLevel": null 1099 | } 1100 | }, 1101 | { 1102 | "cacheTimeout": null, 1103 | "colorBackground": false, 1104 | "colorValue": true, 1105 | "colors": [ 1106 | "#299c46", 1107 | "rgba(237, 129, 40, 0.89)", 1108 | "#d44a3a" 1109 | ], 1110 | "datasource": "${DS_GPU}", 1111 | "decimals": 2, 1112 | "format": "decgbytes", 1113 | "gauge": { 1114 | "maxValue": 100, 1115 | "minValue": 0, 1116 | "show": false, 1117 | "thresholdLabels": false, 1118 | "thresholdMarkers": true 1119 | }, 1120 | "gridPos": { 1121 | "h": 3, 1122 | "w": 4, 1123 | "x": 0, 1124 | "y": 24 1125 | }, 1126 | "hideTimeOverride": true, 1127 | "id": 16, 1128 | "interval": null, 1129 | "links": [], 1130 | "mappingType": 1, 1131 | "mappingTypes": [ 1132 | { 1133 | "name": "value to text", 1134 | "value": 1 1135 | }, 1136 | { 1137 | "name": "range to text", 1138 | "value": 2 1139 | } 1140 | ], 1141 | "maxDataPoints": 100, 1142 | "nullPointMode": "connected", 1143 | "nullText": null, 1144 | "postfix": "", 1145 | "postfixFontSize": "50%", 1146 | "prefix": "", 1147 | "prefixFontSize": "50%", 1148 | "rangeMaps": [ 1149 | { 1150 | "from": "null", 1151 | "text": "N/A", 1152 | "to": "null" 1153 | } 1154 | ], 1155 | "sparkline": { 1156 | "fillColor": "rgba(31, 118, 189, 0.18)", 1157 | "full": false, 1158 | "lineColor": "rgb(31, 120, 193)", 1159 | "show": false 1160 | }, 1161 | "tableColumn": "", 1162 | "targets": [ 1163 | { 1164 | "groupBy": [ 1165 | { 1166 | "params": [ 1167 | "$__interval" 1168 | ], 1169 | "type": "time" 1170 | }, 1171 | { 1172 | "params": [ 1173 | "null" 1174 | ], 1175 | "type": "fill" 1176 | } 1177 | ], 1178 | "measurement": "gpuseries", 1179 | "orderByTime": "ASC", 1180 | "policy": "standard", 1181 | "refId": "A", 1182 | "resultFormat": "time_series", 1183 | "select": [ 1184 | [ 1185 | { 1186 | "params": [ 1187 | "Memory Used" 1188 | ], 1189 | "type": "field" 1190 | }, 1191 | { 1192 | "params": [], 1193 | "type": "max" 1194 | }, 1195 | { 1196 | "params": [ 1197 | " / 1000000000" 1198 | ], 1199 | "type": "math" 1200 | } 1201 | ] 1202 | ], 1203 | "tags": [ 1204 | { 1205 | "key": "GPU", 1206 | "operator": "=", 1207 | "value": "2" 1208 | } 1209 | ] 1210 | } 1211 | ], 1212 | "thresholds": "10,19", 1213 | "timeFrom": "10s", 1214 | "title": "GPU 2 Memory Used", 1215 | "type": "singlestat", 1216 | "valueFontSize": "120%", 1217 | "valueMaps": [ 1218 | { 1219 | "op": "=", 1220 | "text": "N/A", 1221 | "value": "null" 1222 | } 1223 | ], 1224 | "valueName": "current" 1225 | }, 1226 | { 1227 | "cacheTimeout": null, 1228 | "colorBackground": false, 1229 | "colorValue": true, 1230 | "colors": [ 1231 | "#299c46", 1232 | "rgba(237, 129, 40, 0.89)", 1233 | "#d44a3a" 1234 | ], 1235 | "datasource": "${DS_GPU}", 1236 | "decimals": 2, 1237 | "format": "decgbytes", 1238 | "gauge": { 1239 | "maxValue": 100, 1240 | "minValue": 0, 1241 | "show": false, 1242 | "thresholdLabels": false, 1243 | "thresholdMarkers": true 1244 | }, 1245 | "gridPos": { 1246 | "h": 3, 1247 | "w": 4, 1248 | "x": 4, 1249 | "y": 24 1250 | }, 1251 | "hideTimeOverride": true, 1252 | "id": 17, 1253 | "interval": null, 1254 | "links": [], 1255 | "mappingType": 1, 1256 | "mappingTypes": [ 1257 | { 1258 | "name": "value to text", 1259 | "value": 1 1260 | }, 1261 | { 1262 | "name": "range to text", 1263 | "value": 2 1264 | } 1265 | ], 1266 | "maxDataPoints": 100, 1267 | "nullPointMode": "connected", 1268 | "nullText": null, 1269 | "postfix": "", 1270 | "postfixFontSize": "50%", 1271 | "prefix": "", 1272 | "prefixFontSize": "50%", 1273 | "rangeMaps": [ 1274 | { 1275 | "from": "null", 1276 | "text": "N/A", 1277 | "to": "null" 1278 | } 1279 | ], 1280 | "sparkline": { 1281 | "fillColor": "rgba(31, 118, 189, 0.18)", 1282 | "full": false, 1283 | "lineColor": "rgb(31, 120, 193)", 1284 | "show": false 1285 | }, 1286 | "tableColumn": "", 1287 | "targets": [ 1288 | { 1289 | "groupBy": [ 1290 | { 1291 | "params": [ 1292 | "$__interval" 1293 | ], 1294 | "type": "time" 1295 | }, 1296 | { 1297 | "params": [ 1298 | "null" 1299 | ], 1300 | "type": "fill" 1301 | } 1302 | ], 1303 | "measurement": "gpuseries", 1304 | "orderByTime": "ASC", 1305 | "policy": "standard", 1306 | "refId": "A", 1307 | "resultFormat": "time_series", 1308 | "select": [ 1309 | [ 1310 | { 1311 | "params": [ 1312 | "Memory Used" 1313 | ], 1314 | "type": "field" 1315 | }, 1316 | { 1317 | "params": [], 1318 | "type": "max" 1319 | }, 1320 | { 1321 | "params": [ 1322 | " / 1000000000" 1323 | ], 1324 | "type": "math" 1325 | } 1326 | ] 1327 | ], 1328 | "tags": [ 1329 | { 1330 | "key": "GPU", 1331 | "operator": "=", 1332 | "value": "3" 1333 | } 1334 | ] 1335 | } 1336 | ], 1337 | "thresholds": "10,19", 1338 | "timeFrom": "10s", 1339 | "title": "GPU 3 Memory Used", 1340 | "type": "singlestat", 1341 | "valueFontSize": "120%", 1342 | "valueMaps": [ 1343 | { 1344 | "op": "=", 1345 | "text": "N/A", 1346 | "value": "null" 1347 | } 1348 | ], 1349 | "valueName": "current" 1350 | } 1351 | ], 1352 | "refresh": "5s", 1353 | "schemaVersion": 16, 1354 | "style": "dark", 1355 | "tags": [], 1356 | "templating": { 1357 | "list": [ 1358 | { 1359 | "current": { 1360 | "text": "gpu", 1361 | "value": "gpu" 1362 | }, 1363 | "hide": 0, 1364 | "label": null, 1365 | "name": "DS_GPU", 1366 | "options": [], 1367 | "query": "influxdb", 1368 | "refresh": 1, 1369 | "regex": "", 1370 | "type": "datasource" 1371 | } 1372 | ] 1373 | }, 1374 | "time": { 1375 | "from": "now-5m", 1376 | "to": "now" 1377 | }, 1378 | "timepicker": { 1379 | "refresh_intervals": [ 1380 | "5s", 1381 | "10s", 1382 | "30s", 1383 | "1m", 1384 | "5m", 1385 | "15m", 1386 | "30m", 1387 | "1h", 1388 | "2h", 1389 | "1d" 1390 | ], 1391 | "time_options": [ 1392 | "5m", 1393 | "15m", 1394 | "1h", 1395 | "6h", 1396 | "12h", 1397 | "24h", 1398 | "2d", 1399 | "7d", 1400 | "30d" 1401 | ] 1402 | }, 1403 | "timezone": "", 1404 | "title": "GPUDashboard", 1405 | "uid": "3a33_qzik", 1406 | "version": 1 1407 | } -------------------------------------------------------------------------------- /scripts/provisioning/dashboards/gpudash.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: 'default' 5 | orgId: 1 6 | folder: '' 7 | type: file 8 | disableDeletion: false 9 | editable: false 10 | options: 11 | path: /grafana-conf/dashboards -------------------------------------------------------------------------------- /scripts/provisioning/datasources/influxdb-source.yaml: -------------------------------------------------------------------------------- 1 | # # config file version 2 | apiVersion: 1 3 | 4 | datasources: 5 | - name: gpu 6 | type: influxdb 7 | access: proxy 8 | database: gpudb 9 | user: admin 10 | password: password 11 | url: http://influxdb:8086 -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | try: 3 | from setuptools import setup, find_packages 4 | except ImportError: 5 | from distutils.core import setup, find_packages 6 | 7 | 8 | # Package meta-data. 9 | NAME = 'gpumon' 10 | DESCRIPTION = 'A set of handy utilities to monitor GPUs' 11 | URL = 'https://github.com/msalvaris/gpu_monitor' 12 | EMAIL = 'msalvaris@github.com' 13 | AUTHOR = 'Mathew Salvaris' 14 | LICENSE = '' 15 | 16 | 17 | try: 18 | import pypandoc 19 | long_description = pypandoc.convert('README.md', 'rst') 20 | except (IOError, ImportError): 21 | long_description = "Python package for reading GPU properties" 22 | 23 | 24 | with open('requirements.txt') as f: 25 | requirements = f.read().splitlines() 26 | 27 | 28 | here = os.path.abspath(os.path.dirname(__file__)) 29 | 30 | # Load the package's __version__.py module as a dictionary. 31 | about = {} 32 | with open(os.path.join(here, NAME, '__version__.py')) as f: 33 | exec(f.read(), about) 34 | 35 | 36 | setup( 37 | name=NAME, 38 | version=about['__version__'], 39 | url=URL, 40 | license=LICENSE, 41 | author=AUTHOR, 42 | author_email=EMAIL, 43 | description=DESCRIPTION, 44 | long_description=long_description, 45 | scripts=['gpumon/gpumon'], 46 | packages=find_packages(), 47 | include_package_data=True, 48 | install_requires=requirements, 49 | classifiers=[ 50 | 'Development Status :: 1 - Alpha', 51 | 'Intended Audience :: Data Scientists & Developers', 52 | 'Operating System :: POSIX', 53 | 'Operating System :: POSIX :: Linux', 54 | 'Programming Language :: Python :: 3.5', 55 | ] 56 | ) 57 | -------------------------------------------------------------------------------- /static/gpu_dashboard.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msalvaris/gpu_monitor/1e733450b8fba11876363f3270d675308692c4fc/static/gpu_dashboard.gif -------------------------------------------------------------------------------- /static/influxdb_config.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/msalvaris/gpu_monitor/1e733450b8fba11876363f3270d675308692c4fc/static/influxdb_config.png --------------------------------------------------------------------------------