├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── client ├── lua │ ├── README.md │ ├── crayon-0.5-1.rockspec │ ├── crayon.lua │ └── test.lua └── python │ ├── MANIFEST │ ├── README.md │ ├── pycrayon │ ├── __init__.py │ ├── crayon.py │ └── version.py │ ├── setup.py │ └── test │ ├── helper.py │ └── test_crayon.py ├── doc └── specs.md └── server ├── Dockerfile ├── README.md ├── patch_tensorboard.py ├── server.py └── startup.sh /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: required 2 | 3 | language: python 4 | python: 5 | - '2.7' 6 | - '3.5' 7 | 8 | env: 9 | - DOCKER_API_VERSION=1.24 10 | 11 | services: 12 | - docker 13 | 14 | before_install: 15 | - pip install docker 16 | - pushd server 17 | - docker build -t alband/crayon:latest . 18 | - popd 19 | script: 20 | - pushd client/python 21 | - python setup.py install 22 | - python -Wignore -m unittest discover test/ 23 | - popd 24 | 25 | before_deploy: 26 | - pushd client/python 27 | deploy: 28 | provider: pypi 29 | user: alband 30 | password: 31 | secure: HmwpZ4rVgaGghIZsAeLu5+j/khrInhqjS1eRRvG/fktXjV9H94wlAxf3l3o7rlazexOynHke513xm/IERU/OT8ZcPFMLcjCdDCqVuHmurMph2y0je2XsW58ekjEcCGQ1zg/UAWpCMRrHZIm0G0gBtiuq7BeQTaJW3kE8vSlxr++GcGsQiEk0traZqmQLbvvGfFpHCazT/+kltJQPQQqi0m6rdZV1ayAKESxs9trQYzh99D9U2R63hbNhVDhOSTlXztRuafJpe356wCohQMGCst8wY8mHwhkqiJQbfuMPnWxnJ5JqcYeAKeFjqCL19hTQnDSKRHO0iy+oJQ2Xb411/hmZ5SVow6mfhQmhvaiSEZ+zll/QSGAXhkXaQLre/lFxdKtbHgU1jZtrRu/tiNrkkhDzW5vMhsRAMkZlz3RKc2Q95UTWKtyNXdSse9awLLkMPwHwLDxWKRPumXxyiPUR0UkxSxyj95FfP/qNK5VZpjIORW5SN+8CM4ByRCdetoqfzTdb6cZ5ZMUvLAlIkUkBvtQggz4eEkzYYTRaZpVRWmz8z+7nvhER/lOwS51tCrzfcNUHMmMou5cuuQSnUMPpP7xge3SB1HBKkSVrcmz3EZpk68sTenuXi5AH+WiepytCy2rSPbYS4bPCdQyywXS5hSQutL+Ecl+YRYnTy+Yy2nY= 32 | skip_upload_docs: true 33 | on: 34 | tags: true 35 | branch: master 36 | condition: $TRAVIS_PYTHON_VERSION = "2.7" 37 | after_deploy: 38 | -popd 39 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | ## Contributing Code 4 | 5 | * Fork the repo. 6 | * Write your contribution. 7 | * Make sure your code follows our coding style (e.g.Python code should follow PEP8). 8 | * Run tests. For instance, if you are editing the server or the python client: 9 | 10 | ```bash 11 | $ cd client/python 12 | $ $PYTHON_VERSION -m unittest discover test/ 13 | ``` 14 | 15 | * Push to your fork. Write a [good commit message][commit]. Submit a pull request (PR). 16 | 17 | [commit]: http://tbaggery.com/2008/04/19/a-note-about-git-commit-messages.html 18 | 19 | * Wait for your PR to be reviewed by one of the maintainers. 20 | 21 | ## Other details 22 | 23 | ### Bumping versions 24 | 25 | * Edit the `__VERSION__` variable in server code and clients. 26 | * Create a new rockspec: 27 | 28 | ```bash 29 | $ luarocks new_version crayonx.x-1.rockspec y.y-1 \ 30 | https://raw.githubusercontent.com/torrvision/crayon//client/lua/crayon.lua 31 | ``` 32 | 33 | * Commit your changes, and tag commit in the form of `vy.y`. 34 | * Push to master (so that CI will deploy to PyPI). 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Nantas Nardelli 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Crayon [![Build Status](https://travis-ci.org/torrvision/crayon.svg?branch=master)](https://travis-ci.org/torrvision/crayon) [![PyPI](https://img.shields.io/pypi/v/pycrayon.svg)](https://pypi.python.org/pypi/pycrayon/) 2 | 3 | Crayon is a framework that gives you access to the visualisation power 4 | of 5 | [TensorBoard](https://github.com/tensorflow/tensorboard) with 6 | **any language**. Currently it provides a Python and a Lua interface, however 7 | you can easily implement a wrapper around the 8 | provided [RESTful API](doc/specs.md). 9 | 10 | --- 11 | 12 | This system is composed of two parts: 13 | * A server running on a given machine that will be used to display tensorboard 14 | and store all the data. 15 | * A client embedded inside your code that will send the datas to the server. 16 | 17 | Note that the server and the client *do not* have to be on the same machine. 18 | 19 | 20 | ## Install 21 | 22 | ### Server machine 23 | 24 | The machine that will host the server needs to 25 | have [docker](https://www.docker.com/) installed. The server is completely 26 | packaged inside a docker container. To get it, run: 27 | 28 | ```bash 29 | $ docker pull alband/crayon 30 | ``` 31 | 32 | ### Client machine 33 | 34 | The client machine only need to install the client for the required language. 35 | Detailed instructions can be read by nagivating to 36 | their [respective directories](client/). 37 | 38 | TL;DR: 39 | 40 | * Lua / Torch - `$ luarocks install crayon` 41 | * Python 2 - `$ pip install pycrayon` 42 | * Python 3 - `$ pip3 install pycrayon` 43 | 44 | ## Usage 45 | 46 | ### Server machine 47 | 48 | To start the server, run the following: 49 | 50 | ```bash 51 | $ docker run -d -p 8888:8888 -p 8889:8889 --name crayon alband/crayon 52 | ``` 53 | 54 | Tensorboard is now accessible on a browser at `server_machine_address:8888`. The 55 | client should send the data at `server_machine_address:8889`. 56 | 57 | ### Client 58 | 59 | See the documentation for the required language: 60 | 61 | * [Lua](client/lua/README.md#usage-example) 62 | * [Python](client/python/README.md#usage-example) 63 | 64 | -------------------------------------------------------------------------------- /client/lua/README.md: -------------------------------------------------------------------------------- 1 | # `crayon` 2 | 3 | This is the lua client for the crayon package. 4 | 5 | 6 | ## Install 7 | 8 | * From luarocks: 9 | ```bash 10 | $ luarocks install crayon 11 | ``` 12 | 13 | * From source: 14 | ```bash 15 | $ luarocks make 16 | ``` 17 | 18 | ### Dependencies 19 | 20 | * `openssl` 21 | 22 | 23 | #### OpenSSL troubleshooting 24 | 25 | On some distributions / OSs installing `luasec` (which lua-requests depends on) will 26 | fail with some form of this error: 27 | 28 | ``` 29 | Error: Failed installing dependency: https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/lua-requests-1.1-1.src.rock \ 30 | - Failed installing dependency: https://raw.githubusercontent.com/rocks-moonscript-org/moonrocks-mirror/master/luasec-0.6-1.rockspec - Could not find library file for OPENSSL 31 | No file libssl.a in /usr/lib 32 | No file libssl.so in /usr/lib 33 | No file matching libssl.so.* in /usr/lib 34 | ``` 35 | 36 | ##### Linux fix 37 | 38 | To fix this (assuming openssl is already installed): 39 | 40 | ```bash 41 | $ locate libssl.so 42 | [...] 43 | /usr/lib/x86_64-linux-gnu/libssl.so 44 | $ ln -s /usr/lib/x86_64-linux-gnu/libssl.so /usr/lib/libssl.so 45 | ``` 46 | 47 | ##### Mac OS / OS X fix 48 | 49 | Assuming you have Homebrew installed: 50 | 51 | ```bash 52 | $ brew install openssl 53 | $ brew list openssl 54 | [...] 55 | /usr/local//openssl//lib 56 | /usr/local//openssl//bin 57 | $ luarocks install luasec OPENSSL_DIR=/usr/local/$YOUR_PATH_TO_OPEN_SSL/openssl/ 58 | $ luarocks install crayon 59 | ``` 60 | 61 | 62 | ## Testing 63 | 64 | Run: 65 | 66 | ```bash 67 | # Start new test server 68 | $ docker run -d -p 7998:8888 -p 7999:8889 --name crayon_lua_test alband/crayon 69 | 70 | # Run test script 71 | $ lua(jit) test.lua 72 | 73 | # Remove test server 74 | $ docker rm -f crayon_lua_test 75 | ``` 76 | 77 | 78 | ## Usage example 79 | 80 | ```lua 81 | local crayon = require("crayon") 82 | 83 | -- Connect to the server 84 | -- substitute localhost and port with the ones you are using 85 | local cc = crayon.CrayonClient("localhost", 8889) 86 | 87 | -- Create a new experiment 88 | local foo = cc:create_experiment("foo") 89 | 90 | -- Send some scalar values to the server with their time 91 | foo:add_scalar_value("accuracy", 0, 11.3) 92 | foo:add_scalar_value("accuracy", 4, 12.3) 93 | -- You can force the step value also 94 | foo:add_scalar_value("accuracy", 6, 13.3, 4) 95 | 96 | -- Get the datas sent to the server 97 | foo:get_scalar_values("accuracy") 98 | -- >> { 99 | -- 1 : 100 | -- { 101 | -- 1 : 11.3 102 | -- 2 : 0 103 | -- 3 : 0 104 | -- } 105 | -- 2 : 106 | -- { 107 | -- 1 : 12.3 108 | -- 2 : 1 109 | -- 3 : 4 110 | -- } 111 | -- 3 : 112 | -- { 113 | -- 1 : 13.3 114 | -- 2 : 4 115 | -- 3 : 6 116 | -- } 117 | -- } 118 | 119 | -- backup this experiment as a zip file 120 | local filename = foo:to_zip() 121 | 122 | -- delete this experiment from the server 123 | cc:remove_experiment("foo") 124 | -- using the `foo` object from now on will result in an error 125 | 126 | -- Create a new experiment based on foo's backup 127 | local bar = cc:create_experiment("bar", filename) 128 | 129 | -- Get the name of all scalar plots in this experiment 130 | bar:get_scalar_names() 131 | -- >> { 132 | -- 1 : "accuracy" 133 | -- } 134 | 135 | -- Get the data for this experiment 136 | bar:get_scalar_values("accuracy") 137 | -- >> { 138 | -- 1 : 139 | -- { 140 | -- 1 : 11.3 141 | -- 2 : 0 142 | -- 3 : 0 143 | -- } 144 | -- 2 : 145 | -- { 146 | -- 1 : 12.3 147 | -- 2 : 1 148 | -- 3 : 4 149 | -- } 150 | -- 3 : 151 | -- { 152 | -- 1 : 13.3 153 | -- 2 : 4 154 | -- 3 : 6 155 | -- } 156 | -- } 157 | ``` 158 | 159 | 160 | ## Complete API 161 | 162 | ### `CrayonClient` 163 | 164 | * Creation: `CrayonClient(hostname="localhost", port=8889)` 165 | * Create a client object and connect it to the server at address `hostname` and port `port`. 166 | 167 | * `get_experiment_names()` 168 | * Returns a list of string containing the name of all the experiments on the server. 169 | 170 | * `create_experiment(xp_name, zip_file=nil)` 171 | * Creates a new experiment with name `xp_name` and returns a `CrayonExperiment` object. 172 | * If `zip_file` is provided, this experiment is initialized with the content of the zip file (see `CrayonExperiment.to_zip` to get the zip file). 173 | 174 | * `open_experiment(xp_name)` 175 | * Opens the experiment called `xp_name` that already exists on the server. 176 | 177 | * `remove_experiment(xp_name)` 178 | * Removes the experiment `xp_name` from the server. 179 | * WARNING: all elements from this experiment are permanently lost! 180 | 181 | * `remove_all_experiments()` 182 | * Removes all experiment from the server. 183 | * WARNING: all elements from all experiments are permanently lost! 184 | 185 | 186 | ### `CrayonExperiment` 187 | 188 | * Creation: can only be created by the `CrayonClient` 189 | 190 | * `get_scalar_names()` 191 | * Returns a list of string containing the name of all the scalar values in this experiment. 192 | 193 | * `add_scalar_value(name, value, wall_time=-1, step=-1)` 194 | * Adds a new point with value `value` to the scalar plot named `name`. 195 | * If not specified, the `wall_time` will be set to the current time and the `step` to the step of the previous point with this name plus one (or `0` if its the first point with this name). 196 | 197 | * `add_scalar_dict(data, wall_time=-1, step=-1)` 198 | * Add multiple points at the same times where `data` is a dictionary where each key is a scalar name and the associated value the value to add for this scalar plot. 199 | * `wall_time` and `step` are handled the same as for `add_scalar_value` for each entry independently. 200 | 201 | * `get_scalar_values(name)` 202 | * Return a list with one entry for each point added for this scalar plot. 203 | * Each entry is a list containing [wall_time, step, value]. 204 | 205 | * `get_histogram_names()` 206 | * Returns a list of string containing the name of all the histogram values in this experiment. 207 | 208 | * `add_histogram_value(name, hist, tobuild=false, wall_time=-1, step=-1)` 209 | * Adds a new point with value `hist` to the histogram plot named `name`. 210 | * If `tobuild` is `false`, `hist` should be a dictionary containing: `{"min": minimum value, "max": maximum value, "num": number of items in the histogram, "bucket_limit": a list with the right limit of each bucket, "bucker": a list with the number of element in each bucket, "sum": optional, the sum of items in the histogram, "sum_squares": optional, the sum of squares of items in the histogram}`. 211 | * If `tobuild` if `True`, `hist` should be a list of value from which an histogram is going to be built. 212 | * If not specified, the `wall_time` will be set to the current time and the `step` to the step of the previous point with this name plus one (or `0` if its the first point with this name). 213 | 214 | * `get_histogram_values(name)` 215 | * Return a list with one entry for each point added for this histogram plot. 216 | * Each entry is a list containing [wall_time, step, hist]. 217 | * Where each `hist` is a dictionary similar to the one specified above. 218 | 219 | * `to_zip(filename=nil)` 220 | * Retrieve all the datas from this experiment from the server and store it in `filename`. If `filename` is not specified, it is saved in the current folder. 221 | * Returns the name of the file where the datas have been saved. 222 | * This file can then be used to recreate a new experiment with the exact same content as this one. 223 | 224 | -------------------------------------------------------------------------------- /client/lua/crayon-0.5-1.rockspec: -------------------------------------------------------------------------------- 1 | package = "crayon" 2 | version = "0.5-1" 3 | source = { 4 | url = "https://raw.githubusercontent.com/torrvision/crayon/7f2f96b6a44b6729cc0f5feae5d09df70b6eba19/client/lua/crayon.lua" 5 | } 6 | description = { 7 | summary = "A lua client for crayon.", 8 | homepage = "https://github.com/torrvision/crayon" 9 | } 10 | dependencies = { 11 | "lua-requests" 12 | } 13 | build = { 14 | type = "builtin", 15 | modules = { 16 | crayon = "crayon.lua" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /client/lua/crayon.lua: -------------------------------------------------------------------------------- 1 | local requests = require("requests") 2 | local socket = require("socket") 3 | local json = require('cjson.safe') 4 | 5 | local __version__ = 0.5 6 | 7 | local CrayonClient = {} 8 | local CrayonExperiment = {} 9 | 10 | -- CrayonClient class 11 | do 12 | CrayonClient.__index = CrayonClient 13 | 14 | local mt = {} 15 | mt.__call = function(cc, hostname, port) 16 | local self = {} 17 | setmetatable(self, CrayonClient) 18 | 19 | self.hostname = hostname or "localhost" 20 | self.port = port or 8889 21 | self.url = self.hostname .. ":" .. tostring(self.port) 22 | 23 | -- Add http header if missing 24 | function urlstartswith(pattern) 25 | return string.sub(self.url,1,string.len(pattern))==pattern 26 | end 27 | if not (urlstartswith("http://") or urlstartswith("https://")) then 28 | self.url = "http://" .. self.url 29 | end 30 | 31 | -- Check that the server is running and at the same version 32 | local ok, r = pcall(requests.get, self.url) 33 | if not ok then 34 | msg = "The server at " .. self.hostname .. ":" 35 | .. tostring(self.port) .. " does not appear to be up!" 36 | error(msg) 37 | end 38 | if r.status_code ~= 200 then 39 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 40 | error(msg) 41 | end 42 | server_version = tonumber(r.text) 43 | 44 | if not server_version then 45 | local msg = "The page at " .. self.hostname .. ":" 46 | .. tostring(self.port) .. " doesn't seem to be a Crayon server!" 47 | error(msg) 48 | end 49 | if server_version ~= __version__ then 50 | local msg = "Initialised client version " .. __version__ 51 | .. ", however found server running version " .. server_version .. "." 52 | error(msg) 53 | end 54 | 55 | return self 56 | end 57 | setmetatable(CrayonClient, mt) 58 | 59 | function CrayonClient:get_experiment_names() 60 | local query = "/data" 61 | local r = requests.get(self.url..query) 62 | if r.status_code ~= 200 then 63 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 64 | error(msg) 65 | end 66 | return r.json() 67 | end 68 | 69 | function CrayonClient:open_experiment(xp_name) 70 | assert(type(xp_name)=="string", "Expected a string, but got " 71 | .. type(xp_name) .. ".") 72 | return CrayonExperiment(xp_name, self, false) 73 | end 74 | 75 | function CrayonClient:create_experiment(xp_name, zip_file) 76 | assert(type(xp_name)=="string", "Expected a string, but got " 77 | .. type(xp_name) .. ".") 78 | return CrayonExperiment(xp_name, self, true, zip_file) 79 | end 80 | 81 | function CrayonClient:remove_experiment(xp_name) 82 | assert(type(xp_name)=="string", "Expected a string, but got " 83 | .. type(xp_name) .. ".") 84 | query = "/data" 85 | local r = requests.delete{ 86 | url = self.url..query, 87 | params = {xp = xp_name} 88 | } 89 | if r.status_code ~= 200 then 90 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 91 | error(msg) 92 | end 93 | end 94 | 95 | function CrayonClient:remove_all_experiments() 96 | local xp_list = self:get_experiment_names() 97 | for i, xp_name in pairs(xp_list) do 98 | self:remove_experiment(xp_name) 99 | end 100 | end 101 | end 102 | 103 | -- CrayonExperiment class 104 | do 105 | -- Local functions 106 | local __init_from_file, __init_empty, __init_from_existing 107 | local __update_steps, __get_name_list 108 | local __check_histogram_data 109 | 110 | local json_headers = {} 111 | json_headers["Content-Type"] = "application/json" 112 | local zip_headers = {} 113 | zip_headers["Content-Type"] = "application/zip" 114 | 115 | 116 | 117 | CrayonExperiment.__index = CrayonExperiment 118 | 119 | local mt = {} 120 | mt.__call = function(ce, xp_name, client, create, zip_file) 121 | local self = {} 122 | setmetatable(self, CrayonExperiment) 123 | 124 | self.client = client 125 | self.xp_name = xp_name 126 | self.scalar_steps = {} 127 | self.hist_steps = {} 128 | local mt = {__index = function() return 0 end} 129 | setmetatable(self.scalar_steps, mt) 130 | setmetatable(self.hist_steps, mt) 131 | 132 | if zip_file then 133 | if not create then 134 | error("Can only create a new experiment when a zip_file is provided") 135 | end 136 | __init_from_file(self, zip_file, true) 137 | elseif create then 138 | __init_empty(self) 139 | else 140 | __init_from_existing(self) 141 | end 142 | return self 143 | end 144 | setmetatable(CrayonExperiment, mt) 145 | 146 | -- Initialisation 147 | __init_empty = function(self) 148 | local query = "/data" 149 | local r = requests.post{ 150 | url = self.client.url..query, 151 | data = json.encode(self.xp_name), 152 | headers = json_headers 153 | } 154 | if r.status_code ~= 200 then 155 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 156 | error(msg) 157 | end 158 | end 159 | 160 | __init_from_existing = function(self) 161 | local query = "/data" 162 | local r = requests.get{ 163 | url = self.client.url..query, 164 | params = {xp = self.xp_name} 165 | } 166 | if r.status_code ~= 200 then 167 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 168 | error(msg) 169 | end 170 | local content = r.json() 171 | __update_steps(content["scalars"], self.scalar_steps, self.get_scalar_values) 172 | __update_steps(content["histograms"], self.hist_steps, self.get_histogram_values) 173 | end 174 | 175 | __init_from_file = function(self, zip_file, force) 176 | local file = io.open(zip_file, "rb") 177 | local content = file:read("*all") 178 | assert(file, "File cannot be opened. Check it exists!") 179 | file:close() 180 | 181 | local query = "/backup" 182 | local r = requests.post{ 183 | url = self.client.url..query, 184 | params = {xp=self.xp_name, force=force}, 185 | data = content, 186 | headers = zip_headers 187 | } 188 | if r.status_code ~= 200 then 189 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 190 | error(msg) 191 | end 192 | end 193 | 194 | -- Scalar methods 195 | function CrayonExperiment:get_scalar_names() 196 | return __get_name_list(self, "scalars") 197 | end 198 | 199 | function CrayonExperiment:add_scalar_value(name, value, wall_time, step) 200 | if not wall_time then 201 | wall_time = socket.gettime() 202 | end 203 | if not step then 204 | step = self.scalar_steps[name] 205 | end 206 | self.scalar_steps[name] = step + 1 207 | 208 | local query = "/data/scalars" 209 | local r = requests.post{ 210 | url = self.client.url..query, 211 | params = {xp=self.xp_name, name=name}, 212 | data = {wall_time, step, value}, 213 | headers = json_headers 214 | } 215 | if r.status_code ~= 200 then 216 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 217 | error(msg) 218 | end 219 | end 220 | 221 | function CrayonExperiment:add_scalar_dict(data, wall_time, step) 222 | for name, value in pairs(data) do 223 | assert(type(name)=="string", "Expected a string, but got " 224 | .. type(name) .. ".") 225 | self.add_scalar_value(name, value, wall_time, step) 226 | end 227 | end 228 | 229 | function CrayonExperiment:get_scalar_values(name) 230 | query = "/data/scalars" 231 | local r = requests.get{ 232 | url = self.client.url..query, 233 | params = {xp=self.xp_name, name=name} 234 | } 235 | if r.status_code ~= 200 then 236 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 237 | error(msg) 238 | end 239 | return r.json() 240 | end 241 | 242 | -- Histogram methods 243 | function CrayonExperiment:get_histogram_names() 244 | return __get_name_list(self, "histograms") 245 | end 246 | 247 | function CrayonExperiment:add_histogram_value(name, hist, tobuild, wall_time, step) 248 | if not wall_time then 249 | wall_time = socket.gettime() 250 | end 251 | if not step then 252 | step = self.hist_steps[name] 253 | end 254 | self.hist_steps[name] = step + 1 255 | 256 | __check_histogram_data(hist, tobuild) 257 | 258 | local query = "/data/histograms" 259 | local r = requests.post{ 260 | url = self.client.url..query, 261 | params = {xp=self.xp_name, name=name}, 262 | data = {wall_time, step, hist}, 263 | headers = json_headers 264 | } 265 | if r.status_code ~= 200 then 266 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 267 | error(msg) 268 | end 269 | end 270 | 271 | function CrayonExperiment:get_histogram_values(name) 272 | query = "/data/histograms" 273 | local r = requests.get{ 274 | url = self.client.url..query, 275 | params = {xp=self.xp_name, name=name} 276 | } 277 | if r.status_code ~= 200 then 278 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 279 | error(msg) 280 | end 281 | return r.json() 282 | end 283 | 284 | __check_histogram_data = function(data, tobuild) 285 | local required = {bucket=1, bucket_limit=1, max=1, min=1, num=1} 286 | local optionnal = {sum=1, sum_squares=1} 287 | 288 | if tobuild then 289 | if #data == 0 then 290 | error("When building the histogram should be provided as a list in a table.") 291 | end 292 | else 293 | for key,_ in pairs(required) do 294 | if not data[key] then 295 | error("Built histogram is missing argument: " .. key) 296 | end 297 | end 298 | for key,_ in pairs(data) do 299 | if (not required[key]) and (not optionnal[key]) then 300 | error("Built histogram has extra parameter: " .. key) 301 | end 302 | end 303 | end 304 | end 305 | 306 | -- Backup methods 307 | function CrayonExperiment:to_zip(filename) 308 | filename = filename or "backup_" .. self.xp_name .. "_" .. tostring(socket.gettime()) .. ".zip" 309 | local query = "/backup" 310 | local r = requests.get{ 311 | url = self.client.url..query, 312 | params = {xp=self.xp_name} 313 | } 314 | if r.status_code ~= 200 then 315 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 316 | error(msg) 317 | end 318 | 319 | local f = io.open(filename, "w") 320 | f:write(r.text) 321 | f:close() 322 | 323 | return filename 324 | end 325 | 326 | -- helper methods 327 | __get_name_list = function(self, element_type) 328 | local query = "/data" 329 | local r = requests.get{ 330 | url = self.client.url..query, 331 | params = {xp=self.xp_name} 332 | } 333 | if r.status_code ~= 200 then 334 | local msg = "Something went wrong. Server sent: " .. r.text .. "." 335 | error(msg) 336 | end 337 | return r.json()[element_type] 338 | end 339 | 340 | __update_steps = function(self, elements, steps_table, eval_function) 341 | for _, element in pairs(elements) do 342 | print("element", element) 343 | values = eval_function(element) 344 | print("values", values) 345 | if #values > 0 then 346 | steps_table[element] = values[#values][1] + 1 347 | end 348 | end 349 | end 350 | end 351 | 352 | return { 353 | CrayonClient= CrayonClient 354 | } 355 | -------------------------------------------------------------------------------- /client/lua/test.lua: -------------------------------------------------------------------------------- 1 | local crayon = require("crayon") 2 | 3 | 4 | -- A clean server should be running on the test_port with: 5 | -- docker run -it -p 7998:8888 -p 7999:8889 --name crayon_lua_test alband/crayon 6 | local test_port = 7999 7 | 8 | local cc = crayon.CrayonClient("localhost", test_port) 9 | 10 | -- Check empty 11 | local xp_list = cc:get_experiment_names() 12 | for k,v in pairs(xp_list) do 13 | error("The server should be empty") 14 | end 15 | 16 | -- Check create / add scalar 17 | local foo = cc:create_experiment("foo") 18 | foo:add_scalar_value("bar", 3) 19 | foo:add_scalar_value("bar", 4, 123) 20 | xp_list = cc:get_experiment_names() 21 | for k,v in pairs(xp_list) do 22 | if v ~= "foo" then 23 | error("The server should not contain xp: "..v) 24 | end 25 | end 26 | local bar_values = foo:get_scalar_values("bar") 27 | for i,v in ipairs(bar_values) do 28 | if i==2 and v[1] ~= 123 then 29 | error("wall_time was not set properly") 30 | end 31 | if v[2] ~= i-1 then 32 | error("step was not incremented properly") 33 | end 34 | if v[3] ~= i+2 then 35 | error("value was not set properly") 36 | end 37 | end 38 | 39 | -- Check open 40 | local foo_bis = cc:open_experiment("foo") 41 | for k,v in pairs(xp_list) do 42 | if v ~= "foo" then 43 | error("The server should not contain xp: "..v) 44 | end 45 | end 46 | local bar_values = foo_bis:get_scalar_values("bar") 47 | for i,v in ipairs(bar_values) do 48 | if i==2 and v[1] ~= 123 then 49 | error("wall_time was not set properly") 50 | end 51 | if v[2] ~= i-1 then 52 | error("step was not incremented properly") 53 | end 54 | if v[3] ~= i+2 then 55 | error("value was not set properly") 56 | end 57 | end 58 | 59 | -- Check get backup 60 | local zip_file = "back.zip" 61 | foo:to_zip(zip_file) 62 | 63 | -- Check upload backup 64 | local foo2 = cc:create_experiment("foo2", zip_file) 65 | for k,v in pairs(xp_list) do 66 | if v ~= "foo" and v~= "foo2" then 67 | error("The server should not contain xp: "..v) 68 | end 69 | end 70 | local bar_values = foo2:get_scalar_values("bar") 71 | for i,v in ipairs(bar_values) do 72 | if i==2 and v[1] ~= 123 then 73 | error("wall_time was not set properly") 74 | end 75 | if v[2] ~= i-1 then 76 | error("step was not incremented properly") 77 | end 78 | if v[3] ~= i+2 then 79 | error("value was not set properly") 80 | end 81 | end 82 | 83 | os.remove(zip_file) 84 | 85 | print("Success !") 86 | -------------------------------------------------------------------------------- /client/python/MANIFEST: -------------------------------------------------------------------------------- 1 | # file GENERATED by distutils, do NOT edit 2 | setup.py 3 | pycrayon/__init__.py 4 | pycrayon/crayon.py 5 | pycrayon/version.py 6 | test/test_crayon.py 7 | -------------------------------------------------------------------------------- /client/python/README.md: -------------------------------------------------------------------------------- 1 | # [`pycrayon`](https://pypi.python.org/pypi/pycrayon) 2 | 3 | This is the python client for the crayon package. 4 | 5 | ## Install 6 | 7 | * From pip: 8 | ```bash 9 | $ pip install pycrayon 10 | ``` 11 | 12 | * From source: 13 | ```bash 14 | $ python setup.py install 15 | ``` 16 | 17 | ## Testing 18 | 19 | Run: 20 | 21 | ```bash 22 | $ python -m unittest discover 23 | ``` 24 | 25 | ## Usage example 26 | 27 | ```python 28 | from pycrayon import CrayonClient 29 | import time 30 | 31 | # Connect to the server 32 | cc = CrayonClient(hostname="server_machine_address") 33 | 34 | # Create a new experiment 35 | foo = cc.create_experiment("foo") 36 | 37 | # Send some scalar values to the server 38 | foo.add_scalar_value("accuracy", 0, wall_time=11.3) 39 | foo.add_scalar_value("accuracy", 4, wall_time=12.3) 40 | # You can force the time and step values 41 | foo.add_scalar_value("accuracy", 6, wall_time=13.3, step=4) 42 | 43 | # Get the datas sent to the server 44 | foo.get_scalar_values("accuracy") 45 | #>> [[11.3, 0, 0.0], [12.3, 1, 4.0], [13.3, 4, 6.0]]) 46 | 47 | # backup this experiment as a zip file 48 | filename = foo.to_zip() 49 | 50 | # delete this experiment from the server 51 | cc.remove_experiment("foo") 52 | # using the `foo` object from now on will result in an error 53 | 54 | # Create a new experiment based on foo's backup 55 | bar = cc.create_experiment("bar", zip_file=filename) 56 | 57 | # Get the name of all scalar plots in this experiment 58 | bar.get_scalar_names() 59 | #>> ["accuracy"] 60 | 61 | # Get the data for this experiment 62 | bar.get_scalar_values("accuracy") 63 | #>> [[11.3, 0, 0.0], [12.3, 1, 4.0], [13.3, 4, 6.0]]) 64 | ``` 65 | 66 | ## Complete API 67 | 68 | ### `CrayonClient` 69 | 70 | * Creation: `CrayonClient(hostname="localhost", port=8889)` 71 | * Create a client object and connect it to the server at address `hostname` and port `port`. 72 | 73 | * `get_experiment_names()` 74 | * Returns a list of string containing the name of all the experiments on the server. 75 | 76 | * `create_experiment(xp_name, zip_file=None)` 77 | * Creates a new experiment with name `xp_name` and returns a `CrayonExperiment` object. 78 | * If `zip_file` is provided, this experiment is initialized with the content of the zip file (see `CrayonExperiment.to_zip` to get the zip file). 79 | 80 | * `open_experiment(xp_name)` 81 | * Opens the experiment called `xp_name` that already exists on the server. 82 | 83 | * `remove_experiment(xp_name)` 84 | * Removes the experiment `xp_name` from the server. 85 | * WARNING: all elements from this experiment are permanently lost! 86 | 87 | * `remove_all_experiments()` 88 | * Removes all experiment from the server. 89 | * WARNING: all elements from all experiments are permanently lost! 90 | 91 | 92 | ### `CrayonExperiment` 93 | 94 | * Creation: can only be created by the `CrayonClient` 95 | 96 | * `get_scalar_names()` 97 | * Returns a list of string containing the name of all the scalar values in this experiment. 98 | 99 | * `add_scalar_value(name, value, wall_time=-1, step=-1)` 100 | * Adds a new point with value `value` to the scalar plot named `name`. 101 | * If not specified, the `wall_time` will be set to the current time and the `step` to the step of the previous point with this name plus one (or `0` if its the first point with this name). 102 | 103 | * `add_scalar_dict(data, wall_time=-1, step=-1)` 104 | * Add multiple points at the same times where `data` is a dictionary where each key is a scalar name and the associated value the value to add for this scalar plot. 105 | * `wall_time` and `step` are handled the same as for `add_scalar_value` for each entry independently. 106 | 107 | * `get_scalar_values(name)` 108 | * Return a list with one entry for each point added for this scalar plot. 109 | * Each entry is a list containing [wall_time, step, value]. 110 | 111 | * `get_histogram_names()` 112 | * Returns a list of string containing the name of all the histogram values in this experiment. 113 | 114 | * `add_histogram_value(name, hist, tobuild=False, wall_time=-1, step=-1)` 115 | * Adds a new point with value `hist` to the histogram plot named `name`. 116 | * If `tobuild` is `False`, `hist` should be a dictionary containing: `{"min": minimum value, "max": maximum value, "num": number of items in the histogram, "bucket_limit": a list with the right limit of each bucket, "bucker": a list with the number of element in each bucket, "sum": optional, the sum of items in the histogram, "sum_squares": optional, the sum of squares of items in the histogram}`. 117 | * If `tobuild` if `True`, `hist` should be a list of value from which an histogram is going to be built. 118 | * If not specified, the `wall_time` will be set to the current time and the `step` to the step of the previous point with this name plus one (or `0` if its the first point with this name). 119 | 120 | * `get_histogram_values(name)` 121 | * Return a list with one entry for each point added for this histogram plot. 122 | * Each entry is a list containing [wall_time, step, hist]. 123 | * Where each `hist` is a dictionary similar to the one specified above. 124 | 125 | * `to_zip(filename=None)` 126 | * Retrieve all the datas from this experiment from the server and store it in `filename`. If `filename` is not specified, it is saved in the current folder. 127 | * Returns the name of the file where the datas have been saved. 128 | * This file can then be used to recreate a new experiment with the exact same content as this one. 129 | 130 | -------------------------------------------------------------------------------- /client/python/pycrayon/__init__.py: -------------------------------------------------------------------------------- 1 | from .crayon import CrayonClient -------------------------------------------------------------------------------- /client/python/pycrayon/crayon.py: -------------------------------------------------------------------------------- 1 | import requests 2 | import json 3 | import time 4 | import collections 5 | 6 | try: 7 | # Python 2 8 | from urllib import quote_plus 9 | except ImportError: 10 | # Python 3 11 | from urllib.parse import quote_plus 12 | 13 | from .version import __version__ 14 | 15 | try: 16 | basestring 17 | except NameError: 18 | basestring = str 19 | 20 | 21 | class CrayonClient(object): 22 | def __init__(self, hostname="localhost", port=8889): 23 | self.hostname = hostname 24 | self.port = port 25 | self.url = self.hostname + ":" + str(self.port) 26 | # TODO use urlparse 27 | if not (self.url.startswith("http://") or 28 | self.url.startswith("https://")): 29 | self.url = "http://" + self.url 30 | 31 | # check server is working (not only up). 32 | try: 33 | r = requests.get(self.url) 34 | if not r.ok: 35 | raise RuntimeError("Something went wrong!" + 36 | " Server sent: {}.".format(r.text)) 37 | if not r.text == __version__: 38 | msg = "Initialised client version {}, however found " 39 | msg += "server running version {}." 40 | raise RuntimeError(msg.format(r.text, __version__)) 41 | 42 | except requests.ConnectionError: 43 | msg = "The server at {}:{} does not appear to be up!" 44 | raise ValueError(msg.format(self.hostname, self.port)) 45 | 46 | def get_experiment_names(self): 47 | query = "/data" 48 | r = requests.get(self.url + query) 49 | if not r.ok: 50 | msg = "Something went wrong. Server sent: {}." 51 | raise ValueError(msg.format(r.text)) 52 | else: 53 | experiments = json.loads(r.text) 54 | return experiments 55 | 56 | def open_experiment(self, xp_name): 57 | assert(isinstance(xp_name, basestring)) 58 | return CrayonExperiment(xp_name, self, create=False) 59 | 60 | def create_experiment(self, xp_name, zip_file=None): 61 | assert(isinstance(xp_name, basestring)) 62 | return CrayonExperiment(xp_name, self, zip_file=zip_file, create=True) 63 | 64 | def remove_experiment(self, xp_name): 65 | assert(isinstance(xp_name, basestring)) 66 | query = "/data?xp={}".format(quote_plus(xp_name)) 67 | r = requests.delete(self.url + query) 68 | 69 | if not r.ok: 70 | msg = "Something went wrong. Server sent: {}." 71 | raise ValueError(msg.format(r.text)) 72 | 73 | def remove_all_experiments(self): 74 | xp_list = self.get_experiment_names() 75 | for xp_name in xp_list: 76 | self.remove_experiment(xp_name) 77 | 78 | 79 | class CrayonExperiment(object): 80 | 81 | def __init__(self, xp_name, client, zip_file=None, create=False): 82 | self.client = client 83 | self.xp_name = xp_name 84 | self.scalar_steps = collections.defaultdict(int) 85 | self.hist_steps = collections.defaultdict(int) 86 | 87 | if zip_file: 88 | if not create: 89 | msg = "Can only create a new experiment when " 90 | msg += "a zip_file is provided" 91 | raise ValueError(msg) 92 | self.__init_from_file(zip_file, True) 93 | 94 | elif create: 95 | self.__init_empty() 96 | 97 | else: 98 | self.__init_from_existing() 99 | 100 | # Initialisations 101 | def __init_empty(self): 102 | query = "/data" 103 | r = requests.post(self.client.url + query, json=self.xp_name) 104 | 105 | if not r.ok: 106 | msg = "Something went wrong. Server sent: {}." 107 | raise ValueError(msg.format(r.text)) 108 | 109 | def __init_from_existing(self): 110 | query = "/data?xp={}".format(quote_plus(self.xp_name)) 111 | r = requests.get(self.client.url + query) 112 | 113 | if not r.ok: 114 | msg = "Something went wrong. Server sent: {}." 115 | raise ValueError(msg.format(r.text)) 116 | 117 | # Retrieve the current step for existing metrics 118 | content = json.loads(r.text) 119 | self.__update_steps(content["scalars"], 120 | self.scalar_steps, 121 | self.get_scalar_values) 122 | self.__update_steps(content["histograms"], 123 | self.hist_steps, 124 | self.get_histogram_values) 125 | 126 | def __init_from_file(self, zip_file, force=False): 127 | query = "/backup?xp={}&force={}".format( 128 | quote_plus(self.xp_name), force) 129 | fileobj = open(zip_file, 'rb') 130 | r = requests.post(self.client.url + query, data={"mysubmit": "Go"}, 131 | files={"archive": ("backup.zip", fileobj)}) 132 | fileobj.close() 133 | 134 | if not r.ok: 135 | msg = "Something went wrong. Server sent: {}." 136 | raise ValueError(msg.format(r.text)) 137 | 138 | # Scalar methods 139 | def get_scalar_names(self): 140 | return self.__get_name_list("scalars") 141 | 142 | def add_scalar_value(self, name, value, wall_time=-1, step=-1): 143 | if wall_time == -1: 144 | wall_time = time.time() 145 | if step == -1: 146 | step = self.scalar_steps[name] 147 | self.scalar_steps[name] += 1 148 | else: 149 | self.scalar_steps[name] = step + 1 150 | query = "/data/scalars?xp={}&name={}".format(quote_plus(self.xp_name), quote_plus(name)) 151 | data = [wall_time, step, value] 152 | r = requests.post(self.client.url + query, json=data) 153 | 154 | if not r.ok: 155 | msg = "Something went wrong. Server sent: {}." 156 | raise ValueError(msg.format(r.text)) 157 | 158 | def add_scalar_dict(self, data, wall_time=-1, step=-1): 159 | for name, value in data.items(): 160 | if not isinstance(name, basestring): 161 | msg = "Scalar name should be a string, got: {}.".format(name) 162 | raise ValueError(msg) 163 | self.add_scalar_value(name, value, wall_time, step) 164 | 165 | def get_scalar_values(self, name): 166 | query = "/data/scalars?xp={}&name={}".format(quote_plus(self.xp_name), quote_plus(name)) 167 | 168 | r = requests.get(self.client.url + query) 169 | 170 | if not r.ok: 171 | msg = "Something went wrong. Server sent: {}." 172 | raise ValueError(msg.format(r.text)) 173 | 174 | return json.loads(r.text) 175 | 176 | # Histogram methods 177 | def get_histogram_names(self): 178 | return self.__get_name_list("histograms") 179 | 180 | def add_histogram_value(self, name, hist, tobuild=False, 181 | wall_time=-1, step=-1): 182 | if wall_time == -1: 183 | wall_time = time.time() 184 | if step == -1: 185 | step = self.scalar_steps[name] 186 | self.scalar_steps[name] += 1 187 | else: 188 | self.scalar_steps[name] = step 189 | 190 | if not tobuild and (not isinstance(hist, dict) 191 | or not self.__check_histogram_data(hist, tobuild)): 192 | raise ValueError("Data was not provided in a valid format!") 193 | 194 | if tobuild and (not isinstance(hist, list)): 195 | raise ValueError("Data was not provided in a valid format!") 196 | 197 | query = "/data/histograms?xp={}&name={}&tobuild={}".format( 198 | quote_plus(self.xp_name), quote_plus(name), tobuild) 199 | 200 | data = [wall_time, step, hist] 201 | r = requests.post(self.client.url + query, json=data) 202 | if not r.ok: 203 | raise ValueError( 204 | "Something went wrong. Server sent: {}.".format(r.text) 205 | ) 206 | 207 | def get_histogram_values(self, name): 208 | query = "/data/histograms?xp={}&name={}".format(quote_plus(self.xp_name), quote_plus(name)) 209 | r = requests.get(self.client.url + query) 210 | 211 | if not r.ok: 212 | msg = "Something went wrong. Server sent: {}." 213 | raise ValueError(msg.format(r.text)) 214 | 215 | return json.loads(r.text) 216 | 217 | def __check_histogram_data(self, data, tobuild): 218 | # TODO should use a schema here 219 | # Note: all of these are sorted already 220 | 221 | expected = ["bucket", "bucket_limit", "max", "min", "num"] 222 | expected2 = ["bucket", "bucket_limit", "max", "min", "num", 223 | "sum"] 224 | expected3 = ["bucket", "bucket_limit", "max", "min", "num", 225 | "sum", "sum_squares"] 226 | expected4 = ["bucket", "bucket_limit", "max", "min", "num", 227 | "sum_squares"] 228 | ks = tuple(data.keys()) 229 | ks = sorted(ks) 230 | return (ks == expected or ks == expected2 231 | or ks == expected3 or ks == expected4) 232 | 233 | # Backup methods 234 | def to_zip(self, filename=None): 235 | query = "/backup?xp={}".format(quote_plus(self.xp_name)) 236 | r = requests.get(self.client.url + query) 237 | 238 | if not r.ok: 239 | msg = "Something went wrong. Server sent: {}." 240 | raise ValueError(msg.format(r.text)) 241 | 242 | if not filename: 243 | filename = "backup_" + self.xp_name + "_" + str(time.time()) 244 | out = open(filename + ".zip", "wb") 245 | out.write(r.content) 246 | out.close() 247 | return filename + ".zip" 248 | 249 | # Helper methods 250 | def __get_name_list(self, element_type): 251 | query = "/data?xp={}".format(quote_plus(self.xp_name)) 252 | r = requests.get(self.client.url + query) 253 | 254 | if not r.ok: 255 | msg = "Something went wrong. Server sent: {}." 256 | raise ValueError(msg.format(r.text)) 257 | 258 | return json.loads(r.text)[element_type] 259 | 260 | def __update_steps(self, elements, steps_table, eval_function): 261 | for element in elements: 262 | values = eval_function(element) 263 | if len(values) > 0: 264 | steps_table[element] = values[-1][1] + 1 265 | -------------------------------------------------------------------------------- /client/python/pycrayon/version.py: -------------------------------------------------------------------------------- 1 | r""" 2 | Little utility to reveal the package version. 3 | Place in the root dir of the package. 4 | """ 5 | from pkg_resources import get_distribution 6 | 7 | 8 | __version__ = get_distribution(__name__.split('.')[0]).version 9 | -------------------------------------------------------------------------------- /client/python/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup(name='pycrayon', 4 | description='Crayon client for python', 5 | author='torrvision', 6 | url='https://github.com/torrvision/crayon', 7 | packages=['pycrayon'], 8 | version='0.5', 9 | install_requires=[ 10 | "requests" 11 | ] 12 | ) -------------------------------------------------------------------------------- /client/python/test/helper.py: -------------------------------------------------------------------------------- 1 | import time 2 | import requests 3 | try: 4 | import docker 5 | except: 6 | RuntimeError("Please run 'pip install docker' before using this module.") 7 | 8 | 9 | class Helper(object): 10 | 11 | def __init__(self, start=True, tb_ip=8888, server_ip=8889, name="crayon"): 12 | self.client = docker.from_env() 13 | self.tb_ip = tb_ip 14 | self.server_ip = server_ip 15 | self.name = name 16 | if start: 17 | self.start() 18 | 19 | def start(self): 20 | self.container = self.client.containers.run( 21 | "alband/crayon:latest", 22 | ports={8888: self.tb_ip, 23 | 8889: self.server_ip}, 24 | detach=True, 25 | name=self.name) 26 | # check server is working 27 | running = False 28 | retry = 50 29 | while not running: 30 | try: 31 | assert( 32 | requests.get("http://localhost:" + str(self.server_ip)).ok 33 | ) 34 | running = True 35 | except: 36 | retry -= 1 37 | if retry == 0: 38 | # The test will trigger the not running server error 39 | return 40 | time.sleep(0.1) 41 | 42 | def kill(self): 43 | if hasattr(self, "container"): 44 | self.container.kill() 45 | 46 | def remove(self): 47 | if hasattr(self, "container"): 48 | self.container.remove() 49 | self.container = None 50 | 51 | def kill_remove(self): 52 | # Could do with remove -f too 53 | self.kill() 54 | self.remove() 55 | -------------------------------------------------------------------------------- /client/python/test/test_crayon.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import time 3 | import os 4 | 5 | from pycrayon import CrayonClient 6 | from helper import Helper 7 | 8 | 9 | class CrayonClientTestSuite(unittest.TestCase): 10 | 11 | def __init__(self, *args, **kwargs): 12 | super(CrayonClientTestSuite, self).__init__(*args, **kwargs) 13 | self.test_server_port = 8886 14 | self.test_tb_port = 8887 15 | self.container_name = "crayon_test_python" 16 | 17 | def setUp(self): 18 | self.h = Helper( 19 | start=True, 20 | tb_ip=self.test_tb_port, 21 | server_ip=self.test_server_port, 22 | name=self.container_name) 23 | 24 | def tearDown(self): 25 | self.h.kill_remove() 26 | self.h = None 27 | 28 | # INIT 29 | def test_init(self): 30 | CrayonClient(port=self.test_server_port) 31 | 32 | def test_init_wrong_localhost(self): 33 | self.assertRaises(ValueError, CrayonClient, "not_open", 34 | self.test_server_port) 35 | 36 | def test_init_wrong_port(self): 37 | self.assertRaises(ValueError, CrayonClient, "localhost", 123412341234) 38 | 39 | def test_init_xp_empty(self): 40 | cc = CrayonClient(port=self.test_server_port) 41 | self.assertRaises(ValueError, cc.create_experiment, "") 42 | 43 | def test_open_experiment(self): 44 | cc = CrayonClient(port=self.test_server_port) 45 | foo = cc.create_experiment("foo") 46 | foo.add_scalar_value("bar", 1, step=2, wall_time=0) 47 | foo = cc.open_experiment("foo") 48 | foo.add_scalar_value("bar", 3, wall_time=1) 49 | self.assertEqual(foo.get_scalar_values("bar"), 50 | [[0.0, 2, 1.0], [1.0, 3, 3.0]]) 51 | 52 | def test_remove_experiment(self): 53 | cc = CrayonClient(port=self.test_server_port) 54 | self.assertRaises(ValueError, cc.open_experiment, "foo") 55 | foo = cc.create_experiment("foo") 56 | foo.add_scalar_value("bar", 1, step=2, wall_time=0) 57 | self.assertRaises(ValueError, cc.create_experiment, "foo") 58 | cc.open_experiment("foo") 59 | cc.remove_experiment(foo.xp_name) 60 | self.assertRaises(ValueError, cc.remove_experiment, foo.xp_name) 61 | foo = cc.create_experiment("foo") 62 | 63 | # scalars 64 | def test_add_scalar_value(self): 65 | cc = CrayonClient(port=self.test_server_port) 66 | foo = cc.create_experiment("foo") 67 | foo.add_scalar_value("bar", 2, wall_time=time.clock(), step=1) 68 | 69 | def test_add_scalar_less_data(self): 70 | cc = CrayonClient(port=self.test_server_port) 71 | foo = cc.create_experiment("foo") 72 | foo.add_scalar_value("bar", 2) 73 | 74 | # TODO These should really be tested singularly... 75 | def test_add_scalar_wrong_data(self): 76 | cc = CrayonClient(port=self.test_server_port) 77 | foo = cc.create_experiment("foo") 78 | self.assertRaises(ValueError, foo.add_scalar_value, 79 | "bar", "lol") 80 | 81 | def test_add_scalar_wrong_variable(self): 82 | cc = CrayonClient(port=self.test_server_port) 83 | foo = cc.create_experiment("foo") 84 | self.assertRaises(ValueError, foo.add_scalar_value, 85 | "", 2) 86 | 87 | def test_add_scalar_dict(self): 88 | cc = CrayonClient(port=self.test_server_port) 89 | foo = cc.create_experiment("foo") 90 | data = {"fizz": 3, "buzz": 5} 91 | foo.add_scalar_dict(data, wall_time=0, step=5) 92 | data = {"fizz": 6, "buzz": 10} 93 | foo.add_scalar_dict(data) 94 | 95 | def test_add_scalar_dict_wrong_data(self): 96 | cc = CrayonClient(port=self.test_server_port) 97 | foo = cc.create_experiment("foo") 98 | data = {"fizz": "foo", "buzz": 5} 99 | self.assertRaises(ValueError, foo.add_scalar_dict, data) 100 | data = {3: 6, "buzz": 10} 101 | self.assertRaises(ValueError, foo.add_scalar_dict, data) 102 | 103 | def test_get_scalar_values_no_data(self): 104 | cc = CrayonClient(port=self.test_server_port) 105 | foo = cc.create_experiment("foo") 106 | self.assertRaises(ValueError, foo.get_scalar_values, "bar") 107 | 108 | def test_get_scalar_values_one_datum(self): 109 | cc = CrayonClient(port=self.test_server_port) 110 | foo = cc.create_experiment("foo") 111 | foo.add_scalar_value("bar", 0, wall_time=0, step=0) 112 | self.assertEqual(foo.get_scalar_values("bar"), [[0.0, 0, 0.0]]) 113 | 114 | def test_get_scalar_values_two_data(self): 115 | cc = CrayonClient(port=self.test_server_port) 116 | foo = cc.create_experiment("foo") 117 | foo.add_scalar_value("bar", 0, wall_time=0, step=0) 118 | foo.add_scalar_value("bar", 1, wall_time=1, step=1) 119 | self.assertEqual(foo.get_scalar_values("bar"), 120 | [[0.0, 0, 0.0], [1.0, 1, 1.0]]) 121 | 122 | def test_get_scalar_values_auto_step(self): 123 | cc = CrayonClient(port=self.test_server_port) 124 | foo = cc.create_experiment("foo") 125 | foo.add_scalar_value("bar", 0, wall_time=0) 126 | foo.add_scalar_value("bar", 1, wall_time=1) 127 | foo.add_scalar_value("bar", 2, wall_time=2, step=10) 128 | foo.add_scalar_value("bar", 3, wall_time=3) 129 | self.assertEqual(foo.get_scalar_values("bar"), 130 | [[0.0, 0, 0.0], [1.0, 1, 1.0], 131 | [2.0, 10, 2.0], [3.0, 11, 3.0]]) 132 | 133 | def test_get_scalar_values_wrong_variable(self): 134 | cc = CrayonClient(port=self.test_server_port) 135 | foo = cc.create_experiment("foo") 136 | foo.add_scalar_value("bar", 0) 137 | self.assertRaises(ValueError, foo.get_scalar_values, "") 138 | 139 | def test_get_scalar_dict(self): 140 | cc = CrayonClient(port=self.test_server_port) 141 | foo = cc.create_experiment("foo") 142 | data = {"fizz": 3, "buzz": 5} 143 | foo.add_scalar_dict(data, wall_time=0, step=5) 144 | data = {"fizz": 6, "buzz": 10} 145 | foo.add_scalar_dict(data, wall_time=1) 146 | self.assertEqual(foo.get_scalar_values("fizz"), 147 | [[0.0, 5, 3.0], [1.0, 6, 6.0]]) 148 | self.assertEqual(foo.get_scalar_values("buzz"), 149 | [[0.0, 5, 5.0], [1.0, 6, 10.0]]) 150 | 151 | def test_get_scalar_names(self): 152 | cc = CrayonClient(port=self.test_server_port) 153 | foo = cc.create_experiment("foo") 154 | foo.add_scalar_value("fizz", 0, wall_time=0) 155 | foo.add_scalar_value("buzz", 0, wall_time=0) 156 | self.assertEqual(sorted(foo.get_scalar_names()), 157 | sorted(["fizz", "buzz"])) 158 | 159 | # Histograms 160 | def test_add_histogram_value(self): 161 | cc = CrayonClient(port=self.test_server_port) 162 | foo = cc.create_experiment("foo") 163 | data = {"min": 0, 164 | "max": 100, 165 | "num": 3, 166 | "bucket_limit": [10, 50, 30], 167 | "bucket": [5, 45, 25]} 168 | foo.add_histogram_value("bar", data, wall_time=0, step=0) 169 | foo.add_histogram_value("bar", data) 170 | 171 | def test_add_histogram_value_with_sum(self): 172 | cc = CrayonClient(port=self.test_server_port) 173 | foo = cc.create_experiment("foo") 174 | data = {"min": 0, 175 | "max": 100, 176 | "num": 3, 177 | "bucket_limit": [10, 50, 30], 178 | "bucket": [5, 45, 25], 179 | "sum": 75} 180 | foo.add_histogram_value("bar", data) 181 | 182 | def test_add_histogram_value_with_sumsq(self): 183 | cc = CrayonClient(port=self.test_server_port) 184 | foo = cc.create_experiment("foo") 185 | data = {"min": 0, 186 | "max": 100, 187 | "num": 3, 188 | "bucket_limit": [10, 50, 30], 189 | "bucket": [5, 45, 25], 190 | "sum_squares": 5625} 191 | foo.add_histogram_value("bar", data) 192 | 193 | def test_add_histogram_value_with_sum_sumsq(self): 194 | cc = CrayonClient(port=self.test_server_port) 195 | foo = cc.create_experiment("foo") 196 | data = {"min": 0, 197 | "max": 100, 198 | "num": 3, 199 | "bucket_limit": [10, 50, 30], 200 | "bucket": [5, 45, 25], 201 | "sum": 75, 202 | "sum_squares": 2675} 203 | foo.add_histogram_value("bar", data) 204 | 205 | def test_add_histogram_value_to_build(self): 206 | cc = CrayonClient(port=self.test_server_port) 207 | foo = cc.create_experiment("foo") 208 | data = [1,2,3,4,5] 209 | foo.add_histogram_value("bar", data, tobuild=True) 210 | 211 | def test_add_histogram_value_less_data(self): 212 | cc = CrayonClient(port=self.test_server_port) 213 | foo = cc.create_experiment("foo") 214 | data = {"some data": 0} 215 | self.assertRaises(ValueError, foo.add_histogram_value, 216 | "bar", data) 217 | 218 | # TODO These should really be tested singularly... 219 | def test_add_histogram_value_wrong_data(self): 220 | cc = CrayonClient(port=self.test_server_port) 221 | foo = cc.create_experiment("foo") 222 | data = ["lolz", "lulz", "lelz"] 223 | self.assertRaises(ValueError, foo.add_histogram_value, 224 | "bar", data, tobuild=True) 225 | 226 | def test_add_histogram_value_wrong_variable(self): 227 | cc = CrayonClient(port=self.test_server_port) 228 | foo = cc.create_experiment("foo") 229 | data = {"min": 0, 230 | "max": 100, 231 | "num": 3, 232 | "bucket_limit": [10, 50, 30], 233 | "bucket": [5, 45, 25]} 234 | self.assertRaises(ValueError, foo.add_histogram_value, 235 | "", data) 236 | 237 | def test_get_histogram_values_no_data(self): 238 | cc = CrayonClient(port=self.test_server_port) 239 | foo = cc.create_experiment("foo") 240 | self.assertRaises(ValueError, foo.get_histogram_values, "bar") 241 | 242 | def test_get_histogram_values_one_datum(self): 243 | cc = CrayonClient(port=self.test_server_port) 244 | foo = cc.create_experiment("foo") 245 | data = {"min": 0, 246 | "max": 100, 247 | "num": 3, 248 | "bucket_limit": [10, 50, 30], 249 | "bucket": [5, 45, 25]} 250 | foo.add_histogram_value("bar", data, wall_time=0, step=0) 251 | self.assertEqual(foo.get_histogram_values("bar"), 252 | [[0.0, 0, 253 | [0.0, 100.0, 3.0, 0.0, 0.0, 254 | [10.0, 50.0, 30.0], 255 | [5.0, 45.0, 25.0]]]]) 256 | 257 | def test_get_histogram_values_two_data(self): 258 | cc = CrayonClient(port=self.test_server_port) 259 | foo = cc.create_experiment("foo") 260 | data = {"min": 0, 261 | "max": 100, 262 | "num": 3, 263 | "bucket_limit": [10, 50, 30], 264 | "bucket": [5, 45, 25]} 265 | foo.add_histogram_value("bar", data, wall_time=0, step=0) 266 | data = {"min": 0, 267 | "max": 100, 268 | "num": 3, 269 | "bucket_limit": [10, 50, 30], 270 | "bucket": [5, 45, 25]} 271 | foo.add_histogram_value("bar", data, wall_time=1, step=1) 272 | self.assertEqual(foo.get_histogram_values("bar"), 273 | [[0.0, 0, 274 | [0.0, 100.0, 3.0, 0.0, 0.0, 275 | [10.0, 50.0, 30.0], 276 | [5.0, 45.0, 25.0]]], 277 | [1.0, 1, 278 | [0.0, 100.0, 3.0, 0.0, 0.0, 279 | [10.0, 50.0, 30.0], 280 | [5.0, 45.0, 25.0]]]]) 281 | 282 | def test_get_histogram_values_wrong_variable(self): 283 | cc = CrayonClient(port=self.test_server_port) 284 | foo = cc.create_experiment("foo") 285 | data = {"min": 0, 286 | "max": 100, 287 | "num": 3, 288 | "bucket_limit": [10, 50, 30], 289 | "bucket": [5, 45, 25]} 290 | foo.add_histogram_value("bar", data, wall_time=0, step=0) 291 | self.assertRaises(ValueError, foo.get_histogram_values, "") 292 | 293 | def test_get_histogram_names(self): 294 | cc = CrayonClient(port=self.test_server_port) 295 | foo = cc.create_experiment("foo") 296 | data = {"min": 0, 297 | "max": 100, 298 | "num": 3, 299 | "bucket_limit": [10, 50, 30], 300 | "bucket": [5, 45, 25]} 301 | foo.add_histogram_value("fizz", data, wall_time=0, step=0) 302 | foo.add_histogram_value("buzz", data, wall_time=1, step=1) 303 | self.assertEqual(sorted(foo.get_histogram_names()), 304 | sorted(["fizz", "buzz"])) 305 | 306 | # Only checks that we get a zip file. 307 | # TODO open and match data to recorded 308 | def test_to_zip(self): 309 | cc = CrayonClient(port=self.test_server_port) 310 | foo = cc.create_experiment("foo") 311 | foo.add_scalar_value("bar", 2, wall_time=time.time(), step=1) 312 | filename = foo.to_zip() 313 | os.remove(filename) 314 | 315 | # Only checks that we set a zip file. 316 | def test_init_from_file(self): 317 | cc = CrayonClient(port=self.test_server_port) 318 | foo = cc.create_experiment("foo") 319 | foo.add_scalar_value("bar", 2, wall_time=time.time(), step=1) 320 | filename = foo.to_zip() 321 | new = cc.create_experiment("new", filename) 322 | os.remove(filename) 323 | 324 | def test_set_data_wrong_file(self): 325 | cc = CrayonClient(port=self.test_server_port) 326 | self.assertRaises(IOError, cc.create_experiment, "foo", 327 | "random_noise") 328 | 329 | def test_backup(self): 330 | cc = CrayonClient(port=self.test_server_port) 331 | foo = cc.create_experiment("foo") 332 | foo.add_scalar_value("bar", 2, wall_time=time.time(), step=1) 333 | foo.add_scalar_value("bar", 2, wall_time=time.time(), step=2) 334 | foo_data = foo.get_scalar_values("bar") 335 | filename = foo.to_zip() 336 | 337 | cc.remove_experiment("foo") 338 | 339 | foo = cc.create_experiment("foo", zip_file=filename) 340 | new_data = foo.get_scalar_values("bar") 341 | self.assertEqual(foo_data, new_data) 342 | 343 | new = cc.create_experiment("new", zip_file=filename) 344 | new_data = new.get_scalar_values("bar") 345 | self.assertEqual(foo_data, new_data) 346 | 347 | os.remove(filename) 348 | -------------------------------------------------------------------------------- /doc/specs.md: -------------------------------------------------------------------------------- 1 | API specification 2 | ============ 3 | 4 | ## API version 5 | * GET `/` 6 | * Check that the server runs properly and return running version 7 | * return: a string containing the running version of the server 8 | 9 | ## Experience management 10 | * GET `/data` 11 | * Get the list of running experiments 12 | * return: a json list of name of all the running experiments 13 | 14 | * GET `/data?xp=foo` 15 | * Get the informations about the experiment `foo` 16 | * return: a json dictionnary with the following entries: 17 | * `scalars`: the name of all the scalar data entries 18 | * `histograms`: the name of all the histogram data entries 19 | 20 | * POST `/data` 21 | * Add a new experiment 22 | * post content: a string with the name of the new experiment 23 | 24 | * DELETE '/data?xp=foo' 25 | * Delete the experiment named `foo`. Impossible to cancel! 26 | 27 | ## Scalar data 28 | * POST `/data/scalars?xp=foo&name=bar` 29 | * Adds a new scalar point in the experience `foo` for the scalar named `bar` 30 | * param: 31 | * `xp`: the considered experience 32 | * `name`: the name of the scalar metric we want to add a value to 33 | * post content: a json with a single list containing 3 values: 34 | * wall_time of the measure 35 | * step of the measure 36 | * value 37 | 38 | * GET `/data/scalars?xp=foo&name=bar` 39 | * Get the values for the scalar named `bar` in the experience `foo` 40 | * return: a json with a list with one entry per value logged.Each entry is a list containing 3 values: 41 | * wall_time of the measure 42 | * step of the measure 43 | * value 44 | 45 | ## Histogram data 46 | * POST `/data/histograms?xp=foo&name=bar&tobuild=True` 47 | * Adds a new histogram in the experience `foo` for the scalar named `bar` 48 | * param: 49 | * `xp`: the considered experience 50 | * `name`: the name of the scalar metric we want to add a value to 51 | * `tobuild`: if true, the post content should be a list, otherwise the histogram 52 | * post content: a json containing a list of 3 elements: 53 | * wall time of the measure 54 | * step of the measure 55 | * the histogram: 56 | * if `tobuild`=true: a single list containing all the values that will be converted to an histogram 57 | * if `tobuild`=false: json containing a dictionary with the following keys: 58 | * `min`: the minimum value 59 | * `max`: the maximum value 60 | * `num`: the number of entries 61 | * `bucket_limit`: a list of `len` elements containing the (right) limit for each bucket 62 | * `bucket`: a list of `len` elements containing the count for each bucket 63 | * `sum` (optionnal): the sum of all the values 64 | * `sum_squares` (optionnal): the squared sum of all the values 65 | 66 | * GET `/data/histograms?xp=foo&name=bar` 67 | * Get the values for the histogram named `bar` in the experience `foo` 68 | * return: a json containing: 69 | ``` 70 | [ 71 | [ 72 | 1443871386.185149, # wall_time 73 | 235166, # step 74 | [ 75 | -0.66, # minimum value 76 | 0.44, # maximum value 77 | 8.0, # number of items in the histogram 78 | -0.80, # sum of items in the histogram 79 | 0.73, # sum of squares of items in the histogram 80 | [-0.68, -0.62, -0.292, -0.26, -0.11, -0.10, -0.08, -0.07, -0.05, 81 | -0.0525, -0.0434, -0.039, -0.029, -0.026, 0.42, 0.47, 1.8e+308], 82 | # the right edge of each bucket 83 | [0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 1.0, 0.0, 84 | 1.0, 0.0] # the number of elements within each bucket 85 | ] 86 | ] 87 | ] 88 | ``` 89 | 90 | ## Backup data 91 | * GET `/backup?xp=foo` 92 | * Return a zip file containing all the datas for the experiment `foo` 93 | * param: 94 | * `xp`: the experiment to backup 95 | 96 | * POST `/backup?xp=foo&force=True` 97 | * Drop all current datas for the experiment `foo` and replace them with the state contained in the zip 98 | * param: 99 | * `xp`: the experiment to replace 100 | * `force`: has to be set to 1 to be able to delete the old experiment 101 | * post content: a zip file coming from the backup GET request 102 | -------------------------------------------------------------------------------- /server/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM tensorflow/tensorflow:latest 2 | RUN pip install flask 3 | ADD startup.sh / 4 | ADD server.py / 5 | ADD patch_tensorboard.py / 6 | EXPOSE 8889 7 | # TODO: move from cmd to entrypoint 8 | ENTRYPOINT ["/bin/bash", "/startup.sh"] -------------------------------------------------------------------------------- /server/README.md: -------------------------------------------------------------------------------- 1 | # The server is build completely inside a docker image. 2 | 3 | IF you do not have the docker image yet: 4 | ```bash 5 | # Get it from Docker Hub 6 | docker pull alband/crayon 7 | 8 | # OR build it locally with: 9 | docker build -t alband/crayon:latest . 10 | ``` 11 | 12 | Then the server can be started with: 13 | ```bash 14 | docker run -d -p 8888:8888 -p 8889:8889 --name crayon alband/crayon [frontend_refresh] [backend_refresh] 15 | ``` 16 | Where `frontend_refresh` is an optional argument that allows to change the refresh rate of tensorboard to the given value (in seconds). Default is 10 seconds. 17 | Where `backend_refresh` in an optional argument that allows to change how often the tensorflow backend reload the files from disk (in seconds). Default is 0.5 seconds. Reducing this value too much may make tensorboard unstable. 18 | 19 | 20 | Tensorboard can then be accessed from a browser at `localhost:8888`. 21 | The client should be setup to send the datas at `localhost:8889`. 22 | -------------------------------------------------------------------------------- /server/patch_tensorboard.py: -------------------------------------------------------------------------------- 1 | # This script should silently fail 2 | from tensorflow.tensorboard import tensorboard 3 | import os 4 | import sys 5 | 6 | import argparse 7 | parser = argparse.ArgumentParser(description="Backend server for crayon") 8 | parser.add_argument("frontend_reload", type=int, 9 | help="Frontend reload value") 10 | parser.add_argument("backend_reload", type=float, 11 | help="How fast is tensorboard reloading its backend") 12 | cli_args = parser.parse_args() 13 | 14 | frontend_reload = str(cli_args.frontend_reload) 15 | backend_reload = str(cli_args.backend_reload) 16 | 17 | 18 | tb_path = os.path.dirname(os.path.abspath(tensorboard.__file__)) 19 | 20 | frontend_worked = False 21 | try: 22 | print("Patching tensorboard to change the delay to "+frontend_reload+"s") 23 | 24 | dist_path = os.path.join(tb_path, "dist") 25 | html_path = os.path.join(dist_path, "tf-tensorboard.html") 26 | 27 | content = [] 28 | state = 0 29 | # state: 30 | # 0: looking for variable name 31 | # 1: looking for type 32 | # 2: looking for value 33 | # 3: finishing to read the file 34 | with open(html_path, "r") as html_file: 35 | for line in html_file: 36 | if state == 0: 37 | if "autoReloadIntervalSecs:" in line: 38 | state = 1 39 | elif state == 1: 40 | if "type: Number" in line: 41 | state = 2 42 | else: 43 | print("This should not happen, trying to find a new instance") 44 | state = 0 45 | elif state == 2: 46 | if "value: 120" in line: 47 | line = line.replace("120", frontend_reload) 48 | state = 3 49 | else: 50 | print("This should not happen, trying to find a new instance") 51 | state = 0 52 | elif state == 3: 53 | pass 54 | else: 55 | print("This should not happen, trying to find a new instance") 56 | state = 0 57 | 58 | content += [line] 59 | 60 | with open(html_path, "w") as html_file: 61 | html_file.write("\n".join(content)) 62 | 63 | frontend_worked = True 64 | print("Success !") 65 | except: 66 | print("Patching failed") 67 | 68 | backend_worked = False 69 | try: 70 | print("Patching tensorboard to change backend reload to float.") 71 | 72 | tensorboard_path = os.path.join(tb_path, "tensorboard.py") 73 | 74 | content = [] 75 | with open(tensorboard_path, "r") as source_file: 76 | for line in source_file: 77 | if ("DEFINE_integer" in line) and ("reload_interval" in line): 78 | line = line.replace("DEFINE_integer", "DEFINE_float") 79 | content += [line] 80 | 81 | with open(tensorboard_path, "w") as source_file: 82 | source_file.write("\n".join(content)) 83 | 84 | backend_worked = True 85 | print("Success !") 86 | except: 87 | print("Patching failed") 88 | 89 | if backend_worked: 90 | if frontend_worked: 91 | # Both worked, return 3 92 | sys.exit(3) 93 | else: 94 | # Only backend worked, return 2 95 | sys.exit(2) 96 | elif frontend_worked: 97 | # Only frontend worked, return 1 98 | sys.exit(1) 99 | else: 100 | # Nothing worked, return -1 101 | sys.exit(-1) 102 | -------------------------------------------------------------------------------- /server/server.py: -------------------------------------------------------------------------------- 1 | # Flask app server 2 | from flask import Flask, request, json 3 | app = Flask("crayonserver") 4 | 5 | # HTTP client to use the tensorboard api 6 | import urllib2 7 | 8 | # Server version 9 | __version__ = "0.5" 10 | 11 | # Not-supported logging types 12 | not_supported_types = [ 13 | "audio", 14 | "compressedHistograms", 15 | "graph", 16 | "images", 17 | "meta_graph", 18 | "run_metadata", 19 | "firstEventTimestamp"] 20 | 21 | # Supported logging types 22 | supported_types = [ 23 | "scalars", 24 | "histograms"] 25 | 26 | # Tensorboard includes 27 | import tensorflow as tf 28 | import bisect 29 | import time 30 | 31 | # Backup includes 32 | from os import path 33 | from subprocess import Popen, PIPE 34 | from flask import send_file 35 | import shutil 36 | 37 | # Command line arguments 38 | import argparse 39 | parser = argparse.ArgumentParser(description="Backend server for crayon") 40 | parser.add_argument("port", type=int, default=8889, 41 | help="Port where to listen for incoming datas") 42 | parser.add_argument("backend_reload", type=float, default=1, 43 | help="How fast is tensorboard reloading its backend") 44 | cli_args = parser.parse_args() 45 | 46 | # Delay timer 47 | # We add 1s to make sure all files are loaded from disk 48 | request_delay = cli_args.backend_reload + 1 49 | 50 | def to_unicode(experiment): 51 | 52 | assert experiment and isinstance(experiment, basestring) 53 | 54 | return unicode(experiment) 55 | 56 | ### Tensorboard utility functions 57 | tensorboard_folder = "/tmp/tensorboard/{}" 58 | # Make sure we do not access data too fast 59 | xp_modified = {} 60 | def tb_modified_xp(experiment, modified_type=None, wall_time=None): 61 | assert(modified_type is None or modified_type in supported_types) 62 | xp_modified[experiment] = (time.time(), modified_type, wall_time) 63 | 64 | def last_timestamp_loaded(experiment, modified_type, last_timestamp): 65 | req_res = tb_request("runs", safe=False) 66 | tb_data = json.loads(req_res) 67 | if experiment in tb_data: 68 | if modified_type in tb_data[experiment]: 69 | names = tb_data[experiment][modified_type] 70 | for name in names: 71 | req_res = tb_request(modified_type, experiment, name, safe=False) 72 | req_res = json.loads(req_res) 73 | for value in req_res: 74 | if value[0] == last_timestamp: 75 | return True 76 | return False 77 | 78 | def tb_access_xp(experiment): 79 | if experiment not in xp_modified: 80 | return 81 | last_modified, modified_type, last_timestamp = xp_modified[experiment] 82 | 83 | while time.time() < last_modified + request_delay: 84 | # If we know the last timestamp, try to exit early 85 | if modified_type is not None: 86 | if last_timestamp_loaded(experiment, modified_type, last_timestamp): 87 | break 88 | else: 89 | time.sleep(0.01) 90 | del xp_modified[experiment] 91 | 92 | def tb_access_all(): 93 | for experiment in xp_modified.keys(): 94 | tb_access_xp(experiment) 95 | 96 | # Make sure we have writers for all experiments 97 | xp_writers = {} 98 | def tb_get_xp_writer(experiment): 99 | if experiment in xp_writers: 100 | return xp_writers[experiment] 101 | 102 | tb_access_xp(experiment) 103 | xp_folder = tensorboard_folder.format(experiment) 104 | writer = tf.summary.FileWriter(xp_folder, flush_secs=1) 105 | xp_writers[experiment] = writer 106 | tb_modified_xp(experiment) 107 | return writer 108 | 109 | def tb_remove_xp_writer(experiment): 110 | # If the experiment does not exist, does nothing silently 111 | if experiment in xp_writers: 112 | del xp_writers[experiment] 113 | # Prevent recreating it too quickly 114 | tb_modified_xp(experiment) 115 | 116 | def tb_xp_writer_exists(experiment): 117 | return experiment in xp_writers 118 | 119 | # Use writers 120 | def tb_add_scalar(experiment, name, wall_time, step, value): 121 | writer = tb_get_xp_writer(experiment) 122 | summary = tf.Summary(value=[ 123 | tf.Summary.Value(tag=name, simple_value=value), 124 | ]) 125 | event = tf.Event(wall_time=wall_time, step=step, summary=summary) 126 | writer.add_event(event) 127 | writer.flush() 128 | tb_modified_xp(experiment, modified_type="scalars", wall_time=wall_time) 129 | 130 | def tb_add_histogram(experiment, name, wall_time, step, histo): 131 | # Tensorflow does not support key being unicode 132 | histo_string = {} 133 | for k,v in histo.items(): 134 | histo_string[str(k)] = v 135 | histo = histo_string 136 | 137 | writer = tb_get_xp_writer(experiment) 138 | summary = tf.Summary(value=[ 139 | tf.Summary.Value(tag=name, histo=histo), 140 | ]) 141 | event = tf.Event(wall_time=wall_time, step=step, summary=summary) 142 | writer.add_event(event) 143 | writer.flush() 144 | tb_modified_xp(experiment, modified_type="histograms", wall_time=wall_time) 145 | 146 | # Perform requests to tensorboard http api 147 | def tb_request(query_type, run=None, tag=None, safe=True): 148 | request_url = "http://localhost:8888/data/{}" 149 | if run and tag: 150 | request_url += "?run={}&tag={}" 151 | 152 | if safe: 153 | if run: 154 | tb_access_xp(run) 155 | else: 156 | tb_access_all() 157 | 158 | request_url = request_url.format(query_type, run, tag) 159 | try: 160 | return urllib2.urlopen(request_url, timeout=1).read() 161 | except: 162 | raise ValueError 163 | 164 | # Borrowed from tensorflow/tensorboard/scripts/generate_testdata.py 165 | # Create a histogram from a list of values 166 | def _MakeHistogramBuckets(): 167 | v = 1E-12 168 | buckets = [] 169 | neg_buckets = [] 170 | while v < 1E20: 171 | buckets.append(v) 172 | neg_buckets.append(-v) 173 | v *= 1.1 174 | # Should include DBL_MAX, but won't bother for test data. 175 | return neg_buckets[::-1] + [0] + buckets 176 | 177 | 178 | def tb_make_histogram(values): 179 | """Convert values into a histogram proto using logic from histogram.cc.""" 180 | limits = _MakeHistogramBuckets() 181 | counts = [0] * len(limits) 182 | for v in values: 183 | idx = bisect.bisect_left(limits, v) 184 | counts[idx] += 1 185 | 186 | limit_counts = [(limits[i], counts[i]) for i in xrange(len(limits)) 187 | if counts[i]] 188 | bucket_limit = [lc[0] for lc in limit_counts] 189 | bucket = [lc[1] for lc in limit_counts] 190 | sum_sq = sum(v * v for v in values) 191 | return { 192 | "min": min(values), 193 | "max": max(values), 194 | "num": len(values), 195 | "sum": sum(values), 196 | "sum_squares": sum_sq, 197 | "bucket_limit": bucket_limit, 198 | "bucket": bucket} 199 | ## END of borrowed 200 | 201 | 202 | ### Error handler 203 | @app.errorhandler(404) 204 | def not_found(error): 205 | return "This is not the web page you are looking for." 206 | 207 | def wrong_argument(message): 208 | print("wrong_argument: ", message) 209 | return message, 400 210 | 211 | ### Running and version check 212 | @app.route('/', methods=["GET"]) 213 | def get_version(): 214 | # Verify that tensorboard is running 215 | try: 216 | req_res = tb_request("logdir") 217 | except: 218 | return wrong_argument("Server: TensorBoard failed to answer request 'logdir'") 219 | 220 | if not json.loads(req_res)["logdir"] == tensorboard_folder[:-3]: 221 | return wrong_argument("Tensorboard is not running in the correct folder.") 222 | 223 | return __version__ 224 | 225 | 226 | ### Experience management 227 | @app.route('/data', methods=["GET"]) 228 | def get_all_experiments(): 229 | experiment = request.args.get('xp') 230 | 231 | result = "" 232 | try: 233 | req_res = tb_request("runs") 234 | except: 235 | return wrong_argument("Server: TensorBoard failed to answer request 'runs'") 236 | 237 | tb_data = json.loads(req_res) 238 | if experiment: 239 | try: 240 | experiment = to_unicode(experiment) 241 | except: 242 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 243 | if not tb_xp_writer_exists(experiment): 244 | return wrong_argument("Unknown experiment name '{}'".format(experiment)) 245 | if experiment in tb_data: 246 | result = tb_data[experiment] 247 | # Remove the not supported types from the answer 248 | for not_supported_type in not_supported_types: 249 | if not_supported_type in result: 250 | del result[not_supported_type] 251 | else: 252 | # Experience with no data on tensorboard, 253 | # return empty list for all types 254 | result = {} 255 | for t in supported_types: 256 | result[t] = [] 257 | else: 258 | result = tb_data.keys() 259 | return json.dumps(result) 260 | 261 | @app.route('/data', methods=["POST"]) 262 | def post_experiment(): 263 | experiment = request.get_json() 264 | try: 265 | experiment = to_unicode(experiment) 266 | except: 267 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 268 | 269 | if tb_xp_writer_exists(experiment): 270 | return wrong_argument("'{}' experiment already exists".format(experiment)) 271 | 272 | tb_get_xp_writer(experiment) 273 | return "ok" 274 | 275 | @app.route('/data', methods=["DELETE"]) 276 | def delete_experiment(): 277 | experiment = request.args.get('xp') 278 | try: 279 | experiment = to_unicode(experiment) 280 | except: 281 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 282 | 283 | if not tb_xp_writer_exists(experiment): 284 | return wrong_argument("'{}' experiment does not already exist".format(experiment)) 285 | 286 | # Delete folder on disk 287 | folder_path = tensorboard_folder.format(experiment) 288 | shutil.rmtree(folder_path) 289 | 290 | # Delete experience writer 291 | tb_remove_xp_writer(experiment) 292 | 293 | return "ok" 294 | 295 | ### Scalar data 296 | @app.route('/data/scalars', methods=["GET"]) 297 | def get_scalars(): 298 | experiment = request.args.get('xp') 299 | try: 300 | experiment = to_unicode(experiment) 301 | except: 302 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 303 | name = request.args.get('name') 304 | if (not experiment) or (not name): 305 | return wrong_argument("xp and name arguments are required") 306 | if not tb_xp_writer_exists(experiment): 307 | return wrong_argument("Unknown experiment name '{}'".format(experiment)) 308 | 309 | try: 310 | req_res = tb_request("scalars", experiment, name) 311 | return req_res 312 | except: 313 | message = "Combination of experiment '{}' and name '{}' does not exist".format(experiment, name) 314 | return wrong_argument(message) 315 | 316 | 317 | 318 | @app.route('/data/scalars', methods=['POST']) 319 | def post_scalars(): 320 | experiment = request.args.get('xp') 321 | try: 322 | experiment = to_unicode(experiment) 323 | except: 324 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 325 | name = request.args.get('name') 326 | if (not experiment) or (not name): 327 | return wrong_argument("xp and name arguments are required") 328 | if not tb_xp_writer_exists(experiment): 329 | return wrong_argument("Unknown experiment name '{}'".format(experiment)) 330 | 331 | data = request.get_json() 332 | if not data: 333 | return wrong_argument("POST content is not a proper json") 334 | if not isinstance(data, list): 335 | return wrong_argument("POST content is not a list: '{}'".format(request.form.keys())) 336 | if not len(data)==3: 337 | return wrong_argument("POST does not contain a list of 3 elements but '{}'".format(data)) 338 | if not (isinstance(data[2], int) or isinstance(data[2], float)): 339 | return wrong_argument("POST value is not a number but '{}'".format(data[2])) 340 | 341 | tb_add_scalar(experiment, name, data[0], data[1], data[2]) 342 | 343 | return "ok" 344 | 345 | 346 | ### Histogram data 347 | @app.route('/data/histograms', methods=["GET"]) 348 | def get_histograms(): 349 | experiment = request.args.get('xp') 350 | try: 351 | experiment = to_unicode(experiment) 352 | except: 353 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 354 | name = request.args.get('name') 355 | if (not experiment) or (not name): 356 | return wrong_argument("xp and name arguments are required") 357 | if not tb_xp_writer_exists(experiment): 358 | return wrong_argument("Unknown experiment name '{}'".format(experiment)) 359 | 360 | try: 361 | req_res = tb_request("histograms", experiment, name) 362 | return req_res 363 | except: 364 | message = "Combination of experiment '{}' and name '{}' does not exist".format(experiment, name) 365 | return wrong_argument(message) 366 | 367 | 368 | 369 | @app.route('/data/histograms', methods=['POST']) 370 | def post_histograms(): 371 | experiment = request.args.get('xp') 372 | try: 373 | experiment = to_unicode(experiment) 374 | except: 375 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 376 | name = request.args.get('name') 377 | to_build = request.args.get('tobuild') 378 | if (not experiment) or (not name) or (not to_build): 379 | return wrong_argument("xp, name and tobuild arguments are required") 380 | if not tb_xp_writer_exists(experiment): 381 | return wrong_argument("Unknown experiment name '{}'".format(experiment)) 382 | to_build = to_build.lower() == "true" 383 | 384 | data = request.get_json() 385 | if not data: 386 | return wrong_argument("POST content is not a proper json") 387 | if not isinstance(data, list): 388 | return wrong_argument("POST content is not a list: '{}'".format(request.form.keys())) 389 | if not len(data)==3: 390 | return wrong_argument("POST does not contain a list of 3 elements but '{}'".format(data)) 391 | 392 | if to_build: 393 | if (not data[2]) or (not isinstance(data[2], list)): 394 | return wrong_argument("elements to build the histogram are not in a list but '{}'".format(data[2])) 395 | histogram_dict = tb_make_histogram(data[2]) 396 | else: 397 | already_built_required_params = { 398 | "min": [float, int], 399 | "max": [float, int], 400 | "num": [int], 401 | "bucket_limit": [list], 402 | "bucket": [list], 403 | } 404 | histogram_dict = data[2] 405 | for required_param in already_built_required_params: 406 | if not (required_param in histogram_dict): 407 | message = "Missing argument '{}' to the given histogram".format(required_param) 408 | return wrong_argument(message) 409 | is_ok = False 410 | for required_type in already_built_required_params[required_param]: 411 | if isinstance(histogram_dict[required_param], required_type): 412 | is_ok = True 413 | break 414 | if not is_ok: 415 | message = "Argument '{}' should be of type '{}' and is '{}'" 416 | message = message.format(required_param, str(already_built_required_params[required_param]), str(type(histogram_dict[required_param]))) 417 | return wrong_argument(message) 418 | 419 | tb_add_histogram(experiment, name, data[0], data[1], histogram_dict) 420 | 421 | return "ok" 422 | 423 | 424 | ### Backup data 425 | @app.route('/backup', methods=['GET']) 426 | def get_backup(): 427 | experiment = request.args.get('xp') 428 | try: 429 | experiment = to_unicode(experiment) 430 | except: 431 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 432 | if not experiment: 433 | return wrong_argument("xp argument is required") 434 | 435 | folder_path = tensorboard_folder.format(experiment) 436 | 437 | if not path.isdir(folder_path): 438 | return wrong_argument("Requested experiment '{}' does not exist".format(experiment)) 439 | 440 | zip_file = shutil.make_archive("/tmp/{}".format(experiment), 'zip', folder_path) 441 | 442 | return send_file(zip_file, mimetype='application/zip') 443 | 444 | @app.route('/backup', methods=['POST']) 445 | def post_backup(): 446 | experiment = request.args.get('xp') 447 | try: 448 | experiment = to_unicode(experiment) 449 | except: 450 | return wrong_argument("Experiment name should be a non-empty string or unicode instead of '{}'".format(type(experiment))) 451 | force = request.args.get('force') 452 | if (not experiment) or (not force): 453 | return wrong_argument("xp and force argument are required") 454 | if not force.lower() == 'true': 455 | return wrong_argument("Force must be set to 1 to be able to override a folder") 456 | if tb_xp_writer_exists(experiment): 457 | return wrong_argument("Experiment '{}' already exists".format(experiment)) 458 | 459 | folder_path = tensorboard_folder.format(experiment) 460 | zip_file_path = "/tmp/{}.zip".format(experiment) 461 | 462 | if "archive" in request.files: 463 | backup_data = request.files["archive"] 464 | backup_data.save(zip_file_path) 465 | else: 466 | content_type = request.headers.get('Content-type', '') 467 | if (not content_type) or (content_type != "application/zip"): 468 | return wrong_argument("Backup post request should contain a file or a zip") 469 | with open(zip_file_path, "wb") as f: 470 | f.write(request.data) 471 | 472 | folder_path = tensorboard_folder.format(experiment) 473 | Popen("mkdir -p {}".format(folder_path),stdout=PIPE, shell=True) 474 | Popen("cd {}; unzip {}".format(folder_path, zip_file_path),stdout=PIPE, shell=True) 475 | 476 | tb_get_xp_writer(experiment) 477 | 478 | return "ok" 479 | 480 | 481 | app.run(host="0.0.0.0", port=cli_args.port) 482 | -------------------------------------------------------------------------------- /server/startup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Default values for the arguments 4 | CRAYON_FRONTEND_RELOAD="$1" 5 | if [[ $CRAYON_FRONTEND_RELOAD == "" ]]; then 6 | CRAYON_FRONTEND_RELOAD="10" 7 | fi 8 | 9 | CRAYON_BACKEND_RELOAD="$2" 10 | if [[ $CRAYON_BACKEND_RELOAD == "" ]]; then 11 | CRAYON_BACKEND_RELOAD="0.5" 12 | fi 13 | 14 | # Try to patch tensorboard to reduce autorefresh time and allow subsecond backend refresh 15 | python /patch_tensorboard.py "$CRAYON_FRONTEND_RELOAD" "$CRAYON_BACKEND_RELOAD" 16 | RETURN_CODE="$?" 17 | 18 | # If path failed, set backend reload to integer 19 | if [[ ${RETURN_CODE} != "3" ]] && [[ ${RETURN_CODE} != "2" ]]; then 20 | CRAYON_BACKEND_RELOAD="1" 21 | fi 22 | echo "Using $CRAYON_BACKEND_RELOAD for backend reload time." 23 | 24 | tensorboard --reload_interval $CRAYON_BACKEND_RELOAD --logdir /tmp/tensorboard --port 8888 & 25 | 26 | python /server.py 8889 $CRAYON_BACKEND_RELOAD 2>&1 1>/tmp/crayon.log 27 | --------------------------------------------------------------------------------