'+ title + '
' + summary +'
111 |
112 |
113 |
119 | 404
114 | 115 |Page not found
116 | 117 | 118 |
4 |
5 | ' + summary +'
No results found
"); 41 | } 42 | } 43 | 44 | function doSearch () { 45 | var query = document.getElementById('mkdocs-search-query').value; 46 | if (query.length > 2) { 47 | if (!window.Worker) { 48 | displayResults(search(query)); 49 | } else { 50 | searchWorker.postMessage({query: query}); 51 | } 52 | } else { 53 | // Clear results for short queries 54 | displayResults([]); 55 | } 56 | } 57 | 58 | function initSearch () { 59 | var search_input = document.getElementById('mkdocs-search-query'); 60 | if (search_input) { 61 | search_input.addEventListener("keyup", doSearch); 62 | } 63 | var term = getSearchTermFromLocation(); 64 | if (term) { 65 | search_input.value = term; 66 | doSearch(); 67 | } 68 | } 69 | 70 | function onWorkerMessage (e) { 71 | if (e.data.allowSearch) { 72 | initSearch(); 73 | } else if (e.data.results) { 74 | var results = e.data.results; 75 | displayResults(results); 76 | } 77 | } 78 | 79 | if (!window.Worker) { 80 | console.log('Web Worker API not supported'); 81 | // load index in main thread 82 | $.getScript(joinUrl(base_url, "search/worker.js")).done(function () { 83 | console.log('Loaded worker'); 84 | init(); 85 | window.postMessage = function (msg) { 86 | onWorkerMessage({data: msg}); 87 | }; 88 | }).fail(function (jqxhr, settings, exception) { 89 | console.error('Could not load worker.js'); 90 | }); 91 | } else { 92 | // Wrap search in a web worker 93 | var searchWorker = new Worker(joinUrl(base_url, "search/worker.js")); 94 | searchWorker.postMessage({init: true}); 95 | searchWorker.onmessage = onWorkerMessage; 96 | } 97 | -------------------------------------------------------------------------------- /test_tube/hyper_opt_utils/strategies.py: -------------------------------------------------------------------------------- 1 | """Hyperparameter search strategies.""" 2 | import itertools 3 | import json 4 | import random 5 | 6 | 7 | def generate_trials(strategy, flat_params, nb_trials=None): 8 | r"""Generates the parameter combinations to search. 9 | 10 | Two search strategies are implemented: 11 | 1. `grid_search`: creates a search space that consists of the 12 | product of all flat_params. If `nb_trials` is specified 13 | the first `nb_trials` combinations are searched. 14 | 2. `random_search`: Creates random combinations of the 15 | hyperparameters. Can be used for a more efficient search. 16 | See (Bergstra and Bengio, 2012) for more details. 17 | 18 | :param strategy: The hyperparameter search to strategy. Can be 19 | one of: {`grid_search`, `random`}. 20 | :param flat_params: The hyperparameter arguments to iterate over. 21 | :param nb_trials: The number of hyperparameter combinations to try. 22 | Generates the parameter combinations for each requested trial 23 | :param strategy: 24 | :param flat_params: 25 | :param nb_trials: The number of trials to un. 26 | :return: 27 | """ 28 | if strategy == 'grid_search': 29 | trials = generate_grid_search_trials(flat_params, nb_trials) 30 | return trials 31 | elif strategy == 'random_search': 32 | trials = generate_random_search_trials(flat_params, nb_trials) 33 | return trials 34 | else: 35 | raise ValueError( 36 | ('Unknown strategy "{}". Must be one of ' 37 | '{{grid_search, random_search}}').format(strategy)) 38 | 39 | 40 | def generate_grid_search_trials(flat_params, nb_trials): 41 | """ 42 | Standard grid search. Takes the product of `flat_params` 43 | to generate the search space. 44 | 45 | :param params: The hyperparameters options to search. 46 | :param nb_trials: Returns the first `nb_trials` from the 47 | combinations space. If this is None, all combinations 48 | are returned. 49 | :return: A dict containing the hyperparameters. 50 | """ 51 | trials = list(itertools.product(*flat_params)) 52 | if nb_trials: 53 | trials = trials[0:nb_trials] 54 | return trials 55 | 56 | 57 | def generate_random_search_trials(params, nb_trials): 58 | """ 59 | Generates random combination of hyperparameters to try. 60 | See (Bergstra and Bengio, 2012) for more details. 61 | 62 | :param params: The hyperparameters options to search. 63 | :param nb_trials: The number of trials to run. 64 | :return: A dict containing the hyperparameters. 65 | """ 66 | if nb_trials is None: 67 | raise TypeError( 68 | '`random_search` strategy requires nb_trails to be an int.') 69 | results = [] 70 | 71 | # ensures we have unique results 72 | seen_trials = set() 73 | 74 | # shuffle each param list 75 | potential_trials = 1 76 | for param in params: 77 | random.shuffle(param) 78 | potential_trials *= len(param) 79 | 80 | # we can't sample more trials than are possible 81 | max_iters = min(potential_trials, nb_trials) 82 | 83 | # then for the nb of trials requested, create a new param tuple 84 | # by picking a random integer at each param level 85 | while len(results) < max_iters: 86 | trial = [] 87 | for param in params: 88 | sampled_param = random.sample(param, 1)[0] 89 | trial.append(sampled_param) 90 | 91 | # verify this is a unique trial so we 92 | # don't duplicate work 93 | trial_str = json.dumps(trial) 94 | if trial_str not in seen_trials: 95 | seen_trials.add(trial_str) 96 | results.append(trial) 97 | 98 | return results 99 | -------------------------------------------------------------------------------- /site/js/theme.js: -------------------------------------------------------------------------------- 1 | $( document ).ready(function() { 2 | // Shift nav in mobile when clicking the menu. 3 | $(document).on('click', "[data-toggle='wy-nav-top']", function() { 4 | $("[data-toggle='wy-nav-shift']").toggleClass("shift"); 5 | $("[data-toggle='rst-versions']").toggleClass("shift"); 6 | }); 7 | 8 | // Close menu when you click a link. 9 | $(document).on('click', ".wy-menu-vertical .current ul li a", function() { 10 | $("[data-toggle='wy-nav-shift']").removeClass("shift"); 11 | $("[data-toggle='rst-versions']").toggleClass("shift"); 12 | }); 13 | 14 | // Keyboard navigation 15 | document.addEventListener("keydown", function(e) { 16 | var key = e.which || e.keyCode || window.event && window.event.keyCode; 17 | var page; 18 | switch (key) { 19 | case 78: // n 20 | page = $('[role="navigation"] a:contains(Next):first').prop('href'); 21 | break; 22 | case 80: // p 23 | page = $('[role="navigation"] a:contains(Previous):first').prop('href'); 24 | break; 25 | case 13: // enter 26 | if (e.target === document.getElementById('mkdocs-search-query')) { 27 | e.preventDefault(); 28 | } 29 | break; 30 | default: break; 31 | } 32 | if ($(e.target).is(':input')) { 33 | return true; 34 | } else if (page) { 35 | window.location.href = page; 36 | } 37 | }); 38 | 39 | $(document).on('click', "[data-toggle='rst-current-version']", function() { 40 | $("[data-toggle='rst-versions']").toggleClass("shift-up"); 41 | }); 42 | 43 | // Make tables responsive 44 | $("table.docutils:not(.field-list)").wrap(""); 45 | 46 | $('table').addClass('docutils'); 47 | }); 48 | 49 | window.SphinxRtdTheme = (function (jquery) { 50 | var stickyNav = (function () { 51 | var navBar, 52 | win, 53 | stickyNavCssClass = 'stickynav', 54 | applyStickNav = function () { 55 | if (navBar.height() <= win.height()) { 56 | navBar.addClass(stickyNavCssClass); 57 | } else { 58 | navBar.removeClass(stickyNavCssClass); 59 | } 60 | }, 61 | enable = function () { 62 | applyStickNav(); 63 | win.on('resize', applyStickNav); 64 | }, 65 | init = function () { 66 | navBar = jquery('nav.wy-nav-side:first'); 67 | win = jquery(window); 68 | }; 69 | jquery(init); 70 | return { 71 | enable : enable 72 | }; 73 | }()); 74 | return { 75 | StickyNav : stickyNav 76 | }; 77 | }($)); 78 | 79 | // The code below is a copy of @seanmadsen code posted Jan 10, 2017 on issue 803. 80 | // https://github.com/mkdocs/mkdocs/issues/803 81 | // This just incorporates the auto scroll into the theme itself without 82 | // the need for additional custom.js file. 83 | // 84 | $(function() { 85 | $.fn.isFullyWithinViewport = function(){ 86 | var viewport = {}; 87 | viewport.top = $(window).scrollTop(); 88 | viewport.bottom = viewport.top + $(window).height(); 89 | var bounds = {}; 90 | bounds.top = this.offset().top; 91 | bounds.bottom = bounds.top + this.outerHeight(); 92 | return ( ! ( 93 | (bounds.top <= viewport.top) || 94 | (bounds.bottom >= viewport.bottom) 95 | ) ); 96 | }; 97 | if( $('li.toctree-l1.current').length && !$('li.toctree-l1.current').isFullyWithinViewport() ) { 98 | $('.wy-nav-side') 99 | .scrollTop( 100 | $('li.toctree-l1.current').offset().top - 101 | $('.wy-nav-side').offset().top - 102 | 60 103 | ); 104 | } 105 | }); 106 | -------------------------------------------------------------------------------- /site/search/worker.js: -------------------------------------------------------------------------------- 1 | var base_path = 'function' === typeof importScripts ? '.' : '/search/'; 2 | var allowSearch = false; 3 | var index; 4 | var documents = {}; 5 | var lang = ['en']; 6 | var data; 7 | 8 | function getScript(script, callback) { 9 | console.log('Loading script: ' + script); 10 | $.getScript(base_path + script).done(function () { 11 | callback(); 12 | }).fail(function (jqxhr, settings, exception) { 13 | console.log('Error: ' + exception); 14 | }); 15 | } 16 | 17 | function getScriptsInOrder(scripts, callback) { 18 | if (scripts.length === 0) { 19 | callback(); 20 | return; 21 | } 22 | getScript(scripts[0], function() { 23 | getScriptsInOrder(scripts.slice(1), callback); 24 | }); 25 | } 26 | 27 | function loadScripts(urls, callback) { 28 | if( 'function' === typeof importScripts ) { 29 | importScripts.apply(null, urls); 30 | callback(); 31 | } else { 32 | getScriptsInOrder(urls, callback); 33 | } 34 | } 35 | 36 | function onJSONLoaded () { 37 | data = JSON.parse(this.responseText); 38 | var scriptsToLoad = ['lunr.js']; 39 | if (data.config && data.config.lang && data.config.lang.length) { 40 | lang = data.config.lang; 41 | } 42 | if (lang.length > 1 || lang[0] !== "en") { 43 | scriptsToLoad.push('lunr.stemmer.support.js'); 44 | if (lang.length > 1) { 45 | scriptsToLoad.push('lunr.multi.js'); 46 | } 47 | for (var i=0; i < lang.length; i++) { 48 | if (lang[i] != 'en') { 49 | scriptsToLoad.push(['lunr', lang[i], 'js'].join('.')); 50 | } 51 | } 52 | } 53 | loadScripts(scriptsToLoad, onScriptsLoaded); 54 | } 55 | 56 | function onScriptsLoaded () { 57 | console.log('All search scripts loaded, building Lunr index...'); 58 | if (data.config && data.config.separator && data.config.separator.length) { 59 | lunr.tokenizer.separator = new RegExp(data.config.separator); 60 | } 61 | if (data.index) { 62 | index = lunr.Index.load(data.index); 63 | data.docs.forEach(function (doc) { 64 | documents[doc.location] = doc; 65 | }); 66 | console.log('Lunr pre-built index loaded, search ready'); 67 | } else { 68 | index = lunr(function () { 69 | if (lang.length === 1 && lang[0] !== "en" && lunr[lang[0]]) { 70 | this.use(lunr[lang[0]]); 71 | } else if (lang.length > 1) { 72 | this.use(lunr.multiLanguage.apply(null, lang)); // spread operator not supported in all browsers: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Spread_operator#Browser_compatibility 73 | } 74 | this.field('title'); 75 | this.field('text'); 76 | this.ref('location'); 77 | 78 | for (var i=0; i < data.docs.length; i++) { 79 | var doc = data.docs[i]; 80 | this.add(doc); 81 | documents[doc.location] = doc; 82 | } 83 | }); 84 | console.log('Lunr index built, search ready'); 85 | } 86 | allowSearch = true; 87 | postMessage({allowSearch: allowSearch}); 88 | } 89 | 90 | function init () { 91 | var oReq = new XMLHttpRequest(); 92 | oReq.addEventListener("load", onJSONLoaded); 93 | var index_path = base_path + '/search_index.json'; 94 | if( 'function' === typeof importScripts ){ 95 | index_path = 'search_index.json'; 96 | } 97 | oReq.open("GET", index_path); 98 | oReq.send(); 99 | } 100 | 101 | function search (query) { 102 | if (!allowSearch) { 103 | console.error('Assets for search still loading'); 104 | return; 105 | } 106 | 107 | var resultDocuments = []; 108 | var results = index.search(query); 109 | for (var i=0; i < results.length; i++){ 110 | var result = results[i]; 111 | doc = documents[result.ref]; 112 | doc.summary = doc.text.substring(0, 200); 113 | resultDocuments.push(doc); 114 | } 115 | return resultDocuments; 116 | } 117 | 118 | if( 'function' === typeof importScripts ) { 119 | onmessage = function (e) { 120 | if (e.data.init) { 121 | init(); 122 | } else if (e.data.query) { 123 | postMessage({ results: search(e.data.query) }); 124 | } else { 125 | console.error("Worker - Unrecognized message: " + e); 126 | } 127 | }; 128 | } 129 | -------------------------------------------------------------------------------- /docs/experiment_tracking/experiment.md: -------------------------------------------------------------------------------- 1 | # Experiment class API 2 | 3 | [[Github Code](https://github.com/williamFalcon/test-tube/blob/master/test_tube/log.py)] 4 | 5 | An Experiment holds metadata and the results of the training run, you 6 | can instantiate an `Experiment` via: 7 | 8 | ``` {.python} 9 | from test_tube import Experiment 10 | 11 | exp = Experiment(name='dense_model', 12 | debug=False, 13 | save_dir='/Desktop/test_tube') 14 | 15 | exp.tag({'learning_rate': 0.002, 'nb_layers': 2}) 16 | 17 | for step in training_steps: 18 | tng_err = model.eval(tng_x, tng_y) 19 | 20 | exp.log({'tng_err': tng_err}) 21 | 22 | # training complete! 23 | # all your logs and data are ready to be visualized at testtube.williamfalcon.com 24 | ``` 25 | 26 | ------------------------------------------------------------------------ 27 | 28 | ## init options 29 | 30 | ### version 31 | 32 | The same Experiment can have multiple versions. Test tube generates 33 | these automatically each time you run your model. To set your own 34 | version use: 35 | 36 | ``` {.python} 37 | exp = Experiment(name='dense_model',version=1) 38 | ``` 39 | 40 | ### debug 41 | 42 | If you're debugging and don't want to create a log file turn debug to 43 | True 44 | 45 | ``` {.python} 46 | exp = Experiment(name='dense_model',debug=True) 47 | ``` 48 | 49 | ### autosave 50 | 51 | If you only want to save at the end of training, turn autosave off: 52 | 53 | ``` {.python} 54 | exp = Experiment(name='dense_model', autosave=False) 55 | 56 | # run long training... 57 | 58 | # first time any logs are saved 59 | exp.save() 60 | ``` 61 | 62 | ### `create_git_tag` 63 | 64 | Ever wanted a flashback to your code when you ran an experiment? 65 | Snapshot your code for this experiment using git tags: 66 | 67 | ``` {.python} 68 | exp = Experiment(name='dense_model', create_git_tag=True) 69 | ``` 70 | 71 | ------------------------------------------------------------------------ 72 | 73 | ## Methods 74 | 75 | ### tag 76 | 77 | ``` {.python} 78 | exp.tag({k: v}) 79 | ``` 80 | 81 | Adds an arbitrary dictionary of tags to the experiment 82 | 83 | **Example** 84 | 85 | ``` {.python} 86 | exp.tag({'dataset_name': 'imagenet_1', 'learning_rate': 0.0002}) 87 | ``` 88 | 89 | ### log 90 | 91 | ``` {.python} 92 | exp.log({k:v}) 93 | ``` 94 | 95 | Adds a row of data to the experiments 96 | 97 | **Example** 98 | 99 | ``` {.python} 100 | exp.log({'val_loss': 0.22, 'epoch_nb': 1, 'batch_nb': 12}) 101 | 102 | # you can also add other rows that have separate information 103 | exp.log({'tng_loss': 0.01}) 104 | 105 | # or even a numpy array image 106 | image = np.imread('image.png') 107 | exp.log({'fake_png': image}) 108 | ``` 109 | 110 | **Saving images Example** 111 | 112 | ``` {.python} 113 | # name must have either jpg, png or jpeg in it 114 | img = np.imread('a.jpg') 115 | exp.log('test_jpg': img, 'val_err': 0.2) 116 | 117 | # saves image to ../exp/version/media/test_0.jpg 118 | # csv has file path to that image in that cell 119 | ``` 120 | 121 | To save an image, add `jpg`, `png` or `jpeg` to the key corresponding 122 | with the image array. The image must be formatted the same as skimage's 123 | [imsave](http://scikit-image.org/docs/dev/api/skimage.io.html#skimage.io.imsave) 124 | function 125 | 126 | ### argparse 127 | 128 | ``` {.python} 129 | exp.argparse(hparams) 130 | ``` 131 | 132 | Transfers hyperparam information from Argparser or 133 | HyperOptArgumentParser 134 | 135 | **Example** 136 | 137 | ``` {.python} 138 | from test_tube import HyperOptArgumentParser 139 | 140 | # parse args 141 | parser = HyperOptArgumentParser() 142 | parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate') 143 | hparams = parser.parse_args() 144 | 145 | # learning_rate is now a meta tag for your experiment 146 | exp.argparse(hparams) 147 | ``` 148 | 149 | ### save 150 | 151 | ``` {.python} 152 | exp.save() 153 | ``` 154 | 155 | Saves the exp to disk (including images) 156 | 157 | **Example** 158 | 159 | ``` {.python} 160 | exp = Experiment(name='dense_model', autosave=False) 161 | 162 | # run long training... 163 | 164 | # first time any logs are saved 165 | exp.save() 166 | ``` 167 | -------------------------------------------------------------------------------- /site/css/theme_extra.css: -------------------------------------------------------------------------------- 1 | /* 2 | * Sphinx doesn't have support for section dividers like we do in 3 | * MkDocs, this styles the section titles in the nav 4 | * 5 | * https://github.com/mkdocs/mkdocs/issues/175 6 | */ 7 | .wy-menu-vertical span { 8 | line-height: 18px; 9 | padding: 0.4045em 1.618em; 10 | display: block; 11 | position: relative; 12 | font-size: 90%; 13 | color: #838383; 14 | } 15 | 16 | .wy-menu-vertical .subnav a { 17 | padding: 0.4045em 2.427em; 18 | } 19 | 20 | /* 21 | * Long navigations run off the bottom of the screen as the nav 22 | * area doesn't scroll. 23 | * 24 | * https://github.com/mkdocs/mkdocs/pull/202 25 | * 26 | * Builds upon pull 202 https://github.com/mkdocs/mkdocs/pull/202 27 | * to make toc scrollbar end before navigations buttons to not be overlapping. 28 | */ 29 | .wy-nav-side { 30 | height: calc(100% - 45px); 31 | overflow-y: auto; 32 | min-height: 0; 33 | } 34 | 35 | .rst-versions{ 36 | border-top: 0; 37 | height: 45px; 38 | } 39 | 40 | @media screen and (max-width: 768px) { 41 | .wy-nav-side { 42 | height: 100%; 43 | } 44 | } 45 | 46 | /* 47 | * readthedocs theme hides nav items when the window height is 48 | * too small to contain them. 49 | * 50 | * https://github.com/mkdocs/mkdocs/issues/#348 51 | */ 52 | .wy-menu-vertical ul { 53 | margin-bottom: 2em; 54 | } 55 | 56 | /* 57 | * Wrap inline code samples otherwise they shoot of the side and 58 | * can't be read at all. 59 | * 60 | * https://github.com/mkdocs/mkdocs/issues/313 61 | * https://github.com/mkdocs/mkdocs/issues/233 62 | * https://github.com/mkdocs/mkdocs/issues/834 63 | */ 64 | code { 65 | white-space: pre-wrap; 66 | word-wrap: break-word; 67 | padding: 2px 5px; 68 | } 69 | 70 | /** 71 | * Make code blocks display as blocks and give them the appropriate 72 | * font size and padding. 73 | * 74 | * https://github.com/mkdocs/mkdocs/issues/855 75 | * https://github.com/mkdocs/mkdocs/issues/834 76 | * https://github.com/mkdocs/mkdocs/issues/233 77 | */ 78 | pre code { 79 | white-space: pre; 80 | word-wrap: normal; 81 | display: block; 82 | padding: 12px; 83 | font-size: 12px; 84 | } 85 | 86 | /* 87 | * Fix link colors when the link text is inline code. 88 | * 89 | * https://github.com/mkdocs/mkdocs/issues/718 90 | */ 91 | a code { 92 | color: #2980B9; 93 | } 94 | a:hover code { 95 | color: #3091d1; 96 | } 97 | a:visited code { 98 | color: #9B59B6; 99 | } 100 | 101 | /* 102 | * The CSS classes from highlight.js seem to clash with the 103 | * ReadTheDocs theme causing some code to be incorrectly made 104 | * bold and italic. 105 | * 106 | * https://github.com/mkdocs/mkdocs/issues/411 107 | */ 108 | pre .cs, pre .c { 109 | font-weight: inherit; 110 | font-style: inherit; 111 | } 112 | 113 | /* 114 | * Fix some issues with the theme and non-highlighted code 115 | * samples. Without and highlighting styles attached the 116 | * formatting is broken. 117 | * 118 | * https://github.com/mkdocs/mkdocs/issues/319 119 | */ 120 | .no-highlight { 121 | display: block; 122 | padding: 0.5em; 123 | color: #333; 124 | } 125 | 126 | 127 | /* 128 | * Additions specific to the search functionality provided by MkDocs 129 | */ 130 | 131 | .search-results { 132 | margin-top: 23px; 133 | } 134 | 135 | .search-results article { 136 | border-top: 1px solid #E1E4E5; 137 | padding-top: 24px; 138 | } 139 | 140 | .search-results article:first-child { 141 | border-top: none; 142 | } 143 | 144 | form .search-query { 145 | width: 100%; 146 | border-radius: 50px; 147 | padding: 6px 12px; /* csslint allow: box-model */ 148 | border-color: #D1D4D5; 149 | } 150 | 151 | .wy-menu-vertical li ul { 152 | display: inherit; 153 | } 154 | 155 | .wy-menu-vertical li ul.subnav ul.subnav{ 156 | padding-left: 1em; 157 | } 158 | 159 | .wy-menu-vertical .subnav li.current > a { 160 | padding-left: 2.42em; 161 | } 162 | .wy-menu-vertical .subnav li.current > ul li a { 163 | padding-left: 3.23em; 164 | } 165 | 166 | /* 167 | * Improve inline code blocks within admonitions. 168 | * 169 | * https://github.com/mkdocs/mkdocs/issues/656 170 | */ 171 | .admonition code { 172 | color: #404040; 173 | border: 1px solid #c7c9cb; 174 | border: 1px solid rgba(0, 0, 0, 0.2); 175 | background: #f8fbfd; 176 | background: rgba(255, 255, 255, 0.7); 177 | } 178 | 179 | /* 180 | * Account for wide tables which go off the side. 181 | * Override borders to avoid wierdness on narrow tables. 182 | * 183 | * https://github.com/mkdocs/mkdocs/issues/834 184 | * https://github.com/mkdocs/mkdocs/pull/1034 185 | */ 186 | .rst-content .section .docutils { 187 | width: 100%; 188 | overflow: auto; 189 | display: block; 190 | border: none; 191 | } 192 | 193 | td, th { 194 | border: 1px solid #e1e4e5 !important; /* csslint allow: important */ 195 | border-collapse: collapse; 196 | } 197 | 198 | -------------------------------------------------------------------------------- /site/404.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |Page not found
116 | 117 | 118 |
2 |
3 |
4 |
5 |
10 | Log, organize and parallelize hyperparameter search for Deep Learning experiments 11 |
12 |
13 | 
14 | 
15 |
16 |
17 |
Test Tube allows you to easily log metadata and track your machine 138 | learning experiments.
139 |Use Test Tube if you need to:
140 |Test Tube is compatible with: Python 2 and 3
151 |from test_tube import Experiment
155 |
156 | exp = Experiment(name='dense_model',
157 | debug=False,
158 | save_dir='/Desktop/test_tube')
159 |
160 | exp.tag({'learning_rate': 0.002, 'nb_layers': 2})
161 |
162 | for step in training_steps:
163 | tng_err = model.eval(tng_x, tng_y)
164 |
165 | exp.log('tng_err': tng_err)
166 |
167 | # training complete!
168 | # all your logs and data are ready to be visualized at testtube.williamfalcon.com
169 | from test_tube import HyperOptArgumentParser
174 |
175 | # subclass of argparse
176 | parser = HyperOptArgumentParser(strategy='random_search')
177 | parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
178 |
179 | # let's enable optimizing over the number of layers in the network
180 | parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
181 |
182 | # and tune the number of units in each layer
183 | parser.opt_range('--neurons', default=50, type=int, tunable=True, low=100, high=800, nb_samples=10)
184 |
185 | # compile (because it's argparse underneath)
186 | hparams = parser.parse_args()
187 |
188 | # run 20 trials of random search over the hyperparams
189 | for hparam_trial in hparams.trials(20):
190 | train_network(hparam_trial)
191 | import pandas as pd
196 | import matplotlib
197 |
198 | # each experiment is saved to a metrics.csv file which can be imported anywhere
199 | # images save to exp/version/images
200 | df = pd.read_csv('../some/dir/test_tube_data/dense_model/version_0/metrics.csv')
201 | df.tng_err.plot()
202 | An Experiment holds metadata and the results of the training run, you
145 | can instantiate an Experiment via:
from test_tube import Experiment
147 |
148 | exp = Experiment(name='dense_model',
149 | debug=False,
150 | save_dir='/Desktop/test_tube')
151 |
152 | exp.tag({'learning_rate': 0.002, 'nb_layers': 2})
153 |
154 | for step in training_steps:
155 | tng_err = model.eval(tng_x, tng_y)
156 |
157 | exp.log('tng_err': tng_err)
158 |
159 | # training complete!
160 | # all your logs and data are ready to be visualized at testtube.williamfalcon.com
161 | The same Experiment can have multiple versions. Test tube generates 167 | these automatically each time you run your model. To set your own 168 | version use:
169 |exp = Experiment(name='dense_model',version=1)
170 | If you're debugging and don't want to create a log file turn debug to 174 | True
175 |exp = Experiment(name='dense_model',debug=True)
176 | If you only want to save at the end of training, turn autosave off:
180 |exp = Experiment(name='dense_model', autosave=False)
181 |
182 | # run long training...
183 |
184 | # first time any logs are saved
185 | exp.save()
186 | create_git_tagEver wanted a flashback to your code when you ran an experiment? 190 | Snapshot your code for this experiment using git tags:
191 |exp = Experiment(name='dense_model', create_git_tag=True)
192 | exp.tag({k: v})
198 | Adds an arbitrary dictionary of tags to the experiment
201 |Example
202 |exp.tag({'dataset_name': 'imagenet_1', 'learning_rate': 0.0002})
203 | exp.log({k:v})
207 | Adds a row of data to the experiments
210 |Example
211 |exp.log({'val_loss': 0.22, 'epoch_nb': 1, 'batch_nb': 12})
212 |
213 | # you can also add other rows that have separate information
214 | exp.log({'tng_loss': 0.01})
215 |
216 | # or even a numpy array image
217 | image = np.imread('image.png')
218 | exp.log({'fake_png': image})
219 | Saving images Example
222 |# name must have either jpg, png or jpeg in it
223 | img = np.imread('a.jpg')
224 | exp.log('test_jpg': img, 'val_err': 0.2)
225 |
226 | # saves image to ../exp/version/media/test_0.jpg
227 | # csv has file path to that image in that cell
228 | To save an image, add jpg, png or jpeg to the key corresponding
231 | with the image array. The image must be formatted the same as skimage's
232 | imsave
233 | function
exp.argparse(hparams)
236 | Transfers hyperparam information from Argparser or 239 | HyperOptArgumentParser
240 |Example
241 |from test_tube import HyperOptArgumentParser
242 |
243 | # parse args
244 | parser = HyperOptArgumentParser()
245 | parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
246 | hparams = parser.parse_args()
247 |
248 | # learning_rate is now a meta tag for your experiment
249 | exp.argparse(hparams)
250 | exp.save()
254 | Saves the exp to disk (including images)
257 |Example
258 |exp = Experiment(name='dense_model', autosave=False)
259 |
260 | # run long training...
261 |
262 | # first time any logs are saved
263 | exp.save()
264 | The HyperOptArgumentParser is a subclass of python's 145 | argparse, with added 146 | finctionality to change parameters on the fly as determined by a 147 | sampling strategy.
148 |You can instantiate an HyperOptArgumentParser via:
from test_tube import HyperOptArgumentParser
150 |
151 | # subclass of argparse
152 | parser = HyperOptArgumentParser(strategy='random_search')
153 | parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
154 |
155 | # let's enable optimizing over the number of layers in the network
156 | parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
157 |
158 | # and tune the number of units in each layer
159 | parser.opt_range('--neurons', default=50, type=int, tunable=True, low=100, high=800, nb_samples=10)
160 |
161 | # compile (because it's argparse underneath)
162 | hparams = parser.parse_args()
163 |
164 | # run 20 trials of random search over the hyperparams
165 | for hparam_trial in hparams.trials(20):
166 | train_network(hparam_trial)
167 | strategyUse either random 173 | search 174 | or grid 175 | search 176 | for tuning:
177 |parser = HyperOptArgumentParser(strategy='grid_search')
178 | All the functionality from argparse works but we've added the following 183 | functionality:
184 |opt_listparser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
186 | Enables searching over a list of values for this parameter. The tunable 189 | values ONLY replace the argparse values when running a hyperparameter 190 | optimization search. This is on purpose so your code doesn't have to 191 | change when you want to tune it.
192 |Example
193 |parser.opt_list('--nb_layers', default=2, type=int, tunable=True, options=[2, 4, 8])
194 | hparams = parser.parse_args()
195 | # hparams.nb_layers = 2
196 |
197 | for trial in hparams.trials(2):
198 | # trial.nb_layers is now a value in [2, 4, 8]
199 | # but hparams.nb_layers is still 2
200 | opt_rangeparser.opt_range('--neurons', default=50, type=int, tunable=True, low=100, high=800, nb_samples=8, log_base=None)
204 | Enables searching over a range of values chosen randomly using the
207 | nb_samples given. The tunable values only replace the argparse
208 | values when running a hyperparameter optimization search. This is on
209 | purpose so your code doesn't have to change when you want to tune it.
If log_base is set to a positive number, it will randomly search over
211 | a log scale, where the log base is log_base. This is better for search
212 | over several orders of magnitude efficiently.
Example
214 |parser.opt_range('--neurons', default=50, type=int, tunable=True, low=100, high=800, nb_samples=8)
215 | hparams = parser.parse_args()
216 | # hparams.neurons = 50
217 |
218 | for trial in hparams.trials(2):
219 | # trial.nb_layers is now a value in [100, 200, 300, 400, 500, 600 700, 800]
220 | # but hparams.neurons is still 50
221 | json_configparser.json_config('--config', default='example.json')
225 | Replaces default values in the parser with those read from the json file
228 |Example
229 |example.json
230 |{
231 | "learning_rate": 200
232 | }
233 | parser.add_argument('--learning_rate', default=0.002, type=float, help='the learning rate')
236 | parser.json_config('--config', default='example.json')
237 | hparams = parser.parse_args()
238 |
239 | # hparams.learning_rate = 200
240 | trial_generator = hparams.trials(2)
244 | Computes the trials needed for these experiments and serves them via a 247 | generator
248 |Example
249 |hparams = parser.parse_args()
250 | for trial_hparams in hparams.trials(2):
251 | # trial_hparams now has values sampled from the training routine
252 | optimize_parallel_gpuhparams = parser.parse_args()
256 | hparams.optimize_parallel_gpu(function_to_optimize, gpu_ids=['1', '0, 2'])
257 | Parallelize the trials across nb_workers processes. Auto assign the
260 | correct gpus. Argument passed into the function_to_optimize is the
261 | trial_params argument and the gpu_ids.
Example
263 |# parallelize tuning on 2 gpus
264 | # this will place each trial in n into a given gpu
265 | def train_main(trial_params, gpu_ids):
266 | # train your model, etc here...
267 |
268 | hparams = parser.parse_args()
269 | hparams.optimize_parallel_gpu(train_main, gpu_ids=['1', '0, 2'])
270 |
271 | # at the end of the optimize_parallel function, all 20 trials will be completed
272 | # in this case by running 10 sets of 2 trials in parallel
273 | optimize_parallel_cpuhparams = parser.parse_args()
277 | hparams.optimize_parallel_cpu(function_to_optimize, nb_trials=20, nb_workers=2)
278 | Parallelize the trials across nb_workers cpus. Argument passed into
281 | the function_to_optimize is the trial_params argument.
Example
283 |# parallelize tuning on 2 cpus
284 | # this will place each trial in n into a given gpu
285 | def train_main(trial_params):
286 | # train your model, etc here...
287 |
288 | hparams = parser.parse_args()
289 | hparams.optimize_parallel_cpu(train_main, nb_trials=20, nb_workers=2)
290 |
291 | # at the end of the optimize_parallel function, all 20 trials will be completed
292 | # in this case by running 10 sets of 2 trials in parallel
293 | The SlurmCluster class enables hyperparameter search parallelization on a cluster managed via Slurm workload manager.
147 |At a high level, the SlurmCluster creates a submit script for each permutation of hyperparameters requested. If the job hits the walltime but has not completed, the SlurmManager will checkpoint the model and submit a new job to continue training using the saved weights.
148 |You can instantiate a SlurmCluster via:
from test_tube.hpc import SlurmCluster
154 |
155 | # hyperparameters is a test-tube hyper params object
156 | # see https://williamfalcon.github.io/test-tube/hyperparameter_optimization/HyperOptArgumentParser/
157 | hyperparams = args.parse()
158 |
159 | # init cluster
160 | cluster = SlurmCluster(
161 | hyperparam_optimizer=hyperparams,
162 | log_path='/path/to/log/results/to',
163 | python_cmd='python3'
164 | )
165 |
166 | # let the cluster know where to email for a change in job status (ie: complete, fail, etc...)
167 | cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True)
168 |
169 | # set the job options. In this instance, we'll run 20 different models
170 | # each with its own set of hyperparameters giving each one 1 GPU (ie: taking up 20 GPUs)
171 | cluster.per_experiment_nb_gpus = 1
172 | cluster.per_experiment_nb_nodes = 1
173 |
174 | # we'll request 10GB of memory per node
175 | cluster.memory_mb_per_node = 10000
176 |
177 | # set a walltime of 10 minues
178 | cluster.job_time = '10:00'
179 |
180 | # 1 minute before walltime is up, SlurmCluster will launch a continuation job and kill this job.
181 | # you must provide your own loading and saving function which the cluster object will call
182 | cluster.minutes_to_checkpoint_before_walltime = 1
183 |
184 | # run the models on the cluster
185 | cluster.optimize_parallel_cluster_gpu(train, nb_trials=20, job_name='first_tt_batch', job_display_name='my_batch')
186 | hyperparam_optimizerA HyperOptArgumentParser object 192 | which contains all permutations of model hyperparameters to run.
193 |log_pathPath to save the slurm scripts, error logs and out logs created. Usually this would be the experiments folder path where test tube saves Experiment information.
195 |python_cmdThis is the command that starts the python program. Normally it is:
197 |# python 2
198 | python main.py
199 |
200 | # python 3
201 | python3 main.py
202 | enable_log_errIf true, saves slurm error logs to the path at log_path. If anything goes wrong in your job, you'll find the error here.
206 |enable_log_outIf true, saves slurm output logs to the path at log_path. This file contains all outputs that would show up on the console normally.
208 |test_tube_exp_nameWhen this is given, it structures the files in a nice format to fit with the folder structure of the Experiment object's output.
210 |job_time
212 | String. Walltime requested. Examples:
# 1 hour and 10 minutes
214 | cluster.job_time = '1:10:00'
215 |
216 | # 1 day and 1 hour and 10 minutes
217 | cluster.job_time = '1-1:10:00'
218 |
219 | # 1 day and 1 hour and 10 minutes
220 | cluster.job_time = '25:10:00'
221 |
222 | # 10 minutes
223 | cluster.job_time = '10:00'
224 |
225 | # 10 seconds
226 | cluster.job_time = '10'
227 | minutes_to_checkpoint_before_walltime
230 | Int. Minutes before walltime when a continuation job will be auto-submitted.
cluster.job_time = '10:00'
232 | cluster.minutes_to_checkpoint_before_walltime = 2
233 |
234 | # New job will be submited to continue training after 8 minutes of the job running.
235 | per_experiment_nb_gpus
238 | Int. Number of GPUs each job will get.
# EACH job will get 2 GPUs (ie: if a model runs over two GPUs at the same time).
240 | cluster.per_experiment_nb_gpus = 2
241 | per_experiment_nb_cpus
244 | Int. Number of CPUs each job will get.
cluster.per_experiment_nb_cpus = 1
246 | per_experiment_nb_nodes
249 | Int. Number of nodes each job will get.
cluster.per_experiment_nb_nodes = 1
251 | gpu_type
254 | String. Gpu type requested. Example:
cluster.gpu_type = '1080ti'
256 | set_checkpoint_save_functioncluster.set_checkpoint_save_function(fx, kwargs)
262 | Called if the model isn't finished training minutes_to_checkpoint_before_walltime before the walltime. If walltime = '15:00' and minutes_to_checkpoint_before_walltime = '1:00' the SlurmCluster will call your save function after 14 minutes of training.
265 |fx A python function. kwargs Dictionary where keys are the literal argument names to the function. Dictionary values are the values of the arguments. Example
270 |def my_save_function(arg_1, arg_k):
271 | # ... save my model here
272 |
273 | cluster.set_checkpoint_save_function(my_save_function, kwargs={'arg_1': 'whatever', 'arg_k': 'you_want'})
274 |
275 | set_checkpoint_load_functioncluster.set_checkpoint_load_function(fx, kwargs)
279 | Called internally when a job is auto-submitted by the SlurmCluster to give your program a chance to load the model weights or whatever you need to continue training.
282 | This will call your load function immediately whenever you call this method AND training is continuing.
fx A python function. kwargs Dictionary where keys are the literal argument names to the function. Dictionary values are the values of the arguments. Example
288 |def my_load_function(arg_1, arg_k):
289 | # ... restore my model here
290 |
291 | cluster.set_checkpoint_save_function(my_load_function, kwargs={'arg_1': 'whatever', 'arg_k': 'you_want'})
292 |
293 | add_slurm_cmdcluster.add_slurm_cmd(cmd, value, comment)
297 | Adds whatever Slurm command you need manually to the generated script. All possible commands are listed here.
300 |cmd String with the bash command. value String value for the command. Numericals need to be in single quotes '1' comment String with the command comment. Example
306 |cluster.add_slurm_cmd(cmd='cpus-per-task', value='1', comment='nb cpus per task')
307 |
308 | # the above command will add an entry like this to the slurm script
309 |
310 | # #nb cpus per task
311 | # #SBATCH --cpus-per-task=1
312 | # ############
313 |
314 | add_commandcluster.add_command(cmd)
318 | Adds arbitrary bash commands to the script. Use this to activate conda environments, install packages, whatever else you would think about calling on bash.
321 |cmd String with your bash command. Example
325 |# load the anaconda package on the launch node
326 | cluster.add_command('module load anaconda')
327 |
328 | # activate the environment on the launch node
329 | cluster.add_command('source activate myCondaEnv')
330 | load_modulescluster.load_modules(modules)
334 | Loads modules needed to run the job. Your Slurm documentation should have a list of available modules. You can also get those by running module avail.
337 | - modules Array of module names.
Example
339 |cluster.load_modules([
340 | 'python-3',
341 | 'anaconda3'
342 | ])
343 | notify_job_statuscluster.notify_job_status(email, on_done, on_fail)
347 | Loads modules needed to run the job. Your Slurm documentation should have a list of available modules. You can also get those by running module avail.
email String. Email address to get notifications. on_done Boolean. If true, you'll get an email when the job completes. on_fail Boolean. If true, you'll get an email if the job fails. Example
356 |cluster.notify_job_status(email='some@email.com', on_done=True, on_fail=True)
357 | optimize_parallel_cluster_gpucluster.optimize_parallel_cluster_gpu(train_function, nb_trials, job_name, job_display_name=None)
361 | Launches the hyperparameter search across the cluster nodes.
364 | - train_function The entry point to start your training routine.
365 | - nb_trials Number of trials to launch. This is the number of hyperparameter configurations to train over.
366 | - job_name Folder name where the slurm scripts will save to. This should be the same as your Experiment name.
367 | - job_display_name Visible name when slurm lists running jobs (ie: through squeue -u user_name). This should be really short (if using a test tube Experiment, it'll put the experiment version at the end).
Example
369 |def main(hparams, cluster, return_dict):
370 | # do your own generic training code here...
371 | # init model
372 | model = model_build(hparams)
373 |
374 | # set the load and save fxs
375 | cluster.set_checkpoint_save_function(fx, {})
376 | cluster.set_checkpoint_load_function(fx, {})
377 |
378 | # train ...
379 |
380 |
381 | cluster.optimize_parallel_cluster_gpu(main, nb_trials=20, job_name='my_job', job_display_name='mj')
382 | Now if you get the job information, you'll see this:
385 |(conda_env) [user@node dir]$ squeue -u my_name
386 | JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)
387 | 104040 all mjv0 my_name R 58:22 1 nodeName
388 | 104041 all mjv1 my_name R 58:22 1 nodeName
389 | 104042 all mjv2 my_name R 58:22 1 nodeName
390 | 104043 all mjv3 my_name R 58:22 1 nodeName
391 | optimize_parallel_cluster_cpucluster.optimize_parallel_cluster_cpu(train_function, nb_trials, job_name, job_display_name=None)
395 | Launches the hyperparameter search across the cluster nodes using cpus.
398 | - train_function The entry point to start your training routine.
399 | - nb_trials Number of trials to launch. This is the number of hyperparameter configurations to train over.
400 | - job_name Folder name where the slurm scripts will save to. This should be the same as your Experiment name.
401 | - job_display_name Visible name when slurm lists running jobs (ie: through squeue -u user_name). This should be really short (if using a test tube Experiment, it'll put the experiment version at the end).
Example
403 |def main(hparams, cluster, return_dict):
404 | # do your own generic training code here...
405 | # init model
406 | model = model_build(hparams)
407 |
408 | # set the load and save fxs
409 | cluster.set_checkpoint_save_function(fx, {})
410 | cluster.set_checkpoint_load_function(fx, {})
411 |
412 | # train ...
413 |
414 |
415 | cluster.optimize_parallel_cluster_cpu(main, nb_trials=20, job_name='my_job', job_display_name='mj')
416 | Now if you get the job information, you'll see this:
419 |(conda_env) [user@node dir]$ squeue -u my_name
420 | JOBID PARTITION NAME USER ST TIME NODES NODELIST(REASON)
421 | 104040 all mjv0 my_name R 58:22 1 nodeName
422 | 104041 all mjv1 my_name R 58:22 1 nodeName
423 | 104042 all mjv2 my_name R 58:22 1 nodeName
424 | 104043 all mjv3 my_name R 58:22 1 nodeName
425 |