├── .gitignore ├── LICENSE ├── README.md ├── __init__.py ├── app_base.py ├── bindings.py ├── caffevis ├── __init__.py ├── app.py ├── caffe_proc_thread.py ├── caffevis_app_state.py ├── caffevis_helper.py └── jpg_vis_loading_thread.py ├── codependent_thread.py ├── doc ├── computing_per_unit_visualizations.md ├── deep-vis-on-aws.md ├── example_bvlc-googlenet_bus.300.jpg ├── example_bvlc-googlenet_bus.jpg ├── example_caffenet-yos_bus_wheel_unit.300.jpg ├── example_caffenet-yos_bus_wheel_unit.jpg ├── example_caffenet-yos_webcam.300.jpg ├── example_caffenet-yos_webcam.jpg ├── opt_fc8_0130_0_best_X.jpg ├── opt_fc8_0130_0_best_Xpm.jpg └── running-optimize-image.md ├── find_maxes ├── caffe_misc.py ├── crop_max_patches.py ├── find_max_acts.py ├── jby_misc.py ├── loaders.py └── max_tracker.py ├── image_misc.py ├── input_fetcher.py ├── input_images ├── .gitignore ├── ILSVRC2012_val_00000610.jpg ├── ILSVRC2012_val_00006451.jpg ├── ILSVRC2012_val_00006491.jpg ├── ILSVRC2012_val_00008338.jpg ├── ILSVRC2012_val_00011425.jpg ├── ILSVRC2012_val_00011425_gray.jpg ├── ILSVRC2012_val_00016655.jpg ├── ILSVRC2012_val_00018478.jpg ├── ILSVRC2012_val_00018548.jpg ├── ILSVRC2012_val_00020333.jpg ├── ILSVRC2012_val_00029739.jpg └── ILSVRC2012_val_00046554.jpg ├── keys.py ├── live_vis.py ├── misc.py ├── models ├── .gitignore ├── bvlc-googlenet │ ├── bvlc-googlenet-deploy.prototxt │ ├── fetch.sh │ ├── ilsvrc_2012_labels.txt │ └── settings_local.template-bvlc-googlenet.py ├── caffenet-yos │ ├── .gitignore │ ├── caffenet-yos-deploy.prototxt │ ├── fetch.sh │ ├── ilsvrc_2012_labels.txt │ └── settings_local.template-caffenet-yos.py └── squeezenet │ ├── deploy.prototxt │ ├── fetch.sh │ ├── ilsvrc_2012_labels.txt │ └── settings_local.template-squeezenet.py ├── numpy_cache.py ├── optimize ├── __init__.py └── gradient_optimizer.py ├── optimize_image.py ├── run_toolbox.py ├── settings.py └── test_keys.py /.gitignore: -------------------------------------------------------------------------------- 1 | settings_local.py 2 | settings.py.templatec 3 | *.pyc 4 | *.DS_Store 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Jason Yosinski 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Deep Visualization Toolbox 2 | 3 | This is the code required to run the Deep Visualization Toolbox, as well as to generate the neuron-by-neuron visualizations using regularized optimization. 4 | The toolbox and methods are described casually [here](http://yosinski.com/deepvis) and more formally in this paper: 5 | 6 | * Jason Yosinski, Jeff Clune, Anh Nguyen, Thomas Fuchs, and Hod Lipson. [Understanding neural networks through deep visualization](http://arxiv.org/abs/1506.06579). Presented at the Deep Learning Workshop, International Conference on Machine Learning (ICML), 2015. 7 | 8 | If you find this paper or code useful, we encourage you to cite the paper. BibTeX: 9 | 10 | @inproceedings{yosinski-2015-ICML-DL-understanding-neural-networks, 11 | Author = {Jason Yosinski and Jeff Clune and Anh Nguyen and Thomas Fuchs and Hod Lipson}, 12 | Booktitle = {Deep Learning Workshop, International Conference on Machine Learning (ICML)}, 13 | Title = {Understanding Neural Networks Through Deep Visualization}, 14 | Year = {2015}} 15 | 16 | 17 | 18 | # Features 19 | 20 | The main toolbox window looks like this, here showing a convolutional unit that responds to automobile wheels: 21 | 22 | ![DeepVis Toolbox Screenshot bus](doc/example_caffenet-yos_bus_wheel_unit.jpg?raw=true) 23 | 24 | For a quick tour of the toolbox features, including what each pane of the above interface is showing, watch this [4 min YouTube video](https://www.youtube.com/watch?v=AgkfIQ4IGaM). In addition to processing images files from disk, the toolbox can run off a webcam for live network visualization **(below left)**. 25 | The toolbox comes bundled with the default [caffenet-yos](models/caffenet-yos) model weights and pre-computed per-unit visualizations shown in the paper. Weights, but not per-unit visualizations, for [bvlc-googlenet](models/bvlc-googlenet) **(below right)** and [squeezenet](models/squeezenet) can be downloaded by scripts in their respective directories. 26 | 27 | [![DeepVis Toolbox Screenshot webcam](doc/example_caffenet-yos_webcam.300.jpg)](doc/example_caffenet-yos_webcam.jpg?raw=true) 28 | [![DeepVis Toolbox Screenshot bvlc-googlenet](doc/example_bvlc-googlenet_bus.300.jpg)](doc/example_bvlc-googlenet_bus.jpg?raw=true) 29 | 30 | You can visualize your own model as well. However, note that the toolbox provides two rather separate sets of features; the first is easy to use with your own model, and the second is more involved: 31 | 32 | 1. **Forward/backward prop**: Images can be run forward through the network to visualize activations, and derivatives of any unit with respect to any other unit can be computed using backprop. In addition to traditional backprop, deconv from [Zeiler and Fergus (2014)](https://scholar.google.com/scholar?q=Zeiler+Visualizing+and+understanding+convolutional+networks) is supported as a way of flowing information backwards through the network. Doing forward and backward passes works for any model that can be run in Caffe (including yours!). 33 | 34 | 2. **Per-unit visualizations**: Three types of per-unit visualizations can be computed for a network — max image, deconv of max image, activation maximization via regularized optimization — but these visualizations must be computed *outside* the toolbox and saved as jpg. The toolbox then loads these jpgs to display alongside units as they are selected. Visualizations must be pre-computed because they are far too expensive to run live. For example, going through the 1.3m image training set to find the images causing top-9 activations took 40 hours on our system (for all units). Per-unit visualization jpgs are provided for the caffenet-yos model, but not for the bvlc-googlenet or squeezenet models (and not for yours, but you can [compute them yourself](doc/computing_per_unit_visualizations.md)). 35 | 36 | Summary: 37 | 38 | | Model | Forward/Backward prop | Per-unit visualizations | 39 | | ------------- | ---------------- | ----------------------- | 40 | | [caffenet-yos](models/caffenet-yos) | **easy** | **included** | 41 | | [bvlc-googlenet](bvlc-googlenet) | **easy** | not-included, [generate](doc/computing_per_unit_visualizations.md) if desired | 42 | | [squeezenet](models/squeezenet) | **easy** | not-included, [generate](doc/computing_per_unit_visualizations.md) if desired | 43 | | your network | **easy** (just point to your model in `settings_local.py`) | not-included, [generate](doc/computing_per_unit_visualizations.md) if desired | 44 | 45 | 46 | 47 | # Setting up and running the toolbox 48 | 49 | ### Step 0: Compile master branch of caffe (optional but recommended) 50 | 51 | Checkout the master branch of [Caffe](http://caffe.berkeleyvision.org/) and compile it on your 52 | machine. If you've never used Caffe before, it can take a bit of time to get all the required libraries in place. Fortunately, the [installation process is well documented](http://caffe.berkeleyvision.org/installation.html). When you're installing the OpenCV dependency, install the Python bindings as well (see Step 2 below). 53 | 54 | Note: When compiling Caffe, you can set `CPU_ONLY := 1` in your `Makefile.config` to skip all the Cuda/GPU stuff. The Deep Visualization Toolbox can run with Caffe in either CPU or GPU mode, and it's simpler to get Caffe to compile for the first time in `CPU_ONLY` mode. If Caffe is compiled with GPU options enabled, CPU vs. GPU may be switched at runtime via a setting in `settings_local.py`. Also, cuDNN may be enabled or disabled by recompiling Caffe with or without cuDNN. 55 | 56 | 57 | ### Step 1: Compile the deconv-deep-vis-toolbox branch of caffe 58 | 59 | Instead of using the master branch of Caffe, to use the demo 60 | you'll need the slightly modified [deconv-deep-vis-toolbox Caffe branch](https://github.com/yosinski/caffe/tree/deconv-deep-vis-toolbox) (supporting deconv and a few 61 | extra Python bindings). Getting the branch and switching to it is easy. 62 | Starting from your Caffe directory (that is, the directory where you've checked out Caffe, *not* the directory where you've checked out the DeepVis Toolbox), run: 63 | 64 | $ git remote add yosinski https://github.com/yosinski/caffe.git 65 | $ git fetch --all 66 | $ git checkout --track -b deconv-deep-vis-toolbox yosinski/deconv-deep-vis-toolbox 67 | $ < edit Makefile.config to suit your system if not already done in Step 0 > 68 | $ make clean 69 | $ make -j 70 | $ make -j pycaffe 71 | 72 | As noted above, feel free to compile in `CPU_ONLY` mode. 73 | 74 | 75 | 76 | ### Step 2: Install prerequisites 77 | 78 | The only prerequisites beyond those required for Caffe are `python-opencv`, `scipy`, and `scikit-image`, which may be installed as follows (other install options exist as well): 79 | 80 | #### Ubuntu: 81 | 82 | $ sudo apt-get install python-opencv scipy python-skimage 83 | 84 | #### Mac using [homebrew](http://brew.sh/): 85 | 86 | Install `python-opencv` using one of the following two lines, depending on whether you want to compile using Intel TBB to enable parallel operations: 87 | 88 | $ brew install opencv 89 | $ brew install --with-tbb opencv 90 | 91 | Install `scipy` either with OpenBLAS... 92 | 93 | $ brew install openblas 94 | $ brew install --with-openblas scipy 95 | 96 | ...or without it 97 | 98 | $ brew install scipy 99 | 100 | And install `scikit-image` using pip: 101 | 102 | $ pip install scikit-image 103 | 104 | You may have already installed the `python-opencv` bindings as part of the Caffe setup process. If `import cv2` works from Python, then you're all set. Similarly for `import scipy` and `import skimage`. 105 | 106 | 107 | 108 | ### Step 3: Download and configure Deep Visualization Toolbox code 109 | 110 | You can put it wherever you like: 111 | 112 | $ git clone https://github.com/yosinski/deep-visualization-toolbox 113 | $ cd deep-visualization-toolbox 114 | 115 | The settings in the latest version of the toolbox (February 2016) work a bit differently than in earlier versions (April 2015). If you have the latest version (recommended!), 116 | the minimal steps are to create a `settings_local.py` file using the template for the default `caffenet-yos` model: 117 | 118 | $ cp models/caffenet-yos/settings_local.template-caffenet-yos.py settings_local.py 119 | 120 | And then edit the `settings_local.py` file to make the `caffevis_caffe_root` variable point to the directory where you've compiled caffe in Step 1: 121 | 122 | $ < edit settings_local.py > 123 | 124 | *Note on settings:* Settings are now split into two files: a versioned `settings.py` file that provides documentation and default values for all settings and an unversioned `settings_local.py` file. This latter file allows you to override any default setting to tailor the toolbox to your specific setup (Caffe path, CPU vs. GPU, webcam device, etc) and model (model weights, prototxt, sizes of the various panels shown in the toolbox, etc). This also makes it easy to distribute settings tweaks alongside models: for example, `models/bvlc-googlenet/settings_local.template-bvlc-googlenet.py` includes the appropriate window pane sizes and so on for the `bvlc-googlenet` model. To load a new model, just change the details in `settings_local.py`, perhaps by copying from the included template. 125 | 126 | Finally, download the default model weights and corresponding top-9 visualizations saved as jpg (downloads a 230MB model and 1.1GB of jpgs to show as visualization): 127 | 128 | $ cd models/caffenet-yos/ 129 | $ ./fetch.sh 130 | $ cd ../.. 131 | 132 | 133 | 134 | ### Step 4: Run it! 135 | 136 | Simple: 137 | 138 | $ ./run_toolbox.py 139 | 140 | Once the toolbox is running, push 'h' to show a help screen. You can also have a look at `bindings.py` to see what the various keys do. If the window is too large or too small for your screen, set the `global_scale` and `global_font_size` variables in `settings_local.py` to values smaller or larger than 1.0. 141 | 142 | 143 | 144 | # Troubleshooting 145 | 146 | If you have any problems running the Deep Vis Toolbox, here are a few things to try: 147 | 148 | * Make sure you can compile the master branch of Caffe (Step 0 above)! If you can't, see the [detailed compilation instructions for Caffe](http://caffe.berkeleyvision.org/installation.html). If you encounter issues, the [caffe-users](https://groups.google.com/forum/#!forum/caffe-users) mailing list is a good place to look for solutions others have found. 149 | * Try using the `dev` branch of this toolbox instead of `master` (`git checkout dev`). Sometimes it's a little more up to date. 150 | * If you get an error (`AttributeError: 'Classifier' object has no attribute 'backward_from_layer'`) when switching to backprop or deconv modes, it's because your compiled branch of Caffe does not have the necessary Python bindings for backprop/deconv. Follow the directions in "Step 1: Compile the deconv-deep-vis-toolbox branch of caffe" above. 151 | * If the backprop pane in the lower left is just gray, it's probably because backprop and deconv are producing all zeros. By default, Caffe won't compute derivatives at the data layer, because they're not needed to update parameters. The fix is simple: just add `force_backward: true` to your network prototxt, [like this](https://github.com/yosinski/deep-visualization-toolbox/blob/master/models/caffenet-yos/caffenet-yos-deploy.prototxt#L7). 152 | * If the toolbox runs but the keys don't respond as expected, this may be because keys behave differently on different platforms. Run the `test_keys.py` script to test behavior on your system. 153 | * If none of that helps, feel free to [email me](http://yosinski.com/) or [submit an issue](https://github.com/yosinski/deep-visualization-toolbox/issues). I might have left out an important detail here or there :). 154 | 155 | 156 | 157 | # Other ways of running the toolbox 158 | 159 | If running the toolbox on a local Mac or Linux machine isn't working for you, you might want to try one of these other options: 160 | 161 | * John Moeller has put together a [Docker container for the toolbox](https://github.com/fishcorn/dvtb-container). This should even work on Windows! (confirmation needed) 162 | 163 | * If you're desperate, it's also possible to [run the toolbox on Amazon EC2](doc/deep-vis-on-aws.md), but display will be much slower and images can be loaded only from file (not from webcam). 164 | 165 | 166 | 167 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/__init__.py -------------------------------------------------------------------------------- /app_base.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | class BaseApp(object): 4 | '''Base App class.''' 5 | 6 | def __init__(self, settings, key_bindings): 7 | self.debug_level = 0 8 | 9 | def handle_input(self, input_image, panes): 10 | pass 11 | 12 | def handle_key(self, key, panes): 13 | '''Handle key and return either key (to let someone downstream handle it) or None (if this app handled it)''' 14 | pass 15 | 16 | def redraw_needed(self, key, panes): 17 | '''App should return whether or not its internal state has 18 | been updated (perhaps in response to handle_key, handle_input, 19 | or some internal processing finishing). 20 | ''' 21 | return False 22 | 23 | def draw(self, panes): 24 | '''Tells the app to draw in the given panes. Returns True if panes were changed and require a redraw, False if nothing was changed.''' 25 | return False 26 | 27 | def draw_help(self, panes): 28 | '''Tells the app to draw its help screen in the given pane. No return necessary.''' 29 | pass 30 | 31 | def start(self): 32 | '''Notify app to start, possibly creating any necessary threads''' 33 | pass 34 | 35 | def get_heartbeats(self): 36 | '''Returns a list of heartbeat functions, if any, that should be called regularly.''' 37 | return [] 38 | 39 | def set_debug(self, level): 40 | self.debug_level = level 41 | 42 | def quit(self): 43 | '''Notify app to quit, possibly joining any threads''' 44 | pass 45 | -------------------------------------------------------------------------------- /bindings.py: -------------------------------------------------------------------------------- 1 | # Define key bindings 2 | 3 | from keys import key_patterns 4 | 5 | class Bindings(object): 6 | def __init__(self, key_patterns): 7 | self._tag_to_key_labels = {} 8 | self._tag_to_help = {} 9 | self._key_label_to_tag = {} 10 | self._key_patterns = key_patterns 11 | self._cache_keycode_to_tag = {} 12 | 13 | def get_tag(self, keycode): 14 | '''Gets tag for keycode, returns None if no tag found.''' 15 | if keycode is None: 16 | return None 17 | if not keycode in self._cache_keycode_to_tag: 18 | label = self.get_key_label_from_keycode(keycode) 19 | self._cache_keycode_to_tag[keycode] = self.get_tag_from_key_label(label) 20 | return self._cache_keycode_to_tag[keycode] 21 | 22 | def get_tag_from_key_label(self, label): 23 | '''Get tag using key label, if no match, returns None.''' 24 | 25 | return self._key_label_to_tag.get(label, None) 26 | 27 | def get_key_label_from_keycode(self, keycode, extra_info = False): 28 | '''Get tag using keycode, if no match, returns None.''' 29 | 30 | label = None 31 | for mask in reversed(sorted(self._key_patterns.keys())): 32 | masked_keycode = keycode & mask 33 | if masked_keycode in self._key_patterns[mask]: 34 | label = self._key_patterns[mask][masked_keycode] 35 | break 36 | 37 | if extra_info: 38 | return label, [keycode & mask for mask in reversed(sorted(self._key_patterns.keys()))] 39 | else: 40 | return label 41 | 42 | def add(self, tag, key, help_text): 43 | self.add_multikey(tag, (key,), help_text) 44 | 45 | def add_multikey(self, tag, key_labels, help_text): 46 | for key_label in key_labels: 47 | assert key_label not in self._key_label_to_tag, ( 48 | 'Key "%s" cannot be bound to "%s" because it is already bound to "%s"' % 49 | (key_label, tag, self._key_label_to_tag[key_label]) 50 | ) 51 | self._key_label_to_tag[key_label] = tag 52 | self._tag_to_key_labels[tag] = key_labels 53 | self._tag_to_help[tag] = help_text 54 | 55 | def get_key_help(self, tag): 56 | return (self._tag_to_key_labels[tag], self._tag_to_help[tag]) 57 | 58 | _ = Bindings(key_patterns) 59 | 60 | # Core 61 | _.add('freeze_cam', 'f', 62 | 'Freeze or unfreeze camera capture') 63 | _.add('toggle_input_mode', 'c', 64 | 'Toggle between camera and static files') 65 | _.add_multikey('static_file_increment', ['e', 'pgdn'], 66 | 'Load next static file') 67 | _.add_multikey('static_file_decrement', ['w', 'pgup'], 68 | 'Load previous static file') 69 | _.add('help_mode', 'h', 70 | 'Toggle this help screen') 71 | _.add('stretch_mode', '0', 72 | 'Toggle between cropping and stretching static files to be square') 73 | _.add('debug_level', '5', 74 | 'Cycle debug level between 0 (quiet), 1 (some timing info) and 2 (all timing info)') 75 | _.add('quit', 'q', 76 | 'Quit') 77 | 78 | # Caffevis 79 | _.add_multikey('reset_state', ['esc'], 80 | 'Reset: turn off backprop, reset to layer 0, unit 0, default boost.') 81 | _.add_multikey('sel_left', ['left', 'j'], 82 | '') 83 | _.add_multikey('sel_right', ['right', 'l'], 84 | '') 85 | _.add_multikey('sel_down', ['down', 'k'], 86 | '') 87 | _.add_multikey('sel_up', ['up', 'i'], 88 | '') 89 | _.add('sel_left_fast', 'J', 90 | '') 91 | _.add('sel_right_fast', 'L', 92 | '') 93 | _.add('sel_down_fast', 'K', 94 | '') 95 | _.add('sel_up_fast', 'I', 96 | '') 97 | _.add_multikey('sel_layer_left', ['u', 'U'], 98 | 'Select previous layer without moving cursor') 99 | _.add_multikey('sel_layer_right', ['o', 'O'], 100 | 'Select next layer without moving cursor') 101 | 102 | _.add('zoom_mode', 'z', 103 | 'Cycle zooming through {currently selected unit, backprop results, none}') 104 | _.add('pattern_mode', 's', 105 | 'Toggle overlay of preferred input pattern (regularized optimized images)') 106 | 107 | _.add('ez_back_mode_loop', 'b', 108 | 'Cycle through a few common backprop/deconv modes') 109 | _.add('freeze_back_unit', 'd', 110 | 'Freeze the bprop/deconv origin to be the currently selected unit') 111 | _.add('show_back', 'a', 112 | 'Toggle between showing forward activations and back/deconv diffs') 113 | _.add('back_mode', 'n', 114 | '(expert) Change back mode directly.') 115 | _.add('back_filt_mode', 'm', 116 | '(expert) Change back output filter directly.') 117 | 118 | _.add('boost_gamma', 't', 119 | 'Boost contrast using gamma correction') 120 | _.add('boost_individual', 'T', 121 | 'Boost contrast by scaling each channel to use more of its individual range') 122 | _.add('toggle_label_predictions', '8', 123 | 'Turn on or off display of prob label values') 124 | _.add('toggle_unit_jpgs', '9', 125 | 'Turn on or off display of loaded jpg visualization') 126 | 127 | bindings = _ 128 | -------------------------------------------------------------------------------- /caffevis/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/caffevis/__init__.py -------------------------------------------------------------------------------- /caffevis/caffe_proc_thread.py: -------------------------------------------------------------------------------- 1 | import time 2 | import cv2 3 | import numpy as np 4 | 5 | from codependent_thread import CodependentThread 6 | from misc import WithTimer 7 | from caffevis_helper import net_preproc_forward 8 | 9 | 10 | 11 | class CaffeProcThread(CodependentThread): 12 | '''Runs Caffe in separate thread.''' 13 | 14 | def __init__(self, settings, net, state, loop_sleep, pause_after_keys, heartbeat_required, mode_gpu): 15 | CodependentThread.__init__(self, heartbeat_required) 16 | self.daemon = True 17 | self.net = net 18 | self.input_dims = self.net.blobs['data'].data.shape[2:4] # e.g. (227,227) 19 | self.state = state 20 | self.last_process_finished_at = None 21 | self.last_process_elapsed = None 22 | self.frames_processed_fwd = 0 23 | self.frames_processed_back = 0 24 | self.loop_sleep = loop_sleep 25 | self.pause_after_keys = pause_after_keys 26 | self.debug_level = 0 27 | self.mode_gpu = mode_gpu # Needed so the mode can be set again in the spawned thread, because there is a separate Caffe object per thread. 28 | 29 | self.settings = settings 30 | 31 | 32 | def run(self): 33 | print 'CaffeProcThread.run called' 34 | frame = None 35 | 36 | import caffe 37 | # Set the mode to CPU or GPU. Note: in the latest Caffe 38 | # versions, there is one Caffe object *per thread*, so the 39 | # mode must be set per thread! Here we set the mode for the 40 | # CaffeProcThread thread; it is also set in the main thread. 41 | if self.mode_gpu: 42 | caffe.set_mode_gpu() 43 | print 'CaffeVisApp mode (in CaffeProcThread): GPU' 44 | else: 45 | caffe.set_mode_cpu() 46 | print 'CaffeVisApp mode (in CaffeProcThread): CPU' 47 | 48 | while not self.is_timed_out(): 49 | with self.state.lock: 50 | if self.state.quit: 51 | #print 'CaffeProcThread.run: quit is True' 52 | #print self.state.quit 53 | break 54 | 55 | #print 'CaffeProcThread.run: caffe_net_state is:', self.state.caffe_net_state 56 | 57 | #print 'CaffeProcThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % ( 58 | # 'None' if self.state.next_frame is None else 'Avail', 59 | # self.state.caffe_net_state, 60 | # self.state.back_enabled) 61 | 62 | frame = None 63 | run_fwd = False 64 | run_back = False 65 | if self.state.caffe_net_state == 'free' and time.time() - self.state.last_key_at > self.pause_after_keys: 66 | frame = self.state.next_frame 67 | self.state.next_frame = None 68 | back_enabled = self.state.back_enabled 69 | back_mode = self.state.back_mode 70 | back_stale = self.state.back_stale 71 | #state_layer = self.state.layer 72 | #selected_unit = self.state.selected_unit 73 | backprop_layer = self.state.backprop_layer 74 | backprop_unit = self.state.backprop_unit 75 | 76 | # Forward should be run for every new frame 77 | run_fwd = (frame is not None) 78 | # Backward should be run if back_enabled and (there was a new frame OR back is stale (new backprop layer/unit selected)) 79 | run_back = (back_enabled and (run_fwd or back_stale)) 80 | self.state.caffe_net_state = 'proc' if (run_fwd or run_back) else 'free' 81 | 82 | #print 'run_fwd,run_back =', run_fwd, run_back 83 | 84 | if run_fwd: 85 | #print 'TIMING:, processing frame' 86 | self.frames_processed_fwd += 1 87 | 88 | if self.settings.static_files_input_mode == "siamese_image_list": 89 | frame1 = frame[0] 90 | frame2 = frame[1] 91 | 92 | im_small1 = cv2.resize(frame1, self.input_dims) 93 | im_small2 = cv2.resize(frame2, self.input_dims) 94 | im_small = np.concatenate( (im_small1, im_small2), axis=2) 95 | 96 | else: 97 | im_small = cv2.resize(frame, self.input_dims) 98 | 99 | with WithTimer('CaffeProcThread:forward', quiet = self.debug_level < 1): 100 | net_preproc_forward(self.settings, self.net, im_small, self.input_dims) 101 | 102 | if run_back: 103 | diffs = self.net.blobs[backprop_layer].diff * 0 104 | diffs[0][backprop_unit] = self.net.blobs[backprop_layer].data[0,backprop_unit] 105 | 106 | assert back_mode in ('grad', 'deconv') 107 | if back_mode == 'grad': 108 | with WithTimer('CaffeProcThread:backward', quiet = self.debug_level < 1): 109 | #print '**** Doing backprop with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max()) 110 | try: 111 | self.net.backward_from_layer(backprop_layer, diffs, zero_higher = True) 112 | except AttributeError: 113 | print 'ERROR: required bindings (backward_from_layer) not found! Try using the deconv-deep-vis-toolbox branch as described here: https://github.com/yosinski/deep-visualization-toolbox' 114 | raise 115 | else: 116 | with WithTimer('CaffeProcThread:deconv', quiet = self.debug_level < 1): 117 | #print '**** Doing deconv with %s diffs in [%s,%s]' % (backprop_layer, diffs.min(), diffs.max()) 118 | try: 119 | self.net.deconv_from_layer(backprop_layer, diffs, zero_higher = True) 120 | except AttributeError: 121 | print 'ERROR: required bindings (deconv_from_layer) not found! Try using the deconv-deep-vis-toolbox branch as described here: https://github.com/yosinski/deep-visualization-toolbox' 122 | raise 123 | 124 | with self.state.lock: 125 | self.state.back_stale = False 126 | 127 | if run_fwd or run_back: 128 | with self.state.lock: 129 | self.state.caffe_net_state = 'free' 130 | self.state.drawing_stale = True 131 | now = time.time() 132 | if self.last_process_finished_at: 133 | self.last_process_elapsed = now - self.last_process_finished_at 134 | self.last_process_finished_at = now 135 | else: 136 | time.sleep(self.loop_sleep) 137 | 138 | print 'CaffeProcThread.run: finished' 139 | print 'CaffeProcThread.run: processed %d frames fwd, %d frames back' % (self.frames_processed_fwd, self.frames_processed_back) 140 | 141 | def approx_fps(self): 142 | '''Get the approximate frames per second processed by this 143 | thread, considering only the last image processed. If more 144 | than two seconds ago, assume pipeline has stalled elsewhere 145 | (perhaps using static images that are only processed once). 146 | ''' 147 | if self.last_process_elapsed and (time.time() - self.last_process_finished_at) < 2.0: 148 | return 1.0 / (self.last_process_elapsed + 1e-6) 149 | else: 150 | return 0.0 151 | -------------------------------------------------------------------------------- /caffevis/caffevis_app_state.py: -------------------------------------------------------------------------------- 1 | import time 2 | from threading import Lock 3 | 4 | 5 | 6 | class CaffeVisAppState(object): 7 | '''State of CaffeVis app.''' 8 | 9 | def __init__(self, net, settings, bindings, net_layer_info): 10 | self.lock = Lock() # State is accessed in multiple threads 11 | self.settings = settings 12 | self.bindings = bindings 13 | self._layers = net.blobs.keys() 14 | self._layers = self._layers[1:] # chop off data layer 15 | if hasattr(self.settings, 'caffevis_filter_layers'): 16 | for name in self._layers: 17 | if self.settings.caffevis_filter_layers(name): 18 | print ' Layer filtered out by caffevis_filter_layers: %s' % name 19 | self._layers = filter(lambda name: not self.settings.caffevis_filter_layers(name), self._layers) 20 | self.net_layer_info = net_layer_info 21 | self.layer_boost_indiv_choices = self.settings.caffevis_boost_indiv_choices # 0-1, 0 is noop 22 | self.layer_boost_gamma_choices = self.settings.caffevis_boost_gamma_choices # 0-inf, 1 is noop 23 | self.caffe_net_state = 'free' # 'free', 'proc', or 'draw' 24 | self.extra_msg = '' 25 | self.back_stale = True # back becomes stale whenever the last back diffs were not computed using the current backprop unit and method (bprop or deconv) 26 | self.next_frame = None 27 | self.jpgvis_to_load_key = None 28 | self.last_key_at = 0 29 | self.quit = False 30 | 31 | self._reset_user_state() 32 | 33 | def _reset_user_state(self): 34 | self.layer_idx = 0 35 | self.layer = self._layers[0] 36 | self.layer_boost_indiv_idx = self.settings.caffevis_boost_indiv_default_idx 37 | self.layer_boost_indiv = self.layer_boost_indiv_choices[self.layer_boost_indiv_idx] 38 | self.layer_boost_gamma_idx = self.settings.caffevis_boost_gamma_default_idx 39 | self.layer_boost_gamma = self.layer_boost_gamma_choices[self.layer_boost_gamma_idx] 40 | self.cursor_area = 'top' # 'top' or 'bottom' 41 | self.selected_unit = 0 42 | # Which layer and unit (or channel) to use for backprop 43 | self.backprop_layer = self.layer 44 | self.backprop_unit = self.selected_unit 45 | self.backprop_selection_frozen = False # If false, backprop unit tracks selected unit 46 | self.back_enabled = False 47 | self.back_mode = 'grad' # 'grad' or 'deconv' 48 | self.back_filt_mode = 'raw' # 'raw', 'gray', 'norm', 'normblur' 49 | self.pattern_mode = False # Whether or not to show desired patterns instead of activations in layers pane 50 | self.layers_pane_zoom_mode = 0 # 0: off, 1: zoom selected (and show pref in small pane), 2: zoom backprop 51 | self.layers_show_back = False # False: show forward activations. True: show backward diffs 52 | self.show_label_predictions = self.settings.caffevis_init_show_label_predictions 53 | self.show_unit_jpgs = self.settings.caffevis_init_show_unit_jpgs 54 | self.drawing_stale = True 55 | kh,_ = self.bindings.get_key_help('help_mode') 56 | self.extra_msg = '%s for help' % kh[0] 57 | 58 | def handle_key(self, key): 59 | #print 'Ignoring key:', key 60 | if key == -1: 61 | return key 62 | 63 | with self.lock: 64 | key_handled = True 65 | self.last_key_at = time.time() 66 | tag = self.bindings.get_tag(key) 67 | if tag == 'reset_state': 68 | self._reset_user_state() 69 | elif tag == 'sel_layer_left': 70 | #hh,ww = self.tiles_height_width 71 | #self.selected_unit = self.selected_unit % ww # equivalent to scrolling all the way to the top row 72 | #self.cursor_area = 'top' # Then to the control pane 73 | self.layer_idx = max(0, self.layer_idx - 1) 74 | self.layer = self._layers[self.layer_idx] 75 | elif tag == 'sel_layer_right': 76 | #hh,ww = self.tiles_height_width 77 | #self.selected_unit = self.selected_unit % ww # equivalent to scrolling all the way to the top row 78 | #self.cursor_area = 'top' # Then to the control pane 79 | self.layer_idx = min(len(self._layers) - 1, self.layer_idx + 1) 80 | self.layer = self._layers[self.layer_idx] 81 | 82 | elif tag == 'sel_left': 83 | self.move_selection('left') 84 | elif tag == 'sel_right': 85 | self.move_selection('right') 86 | elif tag == 'sel_down': 87 | self.move_selection('down') 88 | elif tag == 'sel_up': 89 | self.move_selection('up') 90 | 91 | elif tag == 'sel_left_fast': 92 | self.move_selection('left', self.settings.caffevis_fast_move_dist) 93 | elif tag == 'sel_right_fast': 94 | self.move_selection('right', self.settings.caffevis_fast_move_dist) 95 | elif tag == 'sel_down_fast': 96 | self.move_selection('down', self.settings.caffevis_fast_move_dist) 97 | elif tag == 'sel_up_fast': 98 | self.move_selection('up', self.settings.caffevis_fast_move_dist) 99 | 100 | elif tag == 'boost_individual': 101 | self.layer_boost_indiv_idx = (self.layer_boost_indiv_idx + 1) % len(self.layer_boost_indiv_choices) 102 | self.layer_boost_indiv = self.layer_boost_indiv_choices[self.layer_boost_indiv_idx] 103 | elif tag == 'boost_gamma': 104 | self.layer_boost_gamma_idx = (self.layer_boost_gamma_idx + 1) % len(self.layer_boost_gamma_choices) 105 | self.layer_boost_gamma = self.layer_boost_gamma_choices[self.layer_boost_gamma_idx] 106 | elif tag == 'pattern_mode': 107 | self.pattern_mode = not self.pattern_mode 108 | if self.pattern_mode and not hasattr(self.settings, 'caffevis_unit_jpg_dir'): 109 | print 'Cannot switch to pattern mode; caffevis_unit_jpg_dir not defined in settings.py.' 110 | self.pattern_mode = False 111 | elif tag == 'show_back': 112 | # If in pattern mode: switch to fwd/back. Else toggle fwd/back mode 113 | if self.pattern_mode: 114 | self.pattern_mode = False 115 | else: 116 | self.layers_show_back = not self.layers_show_back 117 | if self.layers_show_back: 118 | if not self.back_enabled: 119 | self.back_enabled = True 120 | self.back_stale = True 121 | elif tag == 'back_mode': 122 | if not self.back_enabled: 123 | self.back_enabled = True 124 | self.back_mode = 'grad' 125 | self.back_stale = True 126 | else: 127 | if self.back_mode == 'grad': 128 | self.back_mode = 'deconv' 129 | self.back_stale = True 130 | else: 131 | self.back_enabled = False 132 | elif tag == 'back_filt_mode': 133 | if self.back_filt_mode == 'raw': 134 | self.back_filt_mode = 'gray' 135 | elif self.back_filt_mode == 'gray': 136 | self.back_filt_mode = 'norm' 137 | elif self.back_filt_mode == 'norm': 138 | self.back_filt_mode = 'normblur' 139 | else: 140 | self.back_filt_mode = 'raw' 141 | elif tag == 'ez_back_mode_loop': 142 | # Cycle: 143 | # off -> grad (raw) -> grad(gray) -> grad(norm) -> grad(normblur) -> deconv 144 | if not self.back_enabled: 145 | self.back_enabled = True 146 | self.back_mode = 'grad' 147 | self.back_filt_mode = 'raw' 148 | self.back_stale = True 149 | elif self.back_mode == 'grad' and self.back_filt_mode == 'raw': 150 | self.back_filt_mode = 'norm' 151 | elif self.back_mode == 'grad' and self.back_filt_mode == 'norm': 152 | self.back_mode = 'deconv' 153 | self.back_filt_mode = 'raw' 154 | self.back_stale = True 155 | else: 156 | self.back_enabled = False 157 | elif tag == 'freeze_back_unit': 158 | # Freeze selected layer/unit as backprop unit 159 | self.backprop_selection_frozen = not self.backprop_selection_frozen 160 | if self.backprop_selection_frozen: 161 | # Grap layer/selected_unit upon transition from non-frozen -> frozen 162 | self.backprop_layer = self.layer 163 | self.backprop_unit = self.selected_unit 164 | elif tag == 'zoom_mode': 165 | self.layers_pane_zoom_mode = (self.layers_pane_zoom_mode + 1) % 3 166 | if self.layers_pane_zoom_mode == 2 and not self.back_enabled: 167 | # Skip zoom into backprop pane when backprop is off 168 | self.layers_pane_zoom_mode = 0 169 | 170 | elif tag == 'toggle_label_predictions': 171 | self.show_label_predictions = not self.show_label_predictions 172 | 173 | elif tag == 'toggle_unit_jpgs': 174 | self.show_unit_jpgs = not self.show_unit_jpgs 175 | 176 | else: 177 | key_handled = False 178 | 179 | self._ensure_valid_selected() 180 | 181 | self.drawing_stale = key_handled # Request redraw any time we handled the key 182 | 183 | return (None if key_handled else key) 184 | 185 | def redraw_needed(self): 186 | with self.lock: 187 | return self.drawing_stale 188 | 189 | def move_selection(self, direction, dist = 1): 190 | if direction == 'left': 191 | if self.cursor_area == 'top': 192 | self.layer_idx = max(0, self.layer_idx - dist) 193 | self.layer = self._layers[self.layer_idx] 194 | else: 195 | self.selected_unit -= dist 196 | elif direction == 'right': 197 | if self.cursor_area == 'top': 198 | self.layer_idx = min(len(self._layers) - 1, self.layer_idx + dist) 199 | self.layer = self._layers[self.layer_idx] 200 | else: 201 | self.selected_unit += dist 202 | elif direction == 'down': 203 | if self.cursor_area == 'top': 204 | self.cursor_area = 'bottom' 205 | else: 206 | self.selected_unit += self.net_layer_info[self.layer]['tile_cols'] * dist 207 | elif direction == 'up': 208 | if self.cursor_area == 'top': 209 | pass 210 | else: 211 | self.selected_unit -= self.net_layer_info[self.layer]['tile_cols'] * dist 212 | if self.selected_unit < 0: 213 | self.selected_unit += self.net_layer_info[self.layer]['tile_cols'] 214 | self.cursor_area = 'top' 215 | 216 | def _ensure_valid_selected(self): 217 | n_tiles = self.net_layer_info[self.layer]['n_tiles'] 218 | 219 | # Forward selection 220 | self.selected_unit = max(0, self.selected_unit) 221 | self.selected_unit = min(n_tiles-1, self.selected_unit) 222 | 223 | # Backward selection 224 | if not self.backprop_selection_frozen: 225 | # If backprop_selection is not frozen, backprop layer/unit follows selected unit 226 | if not (self.backprop_layer == self.layer and self.backprop_unit == self.selected_unit): 227 | self.backprop_layer = self.layer 228 | self.backprop_unit = self.selected_unit 229 | self.back_stale = True # If there is any change, back diffs are now stale 230 | -------------------------------------------------------------------------------- /caffevis/caffevis_helper.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from image_misc import get_tiles_height_width, caffe_load_image 4 | 5 | 6 | 7 | def net_preproc_forward(settings, net, img, data_hw): 8 | 9 | if settings.static_files_input_mode == "siamese_image_list": 10 | appropriate_shape = data_hw + (6,) 11 | else: 12 | appropriate_shape = data_hw + (3,) 13 | 14 | assert img.shape == appropriate_shape, 'img is wrong size (got %s but expected %s)' % (img.shape, appropriate_shape) 15 | #resized = caffe.io.resize_image(img, net.image_dims) # e.g. (227, 227, 3) 16 | 17 | data_blob = net.transformer.preprocess('data', img) # e.g. (3, 227, 227), mean subtracted and scaled to [0,255] 18 | data_blob = data_blob[np.newaxis,:,:,:] # e.g. (1, 3, 227, 227) 19 | output = net.forward(data=data_blob) 20 | return output 21 | 22 | 23 | def get_pretty_layer_name(settings, layer_name): 24 | has_old_settings = hasattr(settings, 'caffevis_layer_pretty_names') 25 | has_new_settings = hasattr(settings, 'caffevis_layer_pretty_name_fn') 26 | if has_old_settings and not has_new_settings: 27 | print ('WARNING: Your settings.py and/or settings_local.py are out of date.' 28 | 'caffevis_layer_pretty_names has been replaced with caffevis_layer_pretty_name_fn.' 29 | 'Update your settings.py and/or settings_local.py (see documentation in' 30 | 'setttings.py) to remove this warning.') 31 | return settings.caffevis_layer_pretty_names.get(layer_name, layer_name) 32 | 33 | ret = layer_name 34 | if hasattr(settings, 'caffevis_layer_pretty_name_fn'): 35 | ret = settings.caffevis_layer_pretty_name_fn(ret) 36 | if ret != layer_name: 37 | print ' Prettified layer name: "%s" -> "%s"' % (layer_name, ret) 38 | return ret 39 | 40 | 41 | def read_label_file(filename): 42 | ret = [] 43 | with open(filename, 'r') as ff: 44 | for line in ff: 45 | label = line.strip() 46 | if len(label) > 0: 47 | ret.append(label) 48 | return ret 49 | 50 | 51 | def crop_to_corner(img, corner, small_padding = 1, large_padding = 2): 52 | '''Given an large image consisting of 3x3 squares with small_padding padding concatenated into a 2x2 grid with large_padding padding, return one of the four corners (0, 1, 2, 3)''' 53 | assert corner in (0,1,2,3), 'specify corner 0, 1, 2, or 3' 54 | assert img.shape[0] == img.shape[1], 'img is not square' 55 | assert img.shape[0] % 2 == 0, 'even number of pixels assumption violated' 56 | half_size = img.shape[0]/2 57 | big_ii = 0 if corner in (0,1) else 1 58 | big_jj = 0 if corner in (0,2) else 1 59 | tp = small_padding + large_padding 60 | #tp = 0 61 | return img[big_ii*half_size+tp:(big_ii+1)*half_size-tp, 62 | big_jj*half_size+tp:(big_jj+1)*half_size-tp] 63 | 64 | 65 | def load_sprite_image(img_path, rows_cols, n_sprites = None): 66 | '''Load a 2D (3D with color channels) sprite image where 67 | (rows,cols) = rows_cols, slices, and returns as a 3D tensor (4D 68 | with color channels). Sprite shape is computed automatically. If 69 | n_sprites is not given, it is assumed to be rows*cols. Return as 70 | 3D tensor with shape (n_sprites, sprite_height, sprite_width, 71 | sprite_channels). 72 | ''' 73 | 74 | rows,cols = rows_cols 75 | if n_sprites is None: 76 | n_sprites = rows * cols 77 | img = caffe_load_image(img_path, color = True, as_uint = True) 78 | assert img.shape[0] % rows == 0, 'sprite image has shape %s which is not divisible by rows_cols %' % (img.shape, rows_cols) 79 | assert img.shape[1] % cols == 0, 'sprite image has shape %s which is not divisible by rows_cols %' % (img.shape, rows_cols) 80 | sprite_height = img.shape[0] / rows 81 | sprite_width = img.shape[1] / cols 82 | sprite_channels = img.shape[2] 83 | 84 | ret = np.zeros((n_sprites, sprite_height, sprite_width, sprite_channels), dtype = img.dtype) 85 | for idx in xrange(n_sprites): 86 | # Row-major order 87 | ii = idx / cols 88 | jj = idx % cols 89 | ret[idx] = img[ii*sprite_height:(ii+1)*sprite_height, 90 | jj*sprite_width:(jj+1)*sprite_width, :] 91 | return ret 92 | 93 | 94 | def load_square_sprite_image(img_path, n_sprites): 95 | ''' 96 | Just like load_sprite_image but assumes tiled image is square 97 | ''' 98 | 99 | tile_rows,tile_cols = get_tiles_height_width(n_sprites) 100 | return load_sprite_image(img_path, (tile_rows, tile_cols), n_sprites = n_sprites) 101 | 102 | 103 | def check_force_backward_true(prototxt_file): 104 | '''Checks whether the given file contains a line with the following text, ignoring whitespace: 105 | force_backward: true 106 | ''' 107 | 108 | found = False 109 | with open(prototxt_file, 'r') as ff: 110 | for line in ff: 111 | fields = line.strip().split() 112 | if len(fields) == 2 and fields[0] == 'force_backward:' and fields[1] == 'true': 113 | found = True 114 | break 115 | 116 | if not found: 117 | print '\n\nWARNING: the specified prototxt' 118 | print '"%s"' % prototxt_file 119 | print 'does not contain the line "force_backward: true". This may result in backprop' 120 | print 'and deconv producing all zeros at the input layer. You may want to add this line' 121 | print 'to your prototxt file before continuing to force backprop to compute derivatives' 122 | print 'at the data layer as well.\n\n' 123 | 124 | -------------------------------------------------------------------------------- /caffevis/jpg_vis_loading_thread.py: -------------------------------------------------------------------------------- 1 | import os 2 | import time 3 | import numpy as np 4 | 5 | from codependent_thread import CodependentThread 6 | from image_misc import caffe_load_image, ensure_uint255_and_resize_to_fit 7 | from caffevis_helper import crop_to_corner 8 | 9 | 10 | 11 | class JPGVisLoadingThread(CodependentThread): 12 | '''Loads JPGs necessary for caffevis_jpgvis pane in separate 13 | thread and inserts them into the cache. 14 | ''' 15 | 16 | def __init__(self, settings, state, cache, loop_sleep, heartbeat_required): 17 | CodependentThread.__init__(self, heartbeat_required) 18 | self.daemon = True 19 | self.settings = settings 20 | self.state = state 21 | self.cache = cache 22 | self.loop_sleep = loop_sleep 23 | self.debug_level = 0 24 | 25 | def run(self): 26 | print 'JPGVisLoadingThread.run called' 27 | 28 | while not self.is_timed_out(): 29 | with self.state.lock: 30 | if self.state.quit: 31 | break 32 | 33 | #print 'JPGVisLoadingThread.run: caffe_net_state is:', self.state.caffe_net_state 34 | #print 'JPGVisLoadingThread.run loop: next_frame: %s, caffe_net_state: %s, back_enabled: %s' % ( 35 | # 'None' if self.state.next_frame is None else 'Avail', 36 | # self.state.caffe_net_state, 37 | # self.state.back_enabled) 38 | 39 | jpgvis_to_load_key = self.state.jpgvis_to_load_key 40 | 41 | if jpgvis_to_load_key is None: 42 | time.sleep(self.loop_sleep) 43 | continue 44 | 45 | state_layer, state_selected_unit, data_shape = jpgvis_to_load_key 46 | 47 | # Load three images: 48 | images = [None] * 3 49 | 50 | # Resize each component images only using one direction as 51 | # a constraint. This is straightforward but could be very 52 | # wasteful (making an image much larger then much smaller) 53 | # if the proportions of the stacked image are very 54 | # different from the proportions of the data pane. 55 | #resize_shape = (None, data_shape[1]) if self.settings.caffevis_jpgvis_stack_vert else (data_shape[0], None) 56 | # As a heuristic, instead just assume the three images are of the same shape. 57 | if self.settings.caffevis_jpgvis_stack_vert: 58 | resize_shape = (data_shape[0]/3, data_shape[1]) 59 | else: 60 | resize_shape = (data_shape[0], data_shape[1]/3) 61 | 62 | # 0. e.g. regularized_opt/conv1/conv1_0037_montage.jpg 63 | jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir, 64 | 'regularized_opt', 65 | state_layer, 66 | '%s_%04d_montage.jpg' % (state_layer, state_selected_unit)) 67 | try: 68 | img = caffe_load_image(jpg_path, color = True) 69 | img_corner = crop_to_corner(img, 2) 70 | images[0] = ensure_uint255_and_resize_to_fit(img_corner, resize_shape) 71 | except IOError: 72 | print '\nAttempted to load file %s but failed. To supress this warning, remove layer "%s" from settings.caffevis_jpgvis_layers' % (jpg_path, state_layer) 73 | pass 74 | 75 | # 1. e.g. max_im/conv1/conv1_0037.jpg 76 | jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir, 77 | 'max_im', 78 | state_layer, 79 | '%s_%04d.jpg' % (state_layer, state_selected_unit)) 80 | try: 81 | img = caffe_load_image(jpg_path, color = True) 82 | images[1] = ensure_uint255_and_resize_to_fit(img, resize_shape) 83 | except IOError: 84 | pass 85 | 86 | # 2. e.g. max_deconv/conv1/conv1_0037.jpg 87 | try: 88 | jpg_path = os.path.join(self.settings.caffevis_unit_jpg_dir, 89 | 'max_deconv', 90 | state_layer, 91 | '%s_%04d.jpg' % (state_layer, state_selected_unit)) 92 | img = caffe_load_image(jpg_path, color = True) 93 | images[2] = ensure_uint255_and_resize_to_fit(img, resize_shape) 94 | except IOError: 95 | pass 96 | 97 | # Prune images that were not found: 98 | images = [im for im in images if im is not None] 99 | 100 | # Stack together 101 | if len(images) > 0: 102 | #print 'Stacking:', [im.shape for im in images] 103 | stack_axis = 0 if self.settings.caffevis_jpgvis_stack_vert else 1 104 | img_stacked = np.concatenate(images, axis = stack_axis) 105 | #print 'Stacked:', img_stacked.shape 106 | img_resize = ensure_uint255_and_resize_to_fit(img_stacked, data_shape) 107 | #print 'Resized:', img_resize.shape 108 | else: 109 | img_resize = np.zeros(shape=(0,)) # Sentinal value when image is not found. 110 | 111 | self.cache.set(jpgvis_to_load_key, img_resize) 112 | 113 | with self.state.lock: 114 | self.state.jpgvis_to_load_key = None 115 | self.state.drawing_stale = True 116 | 117 | print 'JPGVisLoadingThread.run: finished' 118 | -------------------------------------------------------------------------------- /codependent_thread.py: -------------------------------------------------------------------------------- 1 | import time 2 | from threading import Lock, Thread 3 | 4 | 5 | class CodependentThread(Thread): 6 | '''A Thread that must be occasionally poked to stay alive. ''' 7 | 8 | def __init__(self, heartbeat_timeout = 1.0): 9 | Thread.__init__(self) 10 | self.heartbeat_timeout = heartbeat_timeout 11 | self.heartbeat_lock = Lock() 12 | self.heartbeat() 13 | 14 | def heartbeat(self): 15 | with self.heartbeat_lock: 16 | self.last_beat = time.time() 17 | 18 | def is_timed_out(self): 19 | with self.heartbeat_lock: 20 | now = time.time() 21 | if now - self.last_beat > self.heartbeat_timeout: 22 | print '%s instance %s timed out after %s seconds (%s - %s = %s)' % (self.__class__.__name__, self, self.heartbeat_timeout, now, self.last_beat, now - self.last_beat) 23 | return True 24 | else: 25 | return False 26 | -------------------------------------------------------------------------------- /doc/computing_per_unit_visualizations.md: -------------------------------------------------------------------------------- 1 | # Computing per-unit visualizations for a new network 2 | 3 | Per-unit visualizations are included for the caffenet-yos network but not for other networks (the total size is at least several GB, which becomes cumbersome to distribute). 4 | 5 | But the per-unit visualizations can be computed for any network: 6 | 7 | * To find synthetic images that cause high activation via regularized optimization, use the [optimize_image.py](/optimize_image.py) script. Script usage is [explained here](/doc/running-optimize-image.md). 8 | 9 | * To find images (for FC layers) or crops (for conv layers) from a set of images (e.g. the ImageNet training or validation set) that cause highest activation, use the [find_max_acts.py](/find_maxes/find_max_acts.py) script to go through the set of images and note the top K images/crops and then [crop_max_patches.py](/find_maxes/crop_max_patches.py) to use the noted max images / max locations to output the crops and/or deconv of the crops. 10 | 11 | Results of both of the above steps will be saved as per-unit jpg image files, which can be loaded by the toolbox when browsing units. To do so, just point the `caffevis_unit_jpg_dir` setting to the directory containing the per-unit images. 12 | -------------------------------------------------------------------------------- /doc/deep-vis-on-aws.md: -------------------------------------------------------------------------------- 1 | # Using the deep-vis toolbox on an Amazon AWS EC2 instance 2 | 3 | Building Caffe from scrach on your local machine can be a pain. These steps will help get the deep-vis toolbox set up and displaying on an AWS EC2 instance built from the Caffe/CUDA/CuDNN AMI. Because the EC2 instance is a remote server, the deep-vis toolbox will not be able to display to your desktop directly-- these steps allow you to forward the remote desktop to your local screen using a VNC viewer. 4 | 5 | ## Setting up the AMI and VNC viewer 6 | 7 | 1. First off, grab the AMI that the folks at Caffe built (ami-763a311e) and get it up and running on a g2.2xlarge or g2.8xlarge instance. Edit your inbound security permissions to allow all traffic from your IP address. 8 | 9 | 2. Clone this repo to your AMI and follow the basic instructions in the README to get the deep vis toolbox set up with the necessary dependencies. 10 | 11 | 3. You'll need to set up a new user on the instance in order to forward the display to your local computer. Follow the instructions [here](http://www.brianlinkletter.com/how-to-set-up-a-new-userid-on-your-amazon-aws-server-instance/). Next, you'll need to set up the VNC server, for which there are good instructions [here](http://www.brianlinkletter.com/how-to-run-gui-applications-xfce-on-an-amazon-aws-cloud-server-instance/). If you're on a Mac, I'd recommend using [Chicken of the VNC](http://sourceforge.net/projects/cotvnc/). 12 | 13 | 4. After you've logged in using SSH port forwarding, start the vncserver, and note what number desktop it is located at, ie. 'ip-172-31-48-75:**1**'. In Chicken of the VNC, open a new connection. The host is your instance's public DNS, the display is the # specified by VNC, and the password is the one you set earlier. 14 | 15 | 5. Once you're logged in, you'll be able to move around the terminal on the remote desktop to deep-visualization-toolbox and run `run_toolbox.py` without any display issues! 16 | -------------------------------------------------------------------------------- /doc/example_bvlc-googlenet_bus.300.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_bvlc-googlenet_bus.300.jpg -------------------------------------------------------------------------------- /doc/example_bvlc-googlenet_bus.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_bvlc-googlenet_bus.jpg -------------------------------------------------------------------------------- /doc/example_caffenet-yos_bus_wheel_unit.300.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_caffenet-yos_bus_wheel_unit.300.jpg -------------------------------------------------------------------------------- /doc/example_caffenet-yos_bus_wheel_unit.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_caffenet-yos_bus_wheel_unit.jpg -------------------------------------------------------------------------------- /doc/example_caffenet-yos_webcam.300.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_caffenet-yos_webcam.300.jpg -------------------------------------------------------------------------------- /doc/example_caffenet-yos_webcam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/example_caffenet-yos_webcam.jpg -------------------------------------------------------------------------------- /doc/opt_fc8_0130_0_best_X.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/opt_fc8_0130_0_best_X.jpg -------------------------------------------------------------------------------- /doc/opt_fc8_0130_0_best_Xpm.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/doc/opt_fc8_0130_0_best_Xpm.jpg -------------------------------------------------------------------------------- /find_maxes/caffe_misc.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import skimage.io 4 | import numpy as np 5 | 6 | 7 | 8 | def shownet(net): 9 | '''Print some stats about a net and its activations''' 10 | 11 | print '%-41s%-31s%s' % ('', 'acts', 'act diffs') 12 | print '%-45s%-31s%s' % ('', 'params', 'param diffs') 13 | for k, v in net.blobs.items(): 14 | if k in net.params: 15 | params = net.params[k] 16 | for pp, blob in enumerate(params): 17 | if pp == 0: 18 | print ' ', 'P: %-5s'%k, 19 | else: 20 | print ' ' * 11, 21 | print '%-32s' % repr(blob.data.shape), 22 | print '%-30s' % ('(%g, %g)' % (blob.data.min(), blob.data.max())), 23 | print '(%g, %g)' % (blob.diff.min(), blob.diff.max()) 24 | print '%-5s'%k, '%-34s' % repr(v.data.shape), 25 | print '%-30s' % ('(%g, %g)' % (v.data.min(), v.data.max())), 26 | print '(%g, %g)' % (v.diff.min(), v.diff.max()) 27 | 28 | 29 | 30 | def region_converter(top_slice, bot_size, top_size, filter_width = (1,1), stride = (1,1), pad = (0,0)): 31 | ''' 32 | Works for conv or pool 33 | 34 | vector ConvolutionLayer::JBY_region_of_influence(const vector& slice) { 35 | + CHECK_EQ(slice.size(), 4) << "slice must have length 4 (ii_start, ii_end, jj_start, jj_end)"; 36 | + // Crop region to output size 37 | + vector sl = vector(slice); 38 | + sl[0] = max(0, min(height_out_, slice[0])); 39 | + sl[1] = max(0, min(height_out_, slice[1])); 40 | + sl[2] = max(0, min(width_out_, slice[2])); 41 | + sl[3] = max(0, min(width_out_, slice[3])); 42 | + vector roi; 43 | + roi.resize(4); 44 | + roi[0] = sl[0] * stride_h_ - pad_h_; 45 | + roi[1] = (sl[1]-1) * stride_h_ + kernel_h_ - pad_h_; 46 | + roi[2] = sl[2] * stride_w_ - pad_w_; 47 | + roi[3] = (sl[3]-1) * stride_w_ + kernel_w_ - pad_w_; 48 | + return roi; 49 | +} 50 | ''' 51 | assert len(top_slice) == 4 52 | assert len(bot_size) == 2 53 | assert len(top_size) == 2 54 | assert len(filter_width) == 2 55 | assert len(stride) == 2 56 | assert len(pad) == 2 57 | 58 | # Crop top slice to allowable region 59 | top_slice = [ss for ss in top_slice] # Copy list or array -> list 60 | 61 | top_slice[0] = max(0, min(top_size[0], top_slice[0])) 62 | top_slice[1] = max(0, min(top_size[0], top_slice[1])) 63 | top_slice[2] = max(0, min(top_size[1], top_slice[2])) 64 | top_slice[3] = max(0, min(top_size[1], top_slice[3])) 65 | 66 | bot_slice = [-123] * 4 67 | 68 | bot_slice[0] = top_slice[0] * stride[0] - pad[0]; 69 | bot_slice[1] = (top_slice[1]-1) * stride[0] + filter_width[0] - pad[0]; 70 | bot_slice[2] = top_slice[2] * stride[1] - pad[1]; 71 | bot_slice[3] = (top_slice[3]-1) * stride[1] + filter_width[1] - pad[1]; 72 | 73 | return bot_slice 74 | 75 | def get_conv_converter(bot_size, top_size, filter_width = (1,1), stride = (1,1), pad = (0,0)): 76 | return lambda top_slice : region_converter(top_slice, bot_size, top_size, filter_width, stride, pad) 77 | 78 | def get_pool_converter(bot_size, top_size, filter_width = (1,1), stride = (1,1), pad = (0,0)): 79 | return lambda top_slice : region_converter(top_slice, bot_size, top_size, filter_width, stride, pad) 80 | 81 | 82 | 83 | class RegionComputer(object): 84 | '''Computes regions of possible influcence from higher layers to lower layers. 85 | 86 | Woefully hardcoded''' 87 | 88 | def __init__(self): 89 | #self.net = net 90 | _tmp = [] 91 | _tmp.append(('data', None)) 92 | _tmp.append(('conv1', get_conv_converter((227,227), (55,55), (11,11), (4,4)))) 93 | _tmp.append(('pool1', get_pool_converter((55,55), (27,27), (3,3), (2,2)))) 94 | _tmp.append(('conv2', get_conv_converter((27,27), (27,27), (5,5), (1,1), (2,2)))) 95 | _tmp.append(('pool2', get_pool_converter((27,27), (13,13), (3,3), (2,2)))) 96 | _tmp.append(('conv3', get_conv_converter((13,13), (13,13), (3,3), (1,1), (1,1)))) 97 | _tmp.append(('conv4', get_conv_converter((13,13), (13,13), (3,3), (1,1), (1,1)))) 98 | _tmp.append(('conv5', get_conv_converter((13,13), (13,13), (3,3), (1,1), (1,1)))) 99 | self.names = [tt[0] for tt in _tmp] 100 | self.converters = [tt[1] for tt in _tmp] 101 | 102 | def convert_region(self, from_layer, to_layer, region, verbose = False): 103 | '''region is the slice of the from_layer in the following Python 104 | index format: (ii_start, ii_end, jj_start, jj_end) 105 | ''' 106 | 107 | from_idx = self.names.index(from_layer) 108 | to_idx = self.names.index(to_layer) 109 | assert from_idx >= to_idx, 'wrong order of from_layer and to_layer' 110 | 111 | ret = region 112 | for ii in range(from_idx, to_idx, -1): 113 | converter = self.converters[ii] 114 | if verbose: 115 | print 'pushing', self.names[ii], 'region', ret, 'through converter' 116 | ret = converter(ret) 117 | if verbose: 118 | print 'Final region at ', self.names[to_idx], 'is', ret 119 | 120 | return ret 121 | 122 | 123 | 124 | def norm01c(arr, center): 125 | '''Maps the input range to [0,1] such that the center value maps to .5''' 126 | arr = arr.copy() 127 | arr -= center 128 | arr /= max(2 * arr.max(), -2 * arr.min()) + 1e-10 129 | arr += .5 130 | assert arr.min() >= 0 131 | assert arr.max() <= 1 132 | return arr 133 | 134 | 135 | 136 | def save_caffe_image(img, filename, autoscale = True, autoscale_center = None): 137 | '''Takes an image in caffe format (01) or (c01, BGR) and saves it to a file''' 138 | if len(img.shape) == 2: 139 | # upsample grayscale 01 -> 01c 140 | img = np.tile(img[:,:,np.newaxis], (1,1,3)) 141 | else: 142 | img = img[::-1].transpose((1,2,0)) 143 | if autoscale_center is not None: 144 | img = norm01c(img, autoscale_center) 145 | elif autoscale: 146 | img = img.copy() 147 | img -= img.min() 148 | img *= 1.0 / (img.max() + 1e-10) 149 | skimage.io.imsave(filename, img) 150 | -------------------------------------------------------------------------------- /find_maxes/crop_max_patches.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import argparse 4 | import ipdb as pdb 5 | import cPickle as pickle 6 | 7 | from loaders import load_imagenet_mean, load_labels, caffe 8 | from jby_misc import WithTimer 9 | from max_tracker import output_max_patches 10 | 11 | 12 | 13 | def main(): 14 | parser = argparse.ArgumentParser(description='Loads a pickled NetMaxTracker and outputs one or more of {the patches of the image, a deconv patch, a backprop patch} associated with the maxes.') 15 | parser.add_argument('--N', type = int, default = 9, help = 'Note and save top N activations.') 16 | parser.add_argument('--gpu', action = 'store_true', help = 'Use gpu.') 17 | parser.add_argument('--do-maxes', action = 'store_true', help = 'Output max patches.') 18 | parser.add_argument('--do-deconv', action = 'store_true', help = 'Output deconv patches.') 19 | parser.add_argument('--do-deconv-norm', action = 'store_true', help = 'Output deconv-norm patches.') 20 | parser.add_argument('--do-backprop', action = 'store_true', help = 'Output backprop patches.') 21 | parser.add_argument('--do-backprop-norm', action = 'store_true', help = 'Output backprop-norm patches.') 22 | parser.add_argument('--do-info', action = 'store_true', help = 'Output info file containing max filenames and labels.') 23 | parser.add_argument('--idx-begin', type = int, default = None, help = 'Start at this unit (default: all units).') 24 | parser.add_argument('--idx-end', type = int, default = None, help = 'End at this unit (default: all units).') 25 | 26 | parser.add_argument('nmt_pkl', type = str, help = 'Which pickled NetMaxTracker to load.') 27 | parser.add_argument('net_prototxt', type = str, help = 'Network prototxt to load') 28 | parser.add_argument('net_weights', type = str, help = 'Network weights to load') 29 | parser.add_argument('datadir', type = str, help = 'Directory to look for files in') 30 | parser.add_argument('filelist', type = str, help = 'List of image files to consider, one per line. Must be the same filelist used to produce the NetMaxTracker!') 31 | parser.add_argument('outdir', type = str, help = r'Which output directory to use. Files are output into outdir/layer/unit_%%04d/{maxes,deconv,backprop}_%%03d.png') 32 | parser.add_argument('layer', type = str, help = 'Which layer to output') 33 | #parser.add_argument('--mean', type = str, default = '', help = 'data mean to load') 34 | args = parser.parse_args() 35 | 36 | if args.gpu: 37 | caffe.set_mode_gpu() 38 | else: 39 | caffe.set_mode_cpu() 40 | 41 | imagenet_mean = load_imagenet_mean() 42 | net = caffe.Classifier(args.net_prototxt, args.net_weights, 43 | mean=imagenet_mean, 44 | channel_swap=(2,1,0), 45 | raw_scale=255, 46 | image_dims=(256, 256)) 47 | 48 | assert args.do_maxes or args.do_deconv or args.do_deconv_norm or args.do_backprop or args.do_backprop_norm or args.do_info, 'Specify at least one do_* option to output.' 49 | 50 | with open(args.nmt_pkl, 'rb') as ff: 51 | nmt = pickle.load(ff) 52 | mt = nmt.max_trackers[args.layer] 53 | 54 | if args.idx_begin is None: 55 | args.idx_begin = 0 56 | if args.idx_end is None: 57 | args.idx_end = mt.max_vals.shape[0] 58 | 59 | with WithTimer('Saved %d images per unit for %s units %d:%d.' % (args.N, args.layer, args.idx_begin, args.idx_end)): 60 | output_max_patches(mt, net, args.layer, args.idx_begin, args.idx_end, 61 | args.N, args.datadir, args.filelist, args.outdir, 62 | (args.do_maxes, args.do_deconv, args.do_deconv_norm, args.do_backprop, args.do_backprop_norm, args.do_info)) 63 | 64 | 65 | 66 | if __name__ == '__main__': 67 | main() 68 | -------------------------------------------------------------------------------- /find_maxes/find_max_acts.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import argparse 4 | import ipdb as pdb 5 | import cPickle as pickle 6 | 7 | from loaders import load_imagenet_mean, load_labels, caffe 8 | from jby_misc import WithTimer 9 | from max_tracker import scan_images_for_maxes 10 | 11 | 12 | 13 | def main(): 14 | parser = argparse.ArgumentParser(description='Finds images in a training set that cause max activation for a network; saves results in a pickled NetMaxTracker.') 15 | parser.add_argument('--N', type = int, default = 9, help = 'note and save top N activations') 16 | parser.add_argument('--gpu', action = 'store_true', help = 'use gpu') 17 | parser.add_argument('net_prototxt', type = str, default = '', help = 'network prototxt to load') 18 | parser.add_argument('net_weights', type = str, default = '', help = 'network weights to load') 19 | parser.add_argument('datadir', type = str, default = '.', help = 'directory to look for files in') 20 | parser.add_argument('filelist', type = str, help = 'list of image files to consider, one per line') 21 | parser.add_argument('outfile', type = str, help = 'output filename for pkl') 22 | #parser.add_argument('--mean', type = str, default = '', help = 'data mean to load') 23 | args = parser.parse_args() 24 | 25 | imagenet_mean = load_imagenet_mean() 26 | net = caffe.Classifier(args.net_prototxt, args.net_weights, 27 | mean=imagenet_mean, 28 | channel_swap=(2,1,0), 29 | raw_scale=255, 30 | image_dims=(256, 256)) 31 | 32 | if args.gpu: 33 | caffe.set_mode_gpu() 34 | else: 35 | caffe.set_mode_cpu() 36 | 37 | with WithTimer('Scanning images'): 38 | max_tracker = scan_images_for_maxes(net, args.datadir, args.filelist, args.N) 39 | with WithTimer('Saving maxes'): 40 | with open(args.outfile, 'wb') as ff: 41 | pickle.dump(max_tracker, ff, -1) 42 | 43 | 44 | 45 | if __name__ == '__main__': 46 | main() 47 | -------------------------------------------------------------------------------- /find_maxes/jby_misc.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | 4 | import time 5 | 6 | 7 | class WithTimer: 8 | def __init__(self, title = '', quiet = False): 9 | self.title = title 10 | self.quiet = quiet 11 | 12 | def elapsed(self): 13 | return time.time() - self.wall, time.clock() - self.proc 14 | 15 | def enter(self): 16 | '''Manually trigger enter''' 17 | self.__enter__() 18 | 19 | def __enter__(self): 20 | self.proc = time.clock() 21 | self.wall = time.time() 22 | return self 23 | 24 | def __exit__(self, *args): 25 | if not self.quiet: 26 | titlestr = self.title + ' ' if self.title else '' 27 | print 'Elapsed %sreal: %.06f, sys: %.06f' % ((titlestr,) + self.elapsed()) 28 | 29 | def misc_main(): 30 | print 'Running quick demo' 31 | 32 | print ' -> Before with statement' 33 | with WithTimer('sleepy time'): 34 | print ' -> About to sleep' 35 | time.sleep(.5) 36 | print ' -> Done sleeping' 37 | print ' -> After with statement' 38 | 39 | 40 | if __name__ == '__main__': 41 | misc_main() 42 | -------------------------------------------------------------------------------- /find_maxes/loaders.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | from pylab import * 4 | 5 | # Make sure that caffe is on the python path: 6 | caffe_root = '../../' # this file is expected to be in {caffe_root}/experiments/something 7 | import sys 8 | loadpath = caffe_root + 'python_cpu' 9 | print '= = = CAFFE LOADER: LOADING CPU VERSION from path: %s = = =' % loadpath 10 | sys.path.insert(0, loadpath) # Use CPU compiled code for backprop vis/etc 11 | import caffe 12 | caffe.set_mode_cpu() 13 | 14 | 15 | 16 | def load_labels(): 17 | with open('%s/data/ilsvrc12/synset_words.txt' % caffe_root) as ff: 18 | labels = [line.strip() for line in ff.readlines()] 19 | return labels 20 | 21 | 22 | 23 | def load_trained_net(model_prototxt = None, model_weights = None): 24 | assert (model_prototxt is None) == (model_weights is None), 'Specify both model_prototxt and model_weights or neither' 25 | if model_prototxt is None: 26 | load_dir = '/home/jyosinsk/results/140311_234854_afadfd3_priv_netbase_upgraded/' 27 | model_prototxt = load_dir + 'deploy_1.prototxt' 28 | model_weights = load_dir + 'caffe_imagenet_train_iter_450000' 29 | 30 | print 'LOADER: loading net:' 31 | print ' ', model_prototxt 32 | print ' ', model_weights 33 | net = caffe.Classifier(model_prototxt, model_weights) 34 | #net.set_phase_test() 35 | 36 | return net 37 | 38 | 39 | 40 | def load_imagenet_mean(): 41 | imagenet_mean = np.load(caffe_root + 'python/caffe/imagenet/ilsvrc_2012_mean.npy') 42 | imagenet_mean = imagenet_mean[:, 14:14+227, 14:14+227] # (3,256,256) -> (3,227,227) Crop to center 227x227 section 43 | return imagenet_mean 44 | -------------------------------------------------------------------------------- /find_maxes/max_tracker.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import ipdb as pdb 5 | import errno 6 | from datetime import datetime 7 | 8 | #import caffe 9 | from loaders import load_imagenet_mean, load_labels, caffe 10 | from jby_misc import WithTimer 11 | from caffe_misc import shownet, RegionComputer, save_caffe_image 12 | import numpy as np 13 | 14 | 15 | 16 | default_layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob'] 17 | default_is_conv = [('conv' in ll) for ll in default_layers] 18 | 19 | def hardcoded_get(): 20 | prototxt = '/home/jyosinsk/results/140311_234854_afadfd3_priv_netbase_upgraded/deploy_1.prototxt' 21 | weights = '/home/jyosinsk/results/140311_234854_afadfd3_priv_netbase_upgraded/caffe_imagenet_train_iter_450000' 22 | datadir = '/home/jyosinsk/imagenet2012/val' 23 | filelist = 'mini_valid.txt' 24 | 25 | imagenet_mean = load_imagenet_mean() 26 | net = caffe.Classifier(prototxt, weights, 27 | mean=imagenet_mean, 28 | channel_swap=(2,1,0), 29 | raw_scale=255, 30 | image_dims=(256, 256)) 31 | net.set_phase_test() 32 | net.set_mode_cpu() 33 | labels = load_labels() 34 | 35 | return net, imagenet_mean, labels, datadir, filelist 36 | 37 | 38 | 39 | def mkdir_p(path): 40 | # From https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python 41 | try: 42 | os.makedirs(path) 43 | except OSError as exc: # Python >2.5 44 | if exc.errno == errno.EEXIST and os.path.isdir(path): 45 | pass 46 | else: 47 | raise 48 | 49 | 50 | 51 | class MaxTracker(object): 52 | def __init__(self, is_conv, n_channels, n_top = 10, initial_val = -1e99, dtype = 'float32'): 53 | self.is_conv = is_conv 54 | self.max_vals = np.ones((n_channels, n_top), dtype = dtype) * initial_val 55 | self.n_top = n_top 56 | if is_conv: 57 | self.max_locs = -np.ones((n_channels, n_top, 4), dtype = 'int') # image_idx, image_class, i, j 58 | else: 59 | self.max_locs = -np.ones((n_channels, n_top, 2), dtype = 'int') # image_idx, image_class 60 | 61 | def update(self, blob, image_idx, image_class): 62 | data = blob[0] # Note: makes a copy of blob, e.g. (96,55,55) 63 | n_channels = data.shape[0] 64 | data_unroll = data.reshape((n_channels, -1)) # Note: no copy eg (96,3025). Does nothing if not is_conv 65 | 66 | maxes = data_unroll.argmax(1) # maxes for each channel, eg. (96,) 67 | 68 | #insertion_idx = zeros((n_channels,)) 69 | #pdb.set_trace() 70 | for ii in xrange(n_channels): 71 | idx = np.searchsorted(self.max_vals[ii], data_unroll[ii, maxes[ii]]) 72 | if idx == 0: 73 | # Smaller than all 10 74 | continue 75 | # Store new max in the proper order. Update both arrays: 76 | # self.max_vals: 77 | self.max_vals[ii,:idx-1] = self.max_vals[ii,1:idx] # shift lower values 78 | self.max_vals[ii,idx-1] = data_unroll[ii, maxes[ii]] # store new max value 79 | # self.max_locs 80 | self.max_locs[ii,:idx-1] = self.max_locs[ii,1:idx] # shift lower location data 81 | # store new location 82 | if self.is_conv: 83 | self.max_locs[ii,idx-1] = (image_idx, image_class) + np.unravel_index(maxes[ii], data.shape[1:]) 84 | else: 85 | self.max_locs[ii,idx-1] = (image_idx, image_class) 86 | 87 | 88 | 89 | class NetMaxTracker(object): 90 | def __init__(self, layers = default_layers, is_conv = default_is_conv, n_top = 10, initial_val = -1e99, dtype = 'float32'): 91 | self.layers = layers 92 | self.is_conv = is_conv 93 | self.init_done = False 94 | self.n_top = n_top 95 | self.initial_val = initial_val 96 | 97 | def _init_with_net(self, net): 98 | self.max_trackers = {} 99 | 100 | for layer,is_conv in zip(self.layers, self.is_conv): 101 | blob = net.blobs[layer].data 102 | self.max_trackers[layer] = MaxTracker(is_conv, blob.shape[1], n_top = self.n_top, 103 | initial_val = self.initial_val, 104 | dtype = blob.dtype) 105 | self.init_done = True 106 | 107 | def update(self, net, image_idx, image_class): 108 | '''Updates the maxes found so far with the state of the given net. If a new max is found, it is stored together with the image_idx.''' 109 | if not self.init_done: 110 | self._init_with_net(net) 111 | 112 | for layer in self.layers: 113 | blob = net.blobs[layer].data 114 | self.max_trackers[layer].update(blob, image_idx, image_class) 115 | 116 | 117 | def load_file_list(filelist): 118 | image_filenames = [] 119 | image_labels = [] 120 | with open(filelist, 'r') as ff: 121 | for line in ff.readlines(): 122 | fields = line.strip().split() 123 | image_filenames.append(fields[0]) 124 | image_labels.append(int(fields[1])) 125 | return image_filenames, image_labels 126 | 127 | 128 | 129 | def scan_images_for_maxes(net, datadir, filelist, n_top): 130 | image_filenames, image_labels = load_file_list(filelist) 131 | print 'Scanning %d files' % len(image_filenames) 132 | print ' First file', os.path.join(datadir, image_filenames[0]) 133 | 134 | tracker = NetMaxTracker(n_top = n_top) 135 | for image_idx in xrange(len(image_filenames)): 136 | filename = image_filenames[image_idx] 137 | image_class = image_labels[image_idx] 138 | #im = caffe.io.load_image('../../data/ilsvrc12/mini_ilsvrc_valid/sized/ILSVRC2012_val_00000610.JPEG') 139 | do_print = (image_idx % 100 == 0) 140 | if do_print: 141 | print '%s Image %d/%d' % (datetime.now().ctime(), image_idx, len(image_filenames)) 142 | with WithTimer('Load image', quiet = not do_print): 143 | im = caffe.io.load_image(os.path.join(datadir, filename)) 144 | with WithTimer('Predict ', quiet = not do_print): 145 | net.predict([im], oversample = False) # Just take center crop 146 | with WithTimer('Update ', quiet = not do_print): 147 | tracker.update(net, image_idx, image_class) 148 | 149 | print 'done!' 150 | return tracker 151 | 152 | 153 | 154 | def save_representations(net, datadir, filelist, layer, first_N = None): 155 | image_filenames, image_labels = load_file_list(filelist) 156 | if first_N is None: 157 | first_N = len(image_filenames) 158 | assert first_N <= len(image_filenames) 159 | image_indices = range(first_N) 160 | print 'Scanning %d files' % len(image_indices) 161 | assert len(image_indices) > 0 162 | print ' First file', os.path.join(datadir, image_filenames[image_indices[0]]) 163 | 164 | indices = None 165 | rep = None 166 | for ii,image_idx in enumerate(image_indices): 167 | filename = image_filenames[image_idx] 168 | image_class = image_labels[image_idx] 169 | do_print = (image_idx % 10 == 0) 170 | if do_print: 171 | print '%s Image %d/%d' % (datetime.now().ctime(), image_idx, len(image_indices)) 172 | with WithTimer('Load image', quiet = not do_print): 173 | im = caffe.io.load_image(os.path.join(datadir, filename)) 174 | with WithTimer('Predict ', quiet = not do_print): 175 | net.predict([im], oversample = False) # Just take center crop 176 | with WithTimer('Store ', quiet = not do_print): 177 | if rep is None: 178 | rep_shape = net.blobs[layer].data[0].shape # e.g. (256,13,13) 179 | rep = np.zeros((len(image_indices),) + rep_shape) # e.g. (1000,256,13,13) 180 | indices = [0] * len(image_indices) 181 | indices[ii] = image_idx 182 | rep[ii] = net.blobs[layer].data[0] 183 | 184 | print 'done!' 185 | return indices,rep 186 | 187 | 188 | 189 | def get_max_data_extent(net, layer, rc, is_conv): 190 | '''Gets the maximum size of the data layer that can influence a unit on layer.''' 191 | if is_conv: 192 | conv_size = net.blobs[layer].data.shape[2:4] # e.g. (13,13) for conv5 193 | layer_slice_middle = (conv_size[0]/2,conv_size[0]/2+1, conv_size[1]/2,conv_size[1]/2+1) # e.g. (6,7,6,7,), the single center unit 194 | data_slice = rc.convert_region(layer, 'data', layer_slice_middle) 195 | return data_slice[1]-data_slice[0], data_slice[3]-data_slice[2] # e.g. (163, 163) for conv5 196 | else: 197 | # Whole data region 198 | return net.blobs['data'].data.shape[2:4] # e.g. (227,227) for fc6,fc7,fc8,prop 199 | 200 | 201 | 202 | def output_max_patches(max_tracker, net, layer, idx_begin, idx_end, num_top, datadir, filelist, outdir, do_which): 203 | do_maxes, do_deconv, do_deconv_norm, do_backprop, do_backprop_norm, do_info = do_which 204 | assert do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm or do_info, 'nothing to do' 205 | 206 | mt = max_tracker 207 | rc = RegionComputer() 208 | 209 | image_filenames, image_labels = load_file_list(filelist) 210 | print 'Loaded filenames and labels for %d files' % len(image_filenames) 211 | print ' First file', os.path.join(datadir, image_filenames[0]) 212 | 213 | num_top_in_mt = mt.max_locs.shape[1] 214 | assert num_top <= num_top_in_mt, 'Requested %d top images but MaxTracker contains only %d' % (num_top, num_top_in_mt) 215 | assert idx_end >= idx_begin, 'Range error' 216 | 217 | size_ii, size_jj = get_max_data_extent(net, layer, rc, mt.is_conv) 218 | data_size_ii, data_size_jj = net.blobs['data'].data.shape[2:4] 219 | 220 | n_total_images = (idx_end-idx_begin) * num_top 221 | for cc, channel_idx in enumerate(range(idx_begin, idx_end)): 222 | unit_dir = os.path.join(outdir, layer, 'unit_%04d' % channel_idx) 223 | mkdir_p(unit_dir) 224 | 225 | if do_info: 226 | info_filename = os.path.join(unit_dir, 'info.txt') 227 | info_file = open(info_filename, 'w') 228 | print >>info_file, '# is_conv val image_idx image_class i(if is_conv) j(if is_conv) filename' 229 | 230 | # iterate through maxes from highest (at end) to lowest 231 | for max_idx_0 in range(num_top): 232 | max_idx = num_top_in_mt - 1 - max_idx_0 233 | if mt.is_conv: 234 | im_idx, im_class, ii, jj = mt.max_locs[channel_idx, max_idx] 235 | else: 236 | im_idx, im_class = mt.max_locs[channel_idx, max_idx] 237 | recorded_val = mt.max_vals[channel_idx, max_idx] 238 | filename = image_filenames[im_idx] 239 | do_print = (max_idx_0 == 0) 240 | if do_print: 241 | print '%s Output file/image(s) %d/%d' % (datetime.now().ctime(), cc * num_top, n_total_images) 242 | 243 | if mt.is_conv: 244 | # Compute the focus area of the data layer 245 | layer_indices = (ii,ii+1,jj,jj+1) 246 | data_indices = rc.convert_region(layer, 'data', layer_indices) 247 | data_ii_start,data_ii_end,data_jj_start,data_jj_end = data_indices 248 | 249 | touching_imin = (data_ii_start == 0) 250 | touching_jmin = (data_jj_start == 0) 251 | 252 | # Compute how much of the data slice falls outside the actual data [0,max] range 253 | ii_outside = size_ii - (data_ii_end - data_ii_start) # possibly 0 254 | jj_outside = size_jj - (data_jj_end - data_jj_start) # possibly 0 255 | 256 | if touching_imin: 257 | out_ii_start = ii_outside 258 | out_ii_end = size_ii 259 | else: 260 | out_ii_start = 0 261 | out_ii_end = size_ii - ii_outside 262 | if touching_jmin: 263 | out_jj_start = jj_outside 264 | out_jj_end = size_jj 265 | else: 266 | out_jj_start = 0 267 | out_jj_end = size_jj - jj_outside 268 | else: 269 | ii,jj = 0,0 270 | data_ii_start, out_ii_start, data_jj_start, out_jj_start = 0,0,0,0 271 | data_ii_end, out_ii_end, data_jj_end, out_jj_end = size_ii, size_ii, size_jj, size_jj 272 | 273 | 274 | if do_info: 275 | print >>info_file, 1 if mt.is_conv else 0, '%.6f' % mt.max_vals[channel_idx, max_idx], 276 | if mt.is_conv: 277 | print >>info_file, '%d %d %d %d' % tuple(mt.max_locs[channel_idx, max_idx]), 278 | else: 279 | print >>info_file, '%d %d' % tuple(mt.max_locs[channel_idx, max_idx]), 280 | print >>info_file, filename 281 | 282 | if not (do_maxes or do_deconv or do_deconv_norm or do_backprop or do_backprop_norm): 283 | continue 284 | 285 | 286 | with WithTimer('Load image', quiet = not do_print): 287 | im = caffe.io.load_image(os.path.join(datadir, filename)) 288 | with WithTimer('Predict ', quiet = not do_print): 289 | net.predict([im], oversample = False) # Just take center crop, same as in scan_images_for_maxes 290 | 291 | if len(net.blobs[layer].data.shape) == 4: 292 | reproduced_val = net.blobs[layer].data[0,channel_idx,ii,jj] 293 | else: 294 | reproduced_val = net.blobs[layer].data[0,channel_idx] 295 | if abs(reproduced_val - recorded_val) > .1: 296 | print 'Warning: recorded value %s is suspiciously different from reproduced value %s. Is the filelist the same?' % (recorded_val, reproduced_val) 297 | 298 | if do_maxes: 299 | #grab image from data layer, not from im (to ensure preprocessing / center crop details match between image and deconv/backprop) 300 | out_arr = np.zeros((3,size_ii,size_jj), dtype='float32') 301 | out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].data[0,:,data_ii_start:data_ii_end,data_jj_start:data_jj_end] 302 | with WithTimer('Save img ', quiet = not do_print): 303 | save_caffe_image(out_arr, os.path.join(unit_dir, 'maxim_%03d.png' % max_idx_0), 304 | autoscale = False, autoscale_center = 0) 305 | 306 | if do_deconv or do_deconv_norm: 307 | diffs = net.blobs[layer].diff * 0 308 | if len(diffs.shape) == 4: 309 | diffs[0,channel_idx,ii,jj] = 1.0 310 | else: 311 | diffs[0,channel_idx] = 1.0 312 | with WithTimer('Deconv ', quiet = not do_print): 313 | net.deconv_from_layer(layer, diffs) 314 | 315 | out_arr = np.zeros((3,size_ii,size_jj), dtype='float32') 316 | out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].diff[0,:,data_ii_start:data_ii_end,data_jj_start:data_jj_end] 317 | if out_arr.max() == 0: 318 | print 'Warning: Deconv out_arr in range', out_arr.min(), 'to', out_arr.max(), 'ensure force_backward: true in prototxt' 319 | if do_deconv: 320 | with WithTimer('Save img ', quiet = not do_print): 321 | save_caffe_image(out_arr, os.path.join(unit_dir, 'deconv_%03d.png' % max_idx_0), 322 | autoscale = False, autoscale_center = 0) 323 | if do_deconv_norm: 324 | out_arr = np.linalg.norm(out_arr, axis=0) 325 | with WithTimer('Save img ', quiet = not do_print): 326 | save_caffe_image(out_arr, os.path.join(unit_dir, 'deconvnorm_%03d.png' % max_idx_0)) 327 | 328 | if do_backprop or do_backprop_norm: 329 | diffs = net.blobs[layer].diff * 0 330 | diffs[0,channel_idx,ii,jj] = 1.0 331 | with WithTimer('Backward ', quiet = not do_print): 332 | net.backward_from_layer(layer, diffs) 333 | 334 | out_arr = np.zeros((3,size_ii,size_jj), dtype='float32') 335 | out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = net.blobs['data'].diff[0,:,data_ii_start:data_ii_end,data_jj_start:data_jj_end] 336 | if out_arr.max() == 0: 337 | print 'Warning: Deconv out_arr in range', out_arr.min(), 'to', out_arr.max(), 'ensure force_backward: true in prototxt' 338 | if do_backprop: 339 | with WithTimer('Save img ', quiet = not do_print): 340 | save_caffe_image(out_arr, os.path.join(unit_dir, 'backprop_%03d.png' % max_idx_0), 341 | autoscale = False, autoscale_center = 0) 342 | if do_backprop_norm: 343 | out_arr = np.linalg.norm(out_arr, axis=0) 344 | with WithTimer('Save img ', quiet = not do_print): 345 | save_caffe_image(out_arr, os.path.join(unit_dir, 'backpropnorm_%03d.png' % max_idx_0)) 346 | 347 | if do_info: 348 | info_file.close() 349 | -------------------------------------------------------------------------------- /input_fetcher.py: -------------------------------------------------------------------------------- 1 | import os 2 | import cv2 3 | import re 4 | import time 5 | from threading import RLock 6 | import numpy as np 7 | 8 | from codependent_thread import CodependentThread 9 | from image_misc import cv2_imshow_rgb, cv2_read_file_rgb, read_cam_frame, crop_to_square 10 | from misc import tsplit 11 | 12 | 13 | class InputImageFetcher(CodependentThread): 14 | '''Fetches images from a webcam or loads from a directory.''' 15 | 16 | def __init__(self, settings): 17 | CodependentThread.__init__(self, settings.input_updater_heartbeat_required) 18 | self.daemon = True 19 | self.lock = RLock() 20 | self.quit = False 21 | self.latest_frame_idx = -1 22 | self.latest_frame_data = None 23 | self.latest_frame_is_from_cam = False 24 | 25 | # True for loading from file, False for loading from camera 26 | self.static_file_mode = True 27 | self.settings = settings 28 | 29 | # True for streching the image, False for cropping largest square 30 | self.static_file_stretch_mode = self.settings.static_file_stretch_mode 31 | 32 | # Cam input 33 | self.capture_device = settings.input_updater_capture_device 34 | self.no_cam_present = (self.capture_device is None) # Disable all cam functionality 35 | self.bound_cap_device = None 36 | self.sleep_after_read_frame = settings.input_updater_sleep_after_read_frame 37 | self.latest_cam_frame = None 38 | self.freeze_cam = False 39 | 40 | # Static file input 41 | 42 | # latest image filename selected, used to avoid reloading 43 | self.latest_static_filename = None 44 | 45 | # latest loaded image frame, holds the pixels and used to force reloading 46 | self.latest_static_frame = None 47 | 48 | # keeps current index of loaded file, doesn't seem important 49 | self.static_file_idx = None 50 | 51 | # contains the requested number of increaments for file index 52 | self.static_file_idx_increment = 0 53 | 54 | def bind_camera(self): 55 | # Due to OpenCV limitations, this should be called from the main thread 56 | print 'InputImageFetcher: bind_camera starting' 57 | if self.no_cam_present: 58 | print 'InputImageFetcher: skipping camera bind (device: None)' 59 | else: 60 | self.bound_cap_device = cv2.VideoCapture(self.capture_device) 61 | if self.bound_cap_device.isOpened(): 62 | print 'InputImageFetcher: capture device %s is open' % self.capture_device 63 | else: 64 | print '\n\nWARNING: InputImageFetcher: capture device %s failed to open! Camera will not be available!\n\n' % self.capture_device 65 | self.bound_cap_device = None 66 | self.no_cam_present = True 67 | print 'InputImageFetcher: bind_camera finished' 68 | 69 | def free_camera(self): 70 | # Due to OpenCV limitations, this should be called from the main thread 71 | if self.no_cam_present: 72 | print 'InputImageFetcher: skipping camera free (device: None)' 73 | else: 74 | print 'InputImageFetcher: freeing camera' 75 | del self.bound_cap_device # free the camera 76 | self.bound_cap_device = None 77 | print 'InputImageFetcher: camera freed' 78 | 79 | def set_mode_static(self): 80 | with self.lock: 81 | self.static_file_mode = True 82 | 83 | def set_mode_cam(self): 84 | with self.lock: 85 | if self.no_cam_present: 86 | print 'WARNING: ignoring set_mode_cam, no cam present' 87 | else: 88 | self.static_file_mode = False 89 | assert self.bound_cap_device != None, 'Call bind_camera first' 90 | 91 | def toggle_input_mode(self): 92 | with self.lock: 93 | if self.static_file_mode: 94 | self.set_mode_cam() 95 | else: 96 | self.set_mode_static() 97 | 98 | def set_mode_stretch_on(self): 99 | with self.lock: 100 | if not self.static_file_stretch_mode: 101 | self.static_file_stretch_mode = True 102 | self.latest_static_frame = None # Force reload 103 | #self.latest_frame_is_from_cam = True # Force reload 104 | 105 | def set_mode_stretch_off(self): 106 | with self.lock: 107 | if self.static_file_stretch_mode: 108 | self.static_file_stretch_mode = False 109 | self.latest_static_frame = None # Force reload 110 | #self.latest_frame_is_from_cam = True # Force reload 111 | 112 | def toggle_stretch_mode(self): 113 | with self.lock: 114 | if self.static_file_stretch_mode: 115 | self.set_mode_stretch_off() 116 | else: 117 | self.set_mode_stretch_on() 118 | 119 | def run(self): 120 | while not self.quit and not self.is_timed_out(): 121 | #start_time = time.time() 122 | if self.static_file_mode: 123 | self.check_increment_and_load_image() 124 | else: 125 | if self.freeze_cam and self.latest_cam_frame is not None: 126 | # If static file mode was switched to cam mode but cam is still frozen, we need to push the cam frame again 127 | if not self.latest_frame_is_from_cam: 128 | self._increment_and_set_frame(self.latest_cam_frame, True) 129 | else: 130 | frame_full = read_cam_frame(self.bound_cap_device) 131 | #print '====> just read frame', frame_full.shape 132 | frame = crop_to_square(frame_full) 133 | with self.lock: 134 | self.latest_cam_frame = frame 135 | self._increment_and_set_frame(self.latest_cam_frame, True) 136 | 137 | time.sleep(self.sleep_after_read_frame) 138 | #print 'Reading one frame took', time.time() - start_time 139 | 140 | print 'InputImageFetcher: exiting run method' 141 | #print 'InputImageFetcher: read', self.read_frames, 'frames' 142 | 143 | def get_frame(self): 144 | '''Fetch the latest frame_idx and frame. The idx increments 145 | any time the frame data changes. If the idx is < 0, the frame 146 | is not valid. 147 | ''' 148 | with self.lock: 149 | return (self.latest_frame_idx, self.latest_frame_data) 150 | 151 | def increment_static_file_idx(self, amount = 1): 152 | with self.lock: 153 | self.static_file_idx_increment += amount 154 | 155 | def _increment_and_set_frame(self, frame, from_cam): 156 | assert frame is not None 157 | with self.lock: 158 | self.latest_frame_idx += 1 159 | self.latest_frame_data = frame 160 | self.latest_frame_is_from_cam = from_cam 161 | 162 | def get_files_from_directory(self): 163 | # returns list of files in requested directory 164 | 165 | available_files = [] 166 | match_flags = re.IGNORECASE if self.settings.static_files_ignore_case else 0 167 | for filename in os.listdir(self.settings.static_files_dir): 168 | if re.match(self.settings.static_files_regexp, filename, match_flags): 169 | available_files.append(filename) 170 | 171 | return available_files 172 | 173 | def get_files_from_image_list(self): 174 | # returns list of files in requested image list file 175 | 176 | available_files = [] 177 | 178 | with open(self.settings.static_files_input_file, 'r') as image_list_file: 179 | lines = image_list_file.readlines() 180 | # take first token from each line 181 | available_files = [tsplit(line, True,' ',',','\t')[0] for line in lines if line.strip() != ""] 182 | 183 | return available_files 184 | 185 | def get_files_from_siamese_image_list(self): 186 | # returns list of pair files in requested siamese image list file 187 | 188 | available_files = [] 189 | 190 | with open(self.settings.static_files_input_file, 'r') as image_list_file: 191 | lines = image_list_file.readlines() 192 | # take first and second tokens from each line 193 | available_files = [(tsplit(line, True, ' ', ',','\t')[0], tsplit(line, True, ' ', ',','\t')[1]) 194 | for line in lines if line.strip() != ""] 195 | 196 | return available_files 197 | 198 | def check_increment_and_load_image(self): 199 | with self.lock: 200 | if (self.static_file_idx_increment == 0 and 201 | self.static_file_idx is not None and 202 | not self.latest_frame_is_from_cam and 203 | self.latest_static_frame is not None): 204 | # Skip if a static frame is already loaded and there is no increment 205 | return 206 | 207 | # available_files - local list of files 208 | if self.settings.static_files_input_mode == "directory": 209 | available_files = self.get_files_from_directory() 210 | elif self.settings.static_files_input_mode == "image_list": 211 | available_files = self.get_files_from_image_list() 212 | elif self.settings.static_files_input_mode == "siamese_image_list": 213 | available_files = self.get_files_from_siamese_image_list() 214 | else: 215 | raise Exception(('Error: setting static_files_input_mode has invalid option (%s)' % 216 | (self.settings.static_files_input_mode) )) 217 | 218 | #print 'Found files:' 219 | #for filename in available_files: 220 | # print ' %s' % filename 221 | assert len(available_files) != 0, ('Error: No files found in %s matching %s (current working directory is %s)' % 222 | (self.settings.static_files_dir, self.settings.static_files_regexp, os.getcwd())) 223 | if self.static_file_idx is None: 224 | self.static_file_idx = 0 225 | self.static_file_idx = (self.static_file_idx + self.static_file_idx_increment) % len(available_files) 226 | self.static_file_idx_increment = 0 227 | if self.latest_static_filename != available_files[self.static_file_idx] or self.latest_static_frame is None: 228 | self.latest_static_filename = available_files[self.static_file_idx] 229 | 230 | if self.settings.static_files_input_mode == "siamese_image_list": 231 | # loading two images for siamese network 232 | im1 = cv2_read_file_rgb(os.path.join(self.settings.static_files_dir, self.latest_static_filename[0])) 233 | im2 = cv2_read_file_rgb(os.path.join(self.settings.static_files_dir, self.latest_static_filename[1])) 234 | if not self.static_file_stretch_mode: 235 | im1 = crop_to_square(im1) 236 | im2 = crop_to_square(im2) 237 | 238 | im = (im1,im2) 239 | 240 | else: 241 | im = cv2_read_file_rgb(os.path.join(self.settings.static_files_dir, self.latest_static_filename)) 242 | if not self.static_file_stretch_mode: 243 | im = crop_to_square(im) 244 | 245 | self.latest_static_frame = im 246 | self._increment_and_set_frame(self.latest_static_frame, False) 247 | -------------------------------------------------------------------------------- /input_images/.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | # Ignore images in this directory by default. Add if desired with git add -f 3 | * 4 | -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00000610.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00000610.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00006451.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00006451.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00006491.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00006491.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00008338.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00008338.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00011425.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00011425.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00011425_gray.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00011425_gray.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00016655.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00016655.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00018478.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00018478.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00018548.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00018548.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00020333.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00020333.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00029739.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00029739.jpg -------------------------------------------------------------------------------- /input_images/ILSVRC2012_val_00046554.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/input_images/ILSVRC2012_val_00046554.jpg -------------------------------------------------------------------------------- /keys.py: -------------------------------------------------------------------------------- 1 | # Define keys 2 | 3 | #class KeyPatten(object): 4 | # '''Define a pattern that will be matched against a keycode. 5 | # 6 | # A KeyPattern is used to determine which key was pressed in 7 | # OpenCV. This process is complicated by the fact that different 8 | # platforms define different key codes for each key. Further, on 9 | # some platforms the value returned by OpenCV is different than that 10 | # returned by Python ord(). See the following link for more 11 | # information: 12 | # https://stackoverflow.com/questions/14494101/using-other-keys-for-the-waitkey-function-of-opencv/20577067#20577067 13 | # ''' 14 | # def __init__(self, code, mask = None): 15 | # self.code = code 16 | # self.mask = mask 17 | # #self.mask = 0xffffffff # 64 bits. All codes observed so far are < 2**64 18 | 19 | 20 | 21 | # Larger masks (requiring a more specific pattern) are matched first 22 | key_data = [] 23 | for letter in 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789': 24 | #for letter in 'abefghijklmnopqrstuvwxyzABEFGHIJKLMNOPQRSTUVWXYZ01456789': 25 | key_data.append((letter, ord(letter), 0xff)) # Match only lowest byte 26 | 27 | key_data.extend([ 28 | # Mac (note diff order vs Linux) 29 | ('up', 0xf700, 0xffff), 30 | ('down', 0xf701, 0xffff), 31 | ('left', 0xf702, 0xffff), 32 | ('right', 0xf703, 0xffff), 33 | ('pgup', 0xf72c, 0xffff), 34 | ('pgdn', 0xf72d, 0xffff), 35 | 36 | # Ubuntu US/UK (note diff order vs Mac) 37 | ('left', 0xff51, 0xffff), 38 | ('up', 0xff52, 0xffff), 39 | ('right', 0xff53, 0xffff), 40 | ('down', 0xff54, 0xffff), 41 | 42 | # Ubuntu only; modified keys to not produce separate events on 43 | # Mac. These are included only so they be ignored without 44 | # producing error messages. 45 | ('leftshift', 0xffe1, 0xffff), 46 | ('rightshift', 0xffe2, 0xffff), 47 | ('leftctrl', 0xffe3, 0xffff), 48 | ('rightctrl', 0xffe4, 0xffff), 49 | ('esc', 27, 0xff), # Mac 50 | ('enter', 13, 0xff), # Mac 51 | ('enter', 10, 0xff), # Ubuntu with UK keyboard 52 | ]) 53 | 54 | key_patterns = dict() 55 | # Store key_patterns by mask in a dict of dicts 56 | # Eventually, e.g.: 57 | # key_patterns[0xff][97] = 'a' 58 | for key_datum in key_data: 59 | #print key_datum 60 | assert len(key_datum) in (2,3), 'Key information should be tuple of length 2 or 3 but it is %s' % repr(key_datum) 61 | if len(key_datum) == 3: 62 | label, key_code, mask = key_datum 63 | else: 64 | label, key_code = key_datum 65 | mask = 0xffffffff # 64 bits. All codes observed so far are < 2**64 66 | if not mask in key_patterns: 67 | key_patterns[mask] = dict() 68 | if key_code in key_patterns[mask]: 69 | old_label = key_patterns[mask][code] 70 | if old_label != label: 71 | print 'Warning: key_patterns[%s][%s] old value %s being overwritten with %s' % (mask, key_code, old_label, label) 72 | if key_code != (key_code & mask): 73 | print 'Warning: key_code %s for key label %s will never trigger using mask %s' % (key_code, label, mask) 74 | key_patterns[mask][key_code] = label 75 | #if not label in key_patterns[mask]: 76 | # key_patterns[mask][label] = set() 77 | #key_patterns[mask][label].add(code) 78 | 79 | 80 | 81 | #class Key: 82 | # up=(63232, 'up') 83 | # right=(63235, 'right') 84 | # down=(63233, 'down') 85 | # left=(63234, 'left') 86 | # esc=(27, 'esc') 87 | # enter=(13, 'enter') 88 | # a =(ord('a'),'a') 89 | # b =(ord('b'),'b') 90 | # c =(ord('c'),'c') 91 | # d =(ord('d'),'d') 92 | # e =(ord('e'),'e') 93 | # f =(ord('f'),'f') 94 | # g =(ord('g'),'g') 95 | # h =(ord('h'),'h') 96 | # i =(ord('i'),'i') 97 | # j =(ord('j'),'j') 98 | # k =(ord('k'),'k') 99 | # l =(ord('l'),'l') 100 | # m =(ord('m'),'m') 101 | # n =(ord('n'),'n') 102 | # o =(ord('o'),'o') 103 | # p =(ord('p'),'p') 104 | # q =(ord('q'),'q') 105 | # r =(ord('r'),'r') 106 | # s =(ord('s'),'s') 107 | # t =(ord('t'),'t') 108 | # u =(ord('u'),'u') 109 | # v =(ord('v'),'v') 110 | # w =(ord('w'),'w') 111 | # x =(ord('x'),'x') 112 | # y =(ord('y'),'y') 113 | # z =(ord('z'),'z') 114 | # A =(ord('A'),'A') 115 | # B =(ord('B'),'B') 116 | # C =(ord('C'),'C') 117 | # D =(ord('D'),'D') 118 | # E =(ord('E'),'E') 119 | # F =(ord('F'),'F') 120 | # G =(ord('G'),'G') 121 | # H =(ord('H'),'H') 122 | # I =(ord('I'),'I') 123 | # J =(ord('J'),'J') 124 | # K =(ord('K'),'K') 125 | # L =(ord('L'),'L') 126 | # M =(ord('M'),'M') 127 | # N =(ord('N'),'N') 128 | # O =(ord('O'),'O') 129 | # P =(ord('P'),'P') 130 | # Q =(ord('Q'),'Q') 131 | # R =(ord('R'),'R') 132 | # S =(ord('S'),'S') 133 | # T =(ord('T'),'T') 134 | # U =(ord('U'),'U') 135 | # V =(ord('V'),'V') 136 | # W =(ord('W'),'W') 137 | # X =(ord('X'),'X') 138 | # Y =(ord('Y'),'Y') 139 | # Z =(ord('Z'),'Z') 140 | # n1=(ord('1'),'1') 141 | # n2=(ord('2'),'2') 142 | # n3=(ord('3'),'3') 143 | # n4=(ord('4'),'4') 144 | # n5=(ord('5'),'5') 145 | # n6=(ord('6'),'6') 146 | # n7=(ord('7'),'7') 147 | # n8=(ord('8'),'8') 148 | # n9=(ord('9'),'9') 149 | # n0=(ord('0'),'0') 150 | -------------------------------------------------------------------------------- /live_vis.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import sys 4 | import importlib 5 | from collections import OrderedDict 6 | import numpy as np 7 | from threading import Lock, RLock, Thread 8 | import time 9 | import glob 10 | 11 | try: 12 | import cv2 13 | except ImportError: 14 | print 'Error: Could not import cv2, please install it first.' 15 | raise 16 | 17 | from misc import WithTimer 18 | from image_misc import cv2_imshow_rgb, FormattedString, cv2_typeset_text, to_255 19 | from bindings import bindings 20 | from input_fetcher import InputImageFetcher 21 | 22 | pane_debug_clr = (255, 64, 64) 23 | 24 | class ImproperlyConfigured(Exception): 25 | pass 26 | 27 | 28 | 29 | 30 | class Pane(object): 31 | '''Hold info about one window pane (rectangular region within the main window)''' 32 | 33 | def __init__(self, i_begin, j_begin, i_size, j_size): 34 | self.i_begin = i_begin 35 | self.j_begin = j_begin 36 | self.i_size = i_size 37 | self.j_size = j_size 38 | self.i_end = i_begin + i_size 39 | self.j_end = j_begin + j_size 40 | self.data = None # eventually contains a slice of the window buffer 41 | 42 | 43 | 44 | class LiveVis(object): 45 | '''Runs the demo''' 46 | 47 | def __init__(self, settings): 48 | self.settings = settings 49 | self.bindings = bindings 50 | 51 | self.app_classes = OrderedDict() 52 | self.apps = OrderedDict() 53 | 54 | for module_path, app_name in settings.installed_apps: 55 | module = importlib.import_module(module_path) 56 | print 'got module', module 57 | app_class = getattr(module, app_name) 58 | print 'got app', app_class 59 | self.app_classes[app_name] = app_class 60 | 61 | for app_name, app_class in self.app_classes.iteritems(): 62 | app = app_class(settings, self.bindings) 63 | self.apps[app_name] = app 64 | self.help_mode = False 65 | self.window_name = 'Deep Visualization Toolbox' 66 | self.quit = False 67 | self.debug_level = 0 68 | 69 | self.debug_pane_defaults = { 70 | 'face': getattr(cv2, self.settings.help_face), 71 | 'fsize': self.settings.help_fsize, 72 | 'clr': pane_debug_clr, 73 | 'thick': self.settings.help_thick 74 | } 75 | self.help_pane_defaults = { 76 | 'face': getattr(cv2, self.settings.help_face), 77 | 'fsize': self.settings.help_fsize, 78 | 'clr': to_255(self.settings.help_clr), 79 | 'thick': self.settings.help_thick 80 | } 81 | 82 | 83 | def init_window(self): 84 | cv2.namedWindow(self.window_name) 85 | max_i, max_j = 0, 0 86 | if len(self.settings.window_panes) == 0: 87 | raise ImproperlyConfigured('settings.window_panes is empty.') 88 | self.panes = OrderedDict() 89 | for pane_name, pane_dimensions in self.settings.window_panes: 90 | if len(pane_dimensions) != 4: 91 | raise ImproperlyConfigured('pane dimensions should be a tuple of length 4, but it is "%s"' % repr(pane_dimensions)) 92 | i_begin, j_begin, i_size, j_size = pane_dimensions 93 | max_i = max(max_i, i_begin + i_size) 94 | max_j = max(max_j, j_begin + j_size) 95 | if pane_name in self.panes: 96 | raise Exception('Duplicate pane name in settings: %s' % pane_name) 97 | self.panes[pane_name] = Pane(i_begin, j_begin, i_size, j_size) 98 | self.buffer_height = max_i 99 | self.buffer_width = max_j 100 | 101 | self.window_buffer = np.tile(np.array(np.array(self.settings.window_background) * 255, 'uint8'), 102 | (max_i,max_j,1)) 103 | #print 'BUFFER IS:', self.window_buffer.shape, self.window_buffer.min(), self.window_buffer.max() 104 | 105 | for _,pane in self.panes.iteritems(): 106 | pane.data = self.window_buffer[pane.i_begin:pane.i_end, pane.j_begin:pane.j_end] 107 | 108 | # Allocate help pane 109 | for ll in self.settings.help_pane_loc: 110 | assert ll >= 0 and ll <= 1, 'help_pane_loc values should be in [0,1]' 111 | self.help_pane = Pane(int(self.settings.help_pane_loc[0]*max_i), 112 | int(self.settings.help_pane_loc[1]*max_j), 113 | int(self.settings.help_pane_loc[2]*max_i), 114 | int(self.settings.help_pane_loc[3]*max_j)) 115 | self.help_buffer = self.window_buffer.copy() # For rendering help mode 116 | self.help_pane.data = self.help_buffer[self.help_pane.i_begin:self.help_pane.i_end, self.help_pane.j_begin:self.help_pane.j_end] 117 | 118 | def run_loop(self): 119 | self.quit = False 120 | # Setup 121 | self.init_window() 122 | #cap = cv2.VideoCapture(self.settings.capture_device) 123 | self.input_updater = InputImageFetcher(self.settings) 124 | self.input_updater.bind_camera() 125 | self.input_updater.start() 126 | 127 | heartbeat_functions = [self.input_updater.heartbeat] 128 | for app_name, app in self.apps.iteritems(): 129 | print 'Starting app:', app_name 130 | app.start() 131 | heartbeat_functions.extend(app.get_heartbeats()) 132 | 133 | ii = 0 134 | since_keypress = 999 135 | since_redraw = 999 136 | since_imshow = 0 137 | last_render = time.time() - 999 138 | latest_frame_idx = None 139 | latest_frame_data = None 140 | frame_for_apps = None 141 | redraw_needed = True # Force redraw the first time 142 | imshow_needed = True 143 | while not self.quit: 144 | # Call any heartbeats 145 | for heartbeat in heartbeat_functions: 146 | #print 'Heartbeat: calling', heartbeat 147 | heartbeat() 148 | 149 | # Handle key presses 150 | keys = [] 151 | # Collect key presses (multiple if len(range)>1) 152 | for cc in range(1): 153 | with WithTimer('LiveVis:waitKey', quiet = self.debug_level < 2): 154 | key = cv2.waitKey(self.settings.main_loop_sleep_ms) 155 | if key == -1: 156 | break 157 | else: 158 | if (key != 255): 159 | keys.append(key) 160 | #print 'Got key:', key 161 | now = time.time() 162 | #print 'Since last:', now - last_render 163 | 164 | skip_imshow = False 165 | #if now - last_render > .05 and since_imshow < 1: 166 | # skip_imshow = True 167 | 168 | if skip_imshow: 169 | since_imshow += 1 170 | else: 171 | since_imshow = 0 172 | last_render = now 173 | 174 | #print ' Number of keys:', len(keys) 175 | for key in keys: 176 | since_keypress = 0 177 | #print 'Got Key:', key 178 | key,do_redraw = self.handle_key_pre_apps(key) 179 | redraw_needed |= do_redraw 180 | imshow_needed |= do_redraw 181 | for app_name, app in self.apps.iteritems(): 182 | with WithTimer('%s:handle_key' % app_name, quiet = self.debug_level < 1): 183 | key = app.handle_key(key, self.panes) 184 | key = self.handle_key_post_apps(key) 185 | if self.quit: 186 | break 187 | for app_name, app in self.apps.iteritems(): 188 | redraw_needed |= app.redraw_needed() 189 | 190 | # Grab latest frame from input_updater thread 191 | fr_idx,fr_data = self.input_updater.get_frame() 192 | is_new_frame = (fr_idx != latest_frame_idx and fr_data is not None) 193 | if is_new_frame: 194 | latest_frame_idx = fr_idx 195 | latest_frame_data = fr_data 196 | frame_for_apps = fr_data 197 | 198 | if is_new_frame: 199 | with WithTimer('LiveVis.display_frame', quiet = self.debug_level < 1): 200 | self.display_frame(latest_frame_data) 201 | imshow_needed = True 202 | 203 | do_handle_input = (ii == 0 or 204 | since_keypress >= self.settings.keypress_pause_handle_iterations) 205 | if frame_for_apps is not None and do_handle_input: 206 | # Pass frame to apps for processing 207 | for app_name, app in self.apps.iteritems(): 208 | with WithTimer('%s:handle_input' % app_name, quiet = self.debug_level < 1): 209 | app.handle_input(latest_frame_data, self.panes) 210 | frame_for_apps = None 211 | 212 | # Tell each app to draw 213 | do_redraw = (redraw_needed and 214 | (since_keypress >= self.settings.keypress_pause_redraw_iterations or 215 | since_redraw >= self.settings.redraw_at_least_every)) 216 | if redraw_needed and do_redraw: 217 | for app_name, app in self.apps.iteritems(): 218 | with WithTimer('%s:draw' % app_name, quiet = self.debug_level < 1): 219 | imshow_needed |= app.draw(self.panes) 220 | redraw_needed = False 221 | since_redraw = 0 222 | 223 | # Render buffer 224 | if imshow_needed: 225 | # Only redraw pane debug if display will be updated 226 | if hasattr(self.settings, 'debug_window_panes') and self.settings.debug_window_panes: 227 | for pane_name,pane in self.panes.iteritems(): 228 | print pane_name, pane 229 | pane.data[:] = pane.data * .5 230 | line = [FormattedString('%s |' % pane_name, self.debug_pane_defaults), 231 | FormattedString('pos: %d,%d |' % (pane.i_begin, pane.j_begin), self.debug_pane_defaults), 232 | FormattedString('shape: %d,%d' % (pane.i_size, pane.j_size), self.debug_pane_defaults)] 233 | cv2_typeset_text(pane.data, line, (5,20), line_spacing = 5, wrap = True) 234 | pane.data[:1,:] = pane_debug_clr 235 | pane.data[-1:,:] = pane_debug_clr 236 | pane.data[:,:1] = pane_debug_clr 237 | pane.data[:,-1:] = pane_debug_clr 238 | 239 | with WithTimer('LiveVis:imshow', quiet = self.debug_level < 1): 240 | if self.help_mode: 241 | # Copy main buffer to help buffer 242 | self.help_buffer[:] = self.window_buffer[:] 243 | self.draw_help() 244 | cv2_imshow_rgb(self.window_name, self.help_buffer) 245 | else: 246 | cv2_imshow_rgb(self.window_name, self.window_buffer) 247 | imshow_needed = False 248 | 249 | ii += 1 250 | since_keypress += 1 251 | since_redraw += 1 252 | if ii % 2 == 0 and self.settings.print_dots: 253 | sys.stdout.write('.') 254 | sys.stdout.flush() 255 | # Extra sleep just for debugging. In production all main loop sleep should be in cv2.waitKey. 256 | #time.sleep(2) 257 | 258 | print '\n\nTrying to exit run_loop...' 259 | self.input_updater.quit = True 260 | self.input_updater.join(.01 + float(self.settings.input_updater_sleep_after_read_frame) * 5) 261 | if self.input_updater.is_alive(): 262 | raise Exception('Could not join self.input_updater thread') 263 | else: 264 | self.input_updater.free_camera() 265 | 266 | for app_name, app in self.apps.iteritems(): 267 | print 'Quitting app:', app_name 268 | app.quit() 269 | 270 | print 'Input thread joined and apps quit; exiting run_loop.' 271 | 272 | def handle_key_pre_apps(self, key): 273 | tag = self.bindings.get_tag(key) 274 | if tag == 'freeze_cam': 275 | self.input_updater.freeze_cam = not self.input_updater.freeze_cam 276 | elif tag == 'toggle_input_mode': 277 | self.input_updater.toggle_input_mode() 278 | elif tag == 'static_file_increment': 279 | if self.input_updater.static_file_mode: 280 | self.input_updater.increment_static_file_idx(1) 281 | else: 282 | self.input_updater.static_file_mode = True 283 | elif tag == 'static_file_decrement': 284 | if self.input_updater.static_file_mode: 285 | self.input_updater.increment_static_file_idx(-1) 286 | else: 287 | self.input_updater.static_file_mode = True 288 | elif tag == 'help_mode': 289 | self.help_mode = not self.help_mode 290 | elif tag == 'stretch_mode': 291 | self.input_updater.toggle_stretch_mode() 292 | print 'Stretch mode is now', self.input_updater.static_file_stretch_mode 293 | elif tag == 'debug_level': 294 | self.debug_level = (self.debug_level + 1) % 3 295 | for app_name, app in self.apps.iteritems(): 296 | app.set_debug(self.debug_level) 297 | else: 298 | return key, False 299 | return None, True 300 | 301 | def handle_key_post_apps(self, key): 302 | tag = self.bindings.get_tag(key) 303 | if tag == 'quit': 304 | self.quit = True 305 | elif key == None: 306 | pass 307 | else: 308 | key_label, masked_vals = self.bindings.get_key_label_from_keycode(key, extra_info = True) 309 | masked_vals_pp = ', '.join(['%d (%s)' % (mv, hex(mv)) for mv in masked_vals]) 310 | if key_label is None: 311 | print 'Got key code %d (%s), did not match any known key (masked vals tried: %s)' % (key, hex(key), masked_vals_pp) 312 | elif tag is None: 313 | print 'Got key code %d (%s), matched key "%s", but key is not bound to any function' % (key, hex(key), key_label) 314 | else: 315 | print 'Got key code %d (%s), matched key "%s", bound to "%s", but nobody handled "%s"' % ( 316 | key, hex(key), key_label, tag, tag) 317 | 318 | def display_frame(self, frame): 319 | if self.settings.static_files_input_mode == "siamese_image_list": 320 | frame1 = frame[0] 321 | frame2 = frame[1] 322 | full_pane_shape = self.panes['input'].data.shape[:2][::-1] 323 | half_pane_shape = (full_pane_shape[0] / 2, full_pane_shape[1]) 324 | frame_disp1 = cv2.resize(frame1[:], half_pane_shape) 325 | frame_disp2 = cv2.resize(frame2[:], half_pane_shape) 326 | frame_disp = np.concatenate((frame_disp1, frame_disp2), axis=1) 327 | 328 | else: 329 | frame_disp = cv2.resize(frame[:], self.panes['input'].data.shape[:2][::-1]) 330 | 331 | self.panes['input'].data[:] = frame_disp 332 | 333 | def draw_help(self): 334 | self.help_buffer[:] = self.help_buffer[:] * .7 335 | self.help_pane.data[:] = self.help_pane.data[:] * .7 336 | 337 | loc = self.settings.help_loc[::-1] # Reverse to OpenCV c,r order 338 | defaults = self.help_pane_defaults 339 | lines = [] 340 | lines.append([FormattedString('~ ~ ~ Deep Visualization Toolbox ~ ~ ~', defaults, align='center', width=self.help_pane.j_size)]) 341 | lines.append([FormattedString('', defaults)]) 342 | lines.append([FormattedString('Base keys', defaults)]) 343 | 344 | for tag in ('help_mode', 'freeze_cam', 'toggle_input_mode', 'static_file_increment', 'static_file_decrement', 'stretch_mode', 'quit'): 345 | key_strings, help_string = self.bindings.get_key_help(tag) 346 | label = '%10s:' % (','.join(key_strings)) 347 | lines.append([FormattedString(label, defaults, width=120, align='right'), 348 | FormattedString(help_string, defaults)]) 349 | 350 | locy = cv2_typeset_text(self.help_pane.data, lines, loc, 351 | line_spacing = self.settings.help_line_spacing) 352 | 353 | for app_name, app in self.apps.iteritems(): 354 | locy = app.draw_help(self.help_pane, locy) 355 | 356 | 357 | 358 | if __name__ == '__main__': 359 | print 'You probably want to run ./run_toolbox.py instead.' 360 | -------------------------------------------------------------------------------- /misc.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import time 5 | import errno 6 | import re 7 | 8 | 9 | class WithTimer: 10 | def __init__(self, title = '', quiet = False): 11 | self.title = title 12 | self.quiet = quiet 13 | 14 | def elapsed(self): 15 | return time.time() - self.wall, time.clock() - self.proc 16 | 17 | def enter(self): 18 | '''Manually trigger enter''' 19 | self.__enter__() 20 | 21 | def __enter__(self): 22 | self.proc = time.clock() 23 | self.wall = time.time() 24 | return self 25 | 26 | def __exit__(self, *args): 27 | if not self.quiet: 28 | titlestr = (' ' + self.title) if self.title else '' 29 | print 'Elapsed%s: wall: %.06f, sys: %.06f' % ((titlestr,) + self.elapsed()) 30 | 31 | 32 | 33 | def mkdir_p(path): 34 | # From https://stackoverflow.com/questions/600268/mkdir-p-functionality-in-python 35 | try: 36 | os.makedirs(path) 37 | except OSError as exc: # Python >2.5 38 | if exc.errno == errno.EEXIST and os.path.isdir(path): 39 | pass 40 | else: 41 | raise 42 | 43 | 44 | 45 | def combine_dicts(dicts_tuple): 46 | '''Combines multiple dictionaries into one by adding a prefix to keys''' 47 | ret = {} 48 | for prefix,dictionary in dicts_tuple: 49 | for key in dictionary.keys(): 50 | ret['%s%s' % (prefix, key)] = dictionary[key] 51 | return ret 52 | 53 | 54 | def tsplit(string, no_empty_strings, *delimiters): 55 | # split string using multiple delimiters 56 | 57 | pattern = '|'.join(map(re.escape, delimiters)) 58 | strings = re.split(pattern, string) 59 | if no_empty_strings: 60 | strings = filter(None, strings) 61 | 62 | return strings -------------------------------------------------------------------------------- /models/.gitignore: -------------------------------------------------------------------------------- 1 | *.caffemodel 2 | -------------------------------------------------------------------------------- /models/bvlc-googlenet/fetch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Exit on first error 4 | set -e 5 | 6 | function fetch_file() { 7 | filename="$1" 8 | url="$2" 9 | if [ -e "$filename" ]; then 10 | echo "$url: file already downloaded (remove $filename to force re-download)" 11 | else 12 | echo "$url: fetching..." 13 | wget -O "$filename" "$url" 14 | echo "$url: done." 15 | fi 16 | } 17 | 18 | function fetch_and_extract() { 19 | filename="$1" 20 | url="$2" 21 | dir="$3" 22 | example_filename="$4" 23 | example_path="$dir/$example_filename" 24 | if [ -e "$example_path" ]; then 25 | echo "$url: $example_path already exists, skipping." 26 | else 27 | fetch_file "$filename" "$url" 28 | echo "$url: extracting..." 29 | mkdir -p "$dir" 30 | tar -C "$dir" -xzf "$filename" 31 | echo "$url: done." 32 | fi 33 | } 34 | 35 | fetch_file bvlc-googlenet.caffemodel http://dl.caffe.berkeleyvision.org/bvlc_googlenet.caffemodel 36 | -------------------------------------------------------------------------------- /models/bvlc-googlenet/ilsvrc_2012_labels.txt: -------------------------------------------------------------------------------- 1 | tench 2 | goldfish 3 | great white shark 4 | tiger shark 5 | hammerhead 6 | electric ray 7 | stingray 8 | cock 9 | hen 10 | ostrich 11 | brambling 12 | goldfinch 13 | house finch 14 | junco 15 | indigo bunting 16 | robin 17 | bulbul 18 | jay 19 | magpie 20 | chickadee 21 | water ouzel 22 | kite 23 | bald eagle 24 | vulture 25 | great grey owl 26 | European fire salamander 27 | common newt 28 | eft 29 | spotted salamander 30 | axolotl 31 | bullfrog 32 | tree frog 33 | tailed frog 34 | loggerhead 35 | leatherback turtle 36 | mud turtle 37 | terrapin 38 | box turtle 39 | banded gecko 40 | common iguana 41 | American chameleon 42 | whiptail 43 | agama 44 | frilled lizard 45 | alligator lizard 46 | Gila monster 47 | green lizard 48 | African chameleon 49 | Komodo dragon 50 | African crocodile 51 | American alligator 52 | triceratops 53 | thunder snake 54 | ringneck snake 55 | hognose snake 56 | green snake 57 | king snake 58 | garter snake 59 | water snake 60 | vine snake 61 | night snake 62 | boa constrictor 63 | rock python 64 | Indian cobra 65 | green mamba 66 | sea snake 67 | horned viper 68 | diamondback 69 | sidewinder 70 | trilobite 71 | harvestman 72 | scorpion 73 | black and gold garden spider 74 | barn spider 75 | garden spider 76 | black widow 77 | tarantula 78 | wolf spider 79 | tick 80 | centipede 81 | black grouse 82 | ptarmigan 83 | ruffed grouse 84 | prairie chicken 85 | peacock 86 | quail 87 | partridge 88 | African grey 89 | macaw 90 | sulphur-crested cockatoo 91 | lorikeet 92 | coucal 93 | bee eater 94 | hornbill 95 | hummingbird 96 | jacamar 97 | toucan 98 | drake 99 | red-breasted merganser 100 | goose 101 | black swan 102 | tusker 103 | echidna 104 | platypus 105 | wallaby 106 | koala 107 | wombat 108 | jellyfish 109 | sea anemone 110 | brain coral 111 | flatworm 112 | nematode 113 | conch 114 | snail 115 | slug 116 | sea slug 117 | chiton 118 | chambered nautilus 119 | Dungeness crab 120 | rock crab 121 | fiddler crab 122 | king crab 123 | American lobster 124 | spiny lobster 125 | crayfish 126 | hermit crab 127 | isopod 128 | white stork 129 | black stork 130 | spoonbill 131 | flamingo 132 | little blue heron 133 | American egret 134 | bittern 135 | crane 136 | limpkin 137 | European gallinule 138 | American coot 139 | bustard 140 | ruddy turnstone 141 | red-backed sandpiper 142 | redshank 143 | dowitcher 144 | oystercatcher 145 | pelican 146 | king penguin 147 | albatross 148 | grey whale 149 | killer whale 150 | dugong 151 | sea lion 152 | Chihuahua 153 | Japanese spaniel 154 | Maltese dog 155 | Pekinese 156 | Shih-Tzu 157 | Blenheim spaniel 158 | papillon 159 | toy terrier 160 | Rhodesian ridgeback 161 | Afghan hound 162 | basset 163 | beagle 164 | bloodhound 165 | bluetick 166 | black-and-tan coonhound 167 | Walker hound 168 | English foxhound 169 | redbone 170 | borzoi 171 | Irish wolfhound 172 | Italian greyhound 173 | whippet 174 | Ibizan hound 175 | Norwegian elkhound 176 | otterhound 177 | Saluki 178 | Scottish deerhound 179 | Weimaraner 180 | Staffordshire bullterrier 181 | American Staffordshire terrier 182 | Bedlington terrier 183 | Border terrier 184 | Kerry blue terrier 185 | Irish terrier 186 | Norfolk terrier 187 | Norwich terrier 188 | Yorkshire terrier 189 | wire-haired fox terrier 190 | Lakeland terrier 191 | Sealyham terrier 192 | Airedale 193 | cairn 194 | Australian terrier 195 | Dandie Dinmont 196 | Boston bull 197 | miniature schnauzer 198 | giant schnauzer 199 | standard schnauzer 200 | Scotch terrier 201 | Tibetan terrier 202 | silky terrier 203 | soft-coated wheaten terrier 204 | West Highland white terrier 205 | Lhasa 206 | flat-coated retriever 207 | curly-coated retriever 208 | golden retriever 209 | Labrador retriever 210 | Chesapeake Bay retriever 211 | German short-haired pointer 212 | vizsla 213 | English setter 214 | Irish setter 215 | Gordon setter 216 | Brittany spaniel 217 | clumber 218 | English springer 219 | Welsh springer spaniel 220 | cocker spaniel 221 | Sussex spaniel 222 | Irish water spaniel 223 | kuvasz 224 | schipperke 225 | groenendael 226 | malinois 227 | briard 228 | kelpie 229 | komondor 230 | Old English sheepdog 231 | Shetland sheepdog 232 | collie 233 | Border collie 234 | Bouvier des Flandres 235 | Rottweiler 236 | German shepherd 237 | Doberman 238 | miniature pinscher 239 | Greater Swiss Mountain dog 240 | Bernese mountain dog 241 | Appenzeller 242 | EntleBucher 243 | boxer 244 | bull mastiff 245 | Tibetan mastiff 246 | French bulldog 247 | Great Dane 248 | Saint Bernard 249 | Eskimo dog 250 | malamute 251 | Siberian husky 252 | dalmatian 253 | affenpinscher 254 | basenji 255 | pug 256 | Leonberg 257 | Newfoundland 258 | Great Pyrenees 259 | Samoyed 260 | Pomeranian 261 | chow 262 | keeshond 263 | Brabancon griffon 264 | Pembroke 265 | Cardigan 266 | toy poodle 267 | miniature poodle 268 | standard poodle 269 | Mexican hairless 270 | timber wolf 271 | white wolf 272 | red wolf 273 | coyote 274 | dingo 275 | dhole 276 | African hunting dog 277 | hyena 278 | red fox 279 | kit fox 280 | Arctic fox 281 | grey fox 282 | tabby 283 | tiger cat 284 | Persian cat 285 | Siamese cat 286 | Egyptian cat 287 | cougar 288 | lynx 289 | leopard 290 | snow leopard 291 | jaguar 292 | lion 293 | tiger 294 | cheetah 295 | brown bear 296 | American black bear 297 | ice bear 298 | sloth bear 299 | mongoose 300 | meerkat 301 | tiger beetle 302 | ladybug 303 | ground beetle 304 | long-horned beetle 305 | leaf beetle 306 | dung beetle 307 | rhinoceros beetle 308 | weevil 309 | fly 310 | bee 311 | ant 312 | grasshopper 313 | cricket 314 | walking stick 315 | cockroach 316 | mantis 317 | cicada 318 | leafhopper 319 | lacewing 320 | dragonfly 321 | damselfly 322 | admiral 323 | ringlet 324 | monarch 325 | cabbage butterfly 326 | sulphur butterfly 327 | lycaenid 328 | starfish 329 | sea urchin 330 | sea cucumber 331 | wood rabbit 332 | hare 333 | Angora 334 | hamster 335 | porcupine 336 | fox squirrel 337 | marmot 338 | beaver 339 | guinea pig 340 | sorrel 341 | zebra 342 | hog 343 | wild boar 344 | warthog 345 | hippopotamus 346 | ox 347 | water buffalo 348 | bison 349 | ram 350 | bighorn 351 | ibex 352 | hartebeest 353 | impala 354 | gazelle 355 | Arabian camel 356 | llama 357 | weasel 358 | mink 359 | polecat 360 | black-footed ferret 361 | otter 362 | skunk 363 | badger 364 | armadillo 365 | three-toed sloth 366 | orangutan 367 | gorilla 368 | chimpanzee 369 | gibbon 370 | siamang 371 | guenon 372 | patas 373 | baboon 374 | macaque 375 | langur 376 | colobus 377 | proboscis monkey 378 | marmoset 379 | capuchin 380 | howler monkey 381 | titi 382 | spider monkey 383 | squirrel monkey 384 | Madagascar cat 385 | indri 386 | Indian elephant 387 | African elephant 388 | lesser panda 389 | giant panda 390 | barracouta 391 | eel 392 | coho 393 | rock beauty 394 | anemone fish 395 | sturgeon 396 | gar 397 | lionfish 398 | puffer 399 | abacus 400 | abaya 401 | academic gown 402 | accordion 403 | acoustic guitar 404 | aircraft carrier 405 | airliner 406 | airship 407 | altar 408 | ambulance 409 | amphibian 410 | analog clock 411 | apiary 412 | apron 413 | ashcan 414 | assault rifle 415 | backpack 416 | bakery 417 | balance beam 418 | balloon 419 | ballpoint 420 | Band Aid 421 | banjo 422 | bannister 423 | barbell 424 | barber chair 425 | barbershop 426 | barn 427 | barometer 428 | barrel 429 | barrow 430 | baseball 431 | basketball 432 | bassinet 433 | bassoon 434 | bathing cap 435 | bath towel 436 | bathtub 437 | beach wagon 438 | beacon 439 | beaker 440 | bearskin 441 | beer bottle 442 | beer glass 443 | bell cote 444 | bib 445 | bicycle-built-for-two 446 | bikini 447 | binder 448 | binoculars 449 | birdhouse 450 | boathouse 451 | bobsled 452 | bolo tie 453 | bonnet 454 | bookcase 455 | bookshop 456 | bottlecap 457 | bow 458 | bow tie 459 | brass 460 | brassiere 461 | breakwater 462 | breastplate 463 | broom 464 | bucket 465 | buckle 466 | bulletproof vest 467 | bullet train 468 | butcher shop 469 | cab 470 | caldron 471 | candle 472 | cannon 473 | canoe 474 | can opener 475 | cardigan 476 | car mirror 477 | carousel 478 | carpenter's kit 479 | carton 480 | car wheel 481 | cash machine 482 | cassette 483 | cassette player 484 | castle 485 | catamaran 486 | CD player 487 | cello 488 | cellular telephone 489 | chain 490 | chainlink fence 491 | chain mail 492 | chain saw 493 | chest 494 | chiffonier 495 | chime 496 | china cabinet 497 | Christmas stocking 498 | church 499 | cinema 500 | cleaver 501 | cliff dwelling 502 | cloak 503 | clog 504 | cocktail shaker 505 | coffee mug 506 | coffeepot 507 | coil 508 | combination lock 509 | computer keyboard 510 | confectionery 511 | container ship 512 | convertible 513 | corkscrew 514 | cornet 515 | cowboy boot 516 | cowboy hat 517 | cradle 518 | crane 519 | crash helmet 520 | crate 521 | crib 522 | Crock Pot 523 | croquet ball 524 | crutch 525 | cuirass 526 | dam 527 | desk 528 | desktop computer 529 | dial telephone 530 | diaper 531 | digital clock 532 | digital watch 533 | dining table 534 | dishrag 535 | dishwasher 536 | disk brake 537 | dock 538 | dogsled 539 | dome 540 | doormat 541 | drilling platform 542 | drum 543 | drumstick 544 | dumbbell 545 | Dutch oven 546 | electric fan 547 | electric guitar 548 | electric locomotive 549 | entertainment center 550 | envelope 551 | espresso maker 552 | face powder 553 | feather boa 554 | file 555 | fireboat 556 | fire engine 557 | fire screen 558 | flagpole 559 | flute 560 | folding chair 561 | football helmet 562 | forklift 563 | fountain 564 | fountain pen 565 | four-poster 566 | freight car 567 | French horn 568 | frying pan 569 | fur coat 570 | garbage truck 571 | gasmask 572 | gas pump 573 | goblet 574 | go-kart 575 | golf ball 576 | golfcart 577 | gondola 578 | gong 579 | gown 580 | grand piano 581 | greenhouse 582 | grille 583 | grocery store 584 | guillotine 585 | hair slide 586 | hair spray 587 | half track 588 | hammer 589 | hamper 590 | hand blower 591 | hand-held computer 592 | handkerchief 593 | hard disc 594 | harmonica 595 | harp 596 | harvester 597 | hatchet 598 | holster 599 | home theater 600 | honeycomb 601 | hook 602 | hoopskirt 603 | horizontal bar 604 | horse cart 605 | hourglass 606 | iPod 607 | iron 608 | jack-o'-lantern 609 | jean 610 | jeep 611 | jersey 612 | jigsaw puzzle 613 | jinrikisha 614 | joystick 615 | kimono 616 | knee pad 617 | knot 618 | lab coat 619 | ladle 620 | lampshade 621 | laptop 622 | lawn mower 623 | lens cap 624 | letter opener 625 | library 626 | lifeboat 627 | lighter 628 | limousine 629 | liner 630 | lipstick 631 | Loafer 632 | lotion 633 | loudspeaker 634 | loupe 635 | lumbermill 636 | magnetic compass 637 | mailbag 638 | mailbox 639 | maillot 640 | maillot 641 | manhole cover 642 | maraca 643 | marimba 644 | mask 645 | matchstick 646 | maypole 647 | maze 648 | measuring cup 649 | medicine chest 650 | megalith 651 | microphone 652 | microwave 653 | military uniform 654 | milk can 655 | minibus 656 | miniskirt 657 | minivan 658 | missile 659 | mitten 660 | mixing bowl 661 | mobile home 662 | Model T 663 | modem 664 | monastery 665 | monitor 666 | moped 667 | mortar 668 | mortarboard 669 | mosque 670 | mosquito net 671 | motor scooter 672 | mountain bike 673 | mountain tent 674 | mouse 675 | mousetrap 676 | moving van 677 | muzzle 678 | nail 679 | neck brace 680 | necklace 681 | nipple 682 | notebook 683 | obelisk 684 | oboe 685 | ocarina 686 | odometer 687 | oil filter 688 | organ 689 | oscilloscope 690 | overskirt 691 | oxcart 692 | oxygen mask 693 | packet 694 | paddle 695 | paddlewheel 696 | padlock 697 | paintbrush 698 | pajama 699 | palace 700 | panpipe 701 | paper towel 702 | parachute 703 | parallel bars 704 | park bench 705 | parking meter 706 | passenger car 707 | patio 708 | pay-phone 709 | pedestal 710 | pencil box 711 | pencil sharpener 712 | perfume 713 | Petri dish 714 | photocopier 715 | pick 716 | pickelhaube 717 | picket fence 718 | pickup 719 | pier 720 | piggy bank 721 | pill bottle 722 | pillow 723 | ping-pong ball 724 | pinwheel 725 | pirate 726 | pitcher 727 | plane 728 | planetarium 729 | plastic bag 730 | plate rack 731 | plow 732 | plunger 733 | Polaroid camera 734 | pole 735 | police van 736 | poncho 737 | pool table 738 | pop bottle 739 | pot 740 | potter's wheel 741 | power drill 742 | prayer rug 743 | printer 744 | prison 745 | projectile 746 | projector 747 | puck 748 | punching bag 749 | purse 750 | quill 751 | quilt 752 | racer 753 | racket 754 | radiator 755 | radio 756 | radio telescope 757 | rain barrel 758 | recreational vehicle 759 | reel 760 | reflex camera 761 | refrigerator 762 | remote control 763 | restaurant 764 | revolver 765 | rifle 766 | rocking chair 767 | rotisserie 768 | rubber eraser 769 | rugby ball 770 | rule 771 | running shoe 772 | safe 773 | safety pin 774 | saltshaker 775 | sandal 776 | sarong 777 | sax 778 | scabbard 779 | scale 780 | school bus 781 | schooner 782 | scoreboard 783 | screen 784 | screw 785 | screwdriver 786 | seat belt 787 | sewing machine 788 | shield 789 | shoe shop 790 | shoji 791 | shopping basket 792 | shopping cart 793 | shovel 794 | shower cap 795 | shower curtain 796 | ski 797 | ski mask 798 | sleeping bag 799 | slide rule 800 | sliding door 801 | slot 802 | snorkel 803 | snowmobile 804 | snowplow 805 | soap dispenser 806 | soccer ball 807 | sock 808 | solar dish 809 | sombrero 810 | soup bowl 811 | space bar 812 | space heater 813 | space shuttle 814 | spatula 815 | speedboat 816 | spider web 817 | spindle 818 | sports car 819 | spotlight 820 | stage 821 | steam locomotive 822 | steel arch bridge 823 | steel drum 824 | stethoscope 825 | stole 826 | stone wall 827 | stopwatch 828 | stove 829 | strainer 830 | streetcar 831 | stretcher 832 | studio couch 833 | stupa 834 | submarine 835 | suit 836 | sundial 837 | sunglass 838 | sunglasses 839 | sunscreen 840 | suspension bridge 841 | swab 842 | sweatshirt 843 | swimming trunks 844 | swing 845 | switch 846 | syringe 847 | table lamp 848 | tank 849 | tape player 850 | teapot 851 | teddy 852 | television 853 | tennis ball 854 | thatch 855 | theater curtain 856 | thimble 857 | thresher 858 | throne 859 | tile roof 860 | toaster 861 | tobacco shop 862 | toilet seat 863 | torch 864 | totem pole 865 | tow truck 866 | toyshop 867 | tractor 868 | trailer truck 869 | tray 870 | trench coat 871 | tricycle 872 | trimaran 873 | tripod 874 | triumphal arch 875 | trolleybus 876 | trombone 877 | tub 878 | turnstile 879 | typewriter keyboard 880 | umbrella 881 | unicycle 882 | upright 883 | vacuum 884 | vase 885 | vault 886 | velvet 887 | vending machine 888 | vestment 889 | viaduct 890 | violin 891 | volleyball 892 | waffle iron 893 | wall clock 894 | wallet 895 | wardrobe 896 | warplane 897 | washbasin 898 | washer 899 | water bottle 900 | water jug 901 | water tower 902 | whiskey jug 903 | whistle 904 | wig 905 | window screen 906 | window shade 907 | Windsor tie 908 | wine bottle 909 | wing 910 | wok 911 | wooden spoon 912 | wool 913 | worm fence 914 | wreck 915 | yawl 916 | yurt 917 | web site 918 | comic book 919 | crossword puzzle 920 | street sign 921 | traffic light 922 | book jacket 923 | menu 924 | plate 925 | guacamole 926 | consomme 927 | hot pot 928 | trifle 929 | ice cream 930 | ice lolly 931 | French loaf 932 | bagel 933 | pretzel 934 | cheeseburger 935 | hotdog 936 | mashed potato 937 | head cabbage 938 | broccoli 939 | cauliflower 940 | zucchini 941 | spaghetti squash 942 | acorn squash 943 | butternut squash 944 | cucumber 945 | artichoke 946 | bell pepper 947 | cardoon 948 | mushroom 949 | Granny Smith 950 | strawberry 951 | orange 952 | lemon 953 | fig 954 | pineapple 955 | banana 956 | jackfruit 957 | custard apple 958 | pomegranate 959 | hay 960 | carbonara 961 | chocolate sauce 962 | dough 963 | meat loaf 964 | pizza 965 | potpie 966 | burrito 967 | red wine 968 | espresso 969 | cup 970 | eggnog 971 | alp 972 | bubble 973 | cliff 974 | coral reef 975 | geyser 976 | lakeside 977 | promontory 978 | sandbar 979 | seashore 980 | valley 981 | volcano 982 | ballplayer 983 | groom 984 | scuba diver 985 | rapeseed 986 | daisy 987 | yellow lady's slipper 988 | corn 989 | acorn 990 | hip 991 | buckeye 992 | coral fungus 993 | agaric 994 | gyromitra 995 | stinkhorn 996 | earthstar 997 | hen-of-the-woods 998 | bolete 999 | ear 1000 | toilet tissue 1001 | -------------------------------------------------------------------------------- /models/bvlc-googlenet/settings_local.template-bvlc-googlenet.py: -------------------------------------------------------------------------------- 1 | # Define critical settings and/or override defaults specified in 2 | # settings.py. Copy this file to settings_local.py in the same 3 | # directory as settings.py and edit. Any settings defined here 4 | # will override those defined in settings.py 5 | 6 | 7 | 8 | # Set this to point to your compiled checkout of caffe 9 | caffevis_caffe_root = '/path/to/caffe' 10 | 11 | # Load model: bvlc-googlenet 12 | # Path to caffe deploy prototxt file. Minibatch size should be 1. 13 | caffevis_deploy_prototxt = '%DVT_ROOT%/models/bvlc-googlenet/bvlc-googlenet-deploy.prototxt' 14 | 15 | # Path to network weights to load. 16 | caffevis_network_weights = '%DVT_ROOT%/models/bvlc-googlenet/bvlc-googlenet.caffemodel' 17 | 18 | # Other optional settings; see complete documentation for each in settings.py. 19 | caffevis_data_mean = (104, 117, 123) # per-channel mean 20 | caffevis_labels = '%DVT_ROOT%/models/bvlc-googlenet/ilsvrc_2012_labels.txt' 21 | caffevis_jpgvis_layers = [] 22 | caffevis_prob_layer = 'prob' 23 | caffevis_label_layers = ('loss3/classifier', 'prob') 24 | def caffevis_layer_pretty_name_fn(name): 25 | # Shorten many layer names to fit in control pane (full layer name visible in status bar) 26 | name = name.replace('conv','c').replace('pool','p').replace('norm','n') 27 | name = name.replace('inception_','i').replace('output','o').replace('reduce','r').replace('split_','s') 28 | name = name.replace('__','_').replace('__','_') 29 | return name 30 | # Don't display duplicate *_split_* layers 31 | caffevis_filter_layers = lambda name: '_split_' in name 32 | 33 | # Window panes for bvlc-googlenet (no caffevis_jpgvis pane, larger control pane to fit many layer names). 34 | _control_height = 125 35 | window_panes = ( 36 | # (i, j, i_size, j_size) 37 | ('input', ( 0, 0, 300, 300)), 38 | ('caffevis_aux', (300, 0, 300, 300)), 39 | ('caffevis_back', (600, 0, 300, 300)), 40 | ('caffevis_status', (900, 0, 30, 1500)), 41 | ('caffevis_control', ( 0, 300, _control_height, 1200)), 42 | ('caffevis_layers', ( _control_height, 300, 900 - _control_height, 1200)), 43 | ) 44 | caffevis_layers_aspect_ratio = float(window_panes[-1][1][3])/window_panes[-1][1][2] # Actual ratio from caffevis_layers 45 | caffevis_control_fsize = .85 46 | 47 | # Use GPU? Default is True. 48 | #caffevis_mode_gpu = True 49 | # Display tweaks. 50 | # Scale all window panes in UI by this factor 51 | #global_scale = 1.0 52 | # Scale all fonts by this factor 53 | #global_font_size = 1.0 54 | 55 | -------------------------------------------------------------------------------- /models/caffenet-yos/.gitignore: -------------------------------------------------------------------------------- 1 | ilsvrc_2012_mean.npy 2 | *.tar.gz 3 | *weights 4 | unit_jpg_vis 5 | -------------------------------------------------------------------------------- /models/caffenet-yos/caffenet-yos-deploy.prototxt: -------------------------------------------------------------------------------- 1 | name: "CaffeNet" 2 | input: "data" 3 | input_dim: 1 4 | input_dim: 3 5 | input_dim: 227 6 | input_dim: 227 7 | force_backward: true 8 | layers { 9 | name: "conv1" 10 | type: CONVOLUTION 11 | bottom: "data" 12 | top: "conv1" 13 | convolution_param { 14 | num_output: 96 15 | kernel_size: 11 16 | stride: 4 17 | } 18 | } 19 | layers { 20 | name: "relu1" 21 | type: RELU 22 | bottom: "conv1" 23 | top: "conv1" 24 | } 25 | layers { 26 | name: "pool1" 27 | type: POOLING 28 | bottom: "conv1" 29 | top: "pool1" 30 | pooling_param { 31 | pool: MAX 32 | kernel_size: 3 33 | stride: 2 34 | } 35 | } 36 | layers { 37 | name: "norm1" 38 | type: LRN 39 | bottom: "pool1" 40 | top: "norm1" 41 | lrn_param { 42 | local_size: 5 43 | alpha: 0.0001 44 | beta: 0.75 45 | } 46 | } 47 | layers { 48 | name: "conv2" 49 | type: CONVOLUTION 50 | bottom: "norm1" 51 | top: "conv2" 52 | convolution_param { 53 | num_output: 256 54 | pad: 2 55 | kernel_size: 5 56 | group: 2 57 | } 58 | } 59 | layers { 60 | name: "relu2" 61 | type: RELU 62 | bottom: "conv2" 63 | top: "conv2" 64 | } 65 | layers { 66 | name: "pool2" 67 | type: POOLING 68 | bottom: "conv2" 69 | top: "pool2" 70 | pooling_param { 71 | pool: MAX 72 | kernel_size: 3 73 | stride: 2 74 | } 75 | } 76 | layers { 77 | name: "norm2" 78 | type: LRN 79 | bottom: "pool2" 80 | top: "norm2" 81 | lrn_param { 82 | local_size: 5 83 | alpha: 0.0001 84 | beta: 0.75 85 | } 86 | } 87 | layers { 88 | name: "conv3" 89 | type: CONVOLUTION 90 | bottom: "norm2" 91 | top: "conv3" 92 | convolution_param { 93 | num_output: 384 94 | pad: 1 95 | kernel_size: 3 96 | } 97 | } 98 | layers { 99 | name: "relu3" 100 | type: RELU 101 | bottom: "conv3" 102 | top: "conv3" 103 | } 104 | layers { 105 | name: "conv4" 106 | type: CONVOLUTION 107 | bottom: "conv3" 108 | top: "conv4" 109 | convolution_param { 110 | num_output: 384 111 | pad: 1 112 | kernel_size: 3 113 | group: 2 114 | } 115 | } 116 | layers { 117 | name: "relu4" 118 | type: RELU 119 | bottom: "conv4" 120 | top: "conv4" 121 | } 122 | layers { 123 | name: "conv5" 124 | type: CONVOLUTION 125 | bottom: "conv4" 126 | top: "conv5" 127 | convolution_param { 128 | num_output: 256 129 | pad: 1 130 | kernel_size: 3 131 | group: 2 132 | } 133 | } 134 | layers { 135 | name: "relu5" 136 | type: RELU 137 | bottom: "conv5" 138 | top: "conv5" 139 | } 140 | layers { 141 | name: "pool5" 142 | type: POOLING 143 | bottom: "conv5" 144 | top: "pool5" 145 | pooling_param { 146 | pool: MAX 147 | kernel_size: 3 148 | stride: 2 149 | } 150 | } 151 | layers { 152 | name: "fc6" 153 | type: INNER_PRODUCT 154 | bottom: "pool5" 155 | top: "fc6" 156 | inner_product_param { 157 | num_output: 4096 158 | } 159 | } 160 | layers { 161 | name: "relu6" 162 | type: RELU 163 | bottom: "fc6" 164 | top: "fc6" 165 | } 166 | layers { 167 | name: "drop6" 168 | type: DROPOUT 169 | bottom: "fc6" 170 | top: "fc6" 171 | dropout_param { 172 | dropout_ratio: 0.5 173 | } 174 | } 175 | layers { 176 | name: "fc7" 177 | type: INNER_PRODUCT 178 | bottom: "fc6" 179 | top: "fc7" 180 | inner_product_param { 181 | num_output: 4096 182 | } 183 | } 184 | layers { 185 | name: "relu7" 186 | type: RELU 187 | bottom: "fc7" 188 | top: "fc7" 189 | } 190 | layers { 191 | name: "drop7" 192 | type: DROPOUT 193 | bottom: "fc7" 194 | top: "fc7" 195 | dropout_param { 196 | dropout_ratio: 0.5 197 | } 198 | } 199 | layers { 200 | name: "fc8" 201 | type: INNER_PRODUCT 202 | bottom: "fc7" 203 | top: "fc8" 204 | inner_product_param { 205 | num_output: 1000 206 | } 207 | } 208 | layers { 209 | name: "prob" 210 | type: SOFTMAX 211 | bottom: "fc8" 212 | top: "prob" 213 | } 214 | -------------------------------------------------------------------------------- /models/caffenet-yos/fetch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Exit on first error 4 | set -e 5 | 6 | function fetch_file() { 7 | filename="$1" 8 | url="$2" 9 | if [ -e "$filename" ]; then 10 | echo "$url: file already downloaded (remove $filename to force re-download)" 11 | else 12 | echo "$url: fetching..." 13 | wget -O "$filename" "$url" 14 | echo "$url: done." 15 | fi 16 | } 17 | 18 | function fetch_and_extract() { 19 | filename="$1" 20 | url="$2" 21 | dir="$3" 22 | example_filename="$4" 23 | example_path="$dir/$example_filename" 24 | if [ -e "$example_path" ]; then 25 | echo "$url: $example_path already exists, skipping." 26 | else 27 | fetch_file "$filename" "$url" 28 | echo "$url: extracting..." 29 | mkdir -p "$dir" 30 | tar -C "$dir" -xzf "$filename" 31 | echo "$url: done." 32 | fi 33 | } 34 | 35 | fetch_file caffenet-yos-weights http://c.yosinski.com/caffenet-yos-weights 36 | fetch_file ilsvrc_2012_mean.npy http://c.yosinski.com/ilsvrc_2012_mean.npy 37 | 38 | fetch_and_extract caffenet-yos_regularized_opt.tar.gz http://c.yosinski.com/caffenet-yos_regularized_opt.tar.gz unit_jpg_vis regularized_opt/conv1/conv1_0000_montage.jpg 39 | fetch_and_extract caffenet-yos_max_im.tar.gz http://c.yosinski.com/caffenet-yos_max_im.tar.gz unit_jpg_vis max_im/conv1/conv1_0000.jpg 40 | fetch_and_extract caffenet-yos_max_deconv.tar.gz http://c.yosinski.com/caffenet-yos_max_deconv.tar.gz unit_jpg_vis max_deconv/conv1/conv1_0000.jpg 41 | 42 | if [ "$1" = "all" ]; then 43 | fetch_and_extract caffenet-yos_regularized_opt_fc6_fc7.tar.gz http://c.yosinski.com/caffenet-yos_regularized_opt_fc6_fc7.tar.gz unit_jpg_vis regularized_opt/fc6/fc6_0000_montage.jpg 44 | fetch_and_extract caffenet-yos_max_im_fc6_fc7.tar.gz http://c.yosinski.com/caffenet-yos_max_im_fc6_fc7.tar.gz unit_jpg_vis max_im/fc6/fc6_0000.jpg 45 | fetch_and_extract caffenet-yos_max_deconv_fc6_fc7.tar.gz http://c.yosinski.com/caffenet-yos_max_deconv_fc6_fc7.tar.gz unit_jpg_vis max_deconv/fc6/fc6_0000.jpg 46 | else 47 | echo 48 | echo "Rerun as \"$0 all\" to also fetch fc6 and fc7 unit visualizations (Warning: 4.5G more)" 49 | fi 50 | -------------------------------------------------------------------------------- /models/caffenet-yos/ilsvrc_2012_labels.txt: -------------------------------------------------------------------------------- 1 | tench 2 | goldfish 3 | great white shark 4 | tiger shark 5 | hammerhead 6 | electric ray 7 | stingray 8 | cock 9 | hen 10 | ostrich 11 | brambling 12 | goldfinch 13 | house finch 14 | junco 15 | indigo bunting 16 | robin 17 | bulbul 18 | jay 19 | magpie 20 | chickadee 21 | water ouzel 22 | kite 23 | bald eagle 24 | vulture 25 | great grey owl 26 | European fire salamander 27 | common newt 28 | eft 29 | spotted salamander 30 | axolotl 31 | bullfrog 32 | tree frog 33 | tailed frog 34 | loggerhead 35 | leatherback turtle 36 | mud turtle 37 | terrapin 38 | box turtle 39 | banded gecko 40 | common iguana 41 | American chameleon 42 | whiptail 43 | agama 44 | frilled lizard 45 | alligator lizard 46 | Gila monster 47 | green lizard 48 | African chameleon 49 | Komodo dragon 50 | African crocodile 51 | American alligator 52 | triceratops 53 | thunder snake 54 | ringneck snake 55 | hognose snake 56 | green snake 57 | king snake 58 | garter snake 59 | water snake 60 | vine snake 61 | night snake 62 | boa constrictor 63 | rock python 64 | Indian cobra 65 | green mamba 66 | sea snake 67 | horned viper 68 | diamondback 69 | sidewinder 70 | trilobite 71 | harvestman 72 | scorpion 73 | black and gold garden spider 74 | barn spider 75 | garden spider 76 | black widow 77 | tarantula 78 | wolf spider 79 | tick 80 | centipede 81 | black grouse 82 | ptarmigan 83 | ruffed grouse 84 | prairie chicken 85 | peacock 86 | quail 87 | partridge 88 | African grey 89 | macaw 90 | sulphur-crested cockatoo 91 | lorikeet 92 | coucal 93 | bee eater 94 | hornbill 95 | hummingbird 96 | jacamar 97 | toucan 98 | drake 99 | red-breasted merganser 100 | goose 101 | black swan 102 | tusker 103 | echidna 104 | platypus 105 | wallaby 106 | koala 107 | wombat 108 | jellyfish 109 | sea anemone 110 | brain coral 111 | flatworm 112 | nematode 113 | conch 114 | snail 115 | slug 116 | sea slug 117 | chiton 118 | chambered nautilus 119 | Dungeness crab 120 | rock crab 121 | fiddler crab 122 | king crab 123 | American lobster 124 | spiny lobster 125 | crayfish 126 | hermit crab 127 | isopod 128 | white stork 129 | black stork 130 | spoonbill 131 | flamingo 132 | little blue heron 133 | American egret 134 | bittern 135 | crane 136 | limpkin 137 | European gallinule 138 | American coot 139 | bustard 140 | ruddy turnstone 141 | red-backed sandpiper 142 | redshank 143 | dowitcher 144 | oystercatcher 145 | pelican 146 | king penguin 147 | albatross 148 | grey whale 149 | killer whale 150 | dugong 151 | sea lion 152 | Chihuahua 153 | Japanese spaniel 154 | Maltese dog 155 | Pekinese 156 | Shih-Tzu 157 | Blenheim spaniel 158 | papillon 159 | toy terrier 160 | Rhodesian ridgeback 161 | Afghan hound 162 | basset 163 | beagle 164 | bloodhound 165 | bluetick 166 | black-and-tan coonhound 167 | Walker hound 168 | English foxhound 169 | redbone 170 | borzoi 171 | Irish wolfhound 172 | Italian greyhound 173 | whippet 174 | Ibizan hound 175 | Norwegian elkhound 176 | otterhound 177 | Saluki 178 | Scottish deerhound 179 | Weimaraner 180 | Staffordshire bullterrier 181 | American Staffordshire terrier 182 | Bedlington terrier 183 | Border terrier 184 | Kerry blue terrier 185 | Irish terrier 186 | Norfolk terrier 187 | Norwich terrier 188 | Yorkshire terrier 189 | wire-haired fox terrier 190 | Lakeland terrier 191 | Sealyham terrier 192 | Airedale 193 | cairn 194 | Australian terrier 195 | Dandie Dinmont 196 | Boston bull 197 | miniature schnauzer 198 | giant schnauzer 199 | standard schnauzer 200 | Scotch terrier 201 | Tibetan terrier 202 | silky terrier 203 | soft-coated wheaten terrier 204 | West Highland white terrier 205 | Lhasa 206 | flat-coated retriever 207 | curly-coated retriever 208 | golden retriever 209 | Labrador retriever 210 | Chesapeake Bay retriever 211 | German short-haired pointer 212 | vizsla 213 | English setter 214 | Irish setter 215 | Gordon setter 216 | Brittany spaniel 217 | clumber 218 | English springer 219 | Welsh springer spaniel 220 | cocker spaniel 221 | Sussex spaniel 222 | Irish water spaniel 223 | kuvasz 224 | schipperke 225 | groenendael 226 | malinois 227 | briard 228 | kelpie 229 | komondor 230 | Old English sheepdog 231 | Shetland sheepdog 232 | collie 233 | Border collie 234 | Bouvier des Flandres 235 | Rottweiler 236 | German shepherd 237 | Doberman 238 | miniature pinscher 239 | Greater Swiss Mountain dog 240 | Bernese mountain dog 241 | Appenzeller 242 | EntleBucher 243 | boxer 244 | bull mastiff 245 | Tibetan mastiff 246 | French bulldog 247 | Great Dane 248 | Saint Bernard 249 | Eskimo dog 250 | malamute 251 | Siberian husky 252 | dalmatian 253 | affenpinscher 254 | basenji 255 | pug 256 | Leonberg 257 | Newfoundland 258 | Great Pyrenees 259 | Samoyed 260 | Pomeranian 261 | chow 262 | keeshond 263 | Brabancon griffon 264 | Pembroke 265 | Cardigan 266 | toy poodle 267 | miniature poodle 268 | standard poodle 269 | Mexican hairless 270 | timber wolf 271 | white wolf 272 | red wolf 273 | coyote 274 | dingo 275 | dhole 276 | African hunting dog 277 | hyena 278 | red fox 279 | kit fox 280 | Arctic fox 281 | grey fox 282 | tabby 283 | tiger cat 284 | Persian cat 285 | Siamese cat 286 | Egyptian cat 287 | cougar 288 | lynx 289 | leopard 290 | snow leopard 291 | jaguar 292 | lion 293 | tiger 294 | cheetah 295 | brown bear 296 | American black bear 297 | ice bear 298 | sloth bear 299 | mongoose 300 | meerkat 301 | tiger beetle 302 | ladybug 303 | ground beetle 304 | long-horned beetle 305 | leaf beetle 306 | dung beetle 307 | rhinoceros beetle 308 | weevil 309 | fly 310 | bee 311 | ant 312 | grasshopper 313 | cricket 314 | walking stick 315 | cockroach 316 | mantis 317 | cicada 318 | leafhopper 319 | lacewing 320 | dragonfly 321 | damselfly 322 | admiral 323 | ringlet 324 | monarch 325 | cabbage butterfly 326 | sulphur butterfly 327 | lycaenid 328 | starfish 329 | sea urchin 330 | sea cucumber 331 | wood rabbit 332 | hare 333 | Angora 334 | hamster 335 | porcupine 336 | fox squirrel 337 | marmot 338 | beaver 339 | guinea pig 340 | sorrel 341 | zebra 342 | hog 343 | wild boar 344 | warthog 345 | hippopotamus 346 | ox 347 | water buffalo 348 | bison 349 | ram 350 | bighorn 351 | ibex 352 | hartebeest 353 | impala 354 | gazelle 355 | Arabian camel 356 | llama 357 | weasel 358 | mink 359 | polecat 360 | black-footed ferret 361 | otter 362 | skunk 363 | badger 364 | armadillo 365 | three-toed sloth 366 | orangutan 367 | gorilla 368 | chimpanzee 369 | gibbon 370 | siamang 371 | guenon 372 | patas 373 | baboon 374 | macaque 375 | langur 376 | colobus 377 | proboscis monkey 378 | marmoset 379 | capuchin 380 | howler monkey 381 | titi 382 | spider monkey 383 | squirrel monkey 384 | Madagascar cat 385 | indri 386 | Indian elephant 387 | African elephant 388 | lesser panda 389 | giant panda 390 | barracouta 391 | eel 392 | coho 393 | rock beauty 394 | anemone fish 395 | sturgeon 396 | gar 397 | lionfish 398 | puffer 399 | abacus 400 | abaya 401 | academic gown 402 | accordion 403 | acoustic guitar 404 | aircraft carrier 405 | airliner 406 | airship 407 | altar 408 | ambulance 409 | amphibian 410 | analog clock 411 | apiary 412 | apron 413 | ashcan 414 | assault rifle 415 | backpack 416 | bakery 417 | balance beam 418 | balloon 419 | ballpoint 420 | Band Aid 421 | banjo 422 | bannister 423 | barbell 424 | barber chair 425 | barbershop 426 | barn 427 | barometer 428 | barrel 429 | barrow 430 | baseball 431 | basketball 432 | bassinet 433 | bassoon 434 | bathing cap 435 | bath towel 436 | bathtub 437 | beach wagon 438 | beacon 439 | beaker 440 | bearskin 441 | beer bottle 442 | beer glass 443 | bell cote 444 | bib 445 | bicycle-built-for-two 446 | bikini 447 | binder 448 | binoculars 449 | birdhouse 450 | boathouse 451 | bobsled 452 | bolo tie 453 | bonnet 454 | bookcase 455 | bookshop 456 | bottlecap 457 | bow 458 | bow tie 459 | brass 460 | brassiere 461 | breakwater 462 | breastplate 463 | broom 464 | bucket 465 | buckle 466 | bulletproof vest 467 | bullet train 468 | butcher shop 469 | cab 470 | caldron 471 | candle 472 | cannon 473 | canoe 474 | can opener 475 | cardigan 476 | car mirror 477 | carousel 478 | carpenter's kit 479 | carton 480 | car wheel 481 | cash machine 482 | cassette 483 | cassette player 484 | castle 485 | catamaran 486 | CD player 487 | cello 488 | cellular telephone 489 | chain 490 | chainlink fence 491 | chain mail 492 | chain saw 493 | chest 494 | chiffonier 495 | chime 496 | china cabinet 497 | Christmas stocking 498 | church 499 | cinema 500 | cleaver 501 | cliff dwelling 502 | cloak 503 | clog 504 | cocktail shaker 505 | coffee mug 506 | coffeepot 507 | coil 508 | combination lock 509 | computer keyboard 510 | confectionery 511 | container ship 512 | convertible 513 | corkscrew 514 | cornet 515 | cowboy boot 516 | cowboy hat 517 | cradle 518 | crane 519 | crash helmet 520 | crate 521 | crib 522 | Crock Pot 523 | croquet ball 524 | crutch 525 | cuirass 526 | dam 527 | desk 528 | desktop computer 529 | dial telephone 530 | diaper 531 | digital clock 532 | digital watch 533 | dining table 534 | dishrag 535 | dishwasher 536 | disk brake 537 | dock 538 | dogsled 539 | dome 540 | doormat 541 | drilling platform 542 | drum 543 | drumstick 544 | dumbbell 545 | Dutch oven 546 | electric fan 547 | electric guitar 548 | electric locomotive 549 | entertainment center 550 | envelope 551 | espresso maker 552 | face powder 553 | feather boa 554 | file 555 | fireboat 556 | fire engine 557 | fire screen 558 | flagpole 559 | flute 560 | folding chair 561 | football helmet 562 | forklift 563 | fountain 564 | fountain pen 565 | four-poster 566 | freight car 567 | French horn 568 | frying pan 569 | fur coat 570 | garbage truck 571 | gasmask 572 | gas pump 573 | goblet 574 | go-kart 575 | golf ball 576 | golfcart 577 | gondola 578 | gong 579 | gown 580 | grand piano 581 | greenhouse 582 | grille 583 | grocery store 584 | guillotine 585 | hair slide 586 | hair spray 587 | half track 588 | hammer 589 | hamper 590 | hand blower 591 | hand-held computer 592 | handkerchief 593 | hard disc 594 | harmonica 595 | harp 596 | harvester 597 | hatchet 598 | holster 599 | home theater 600 | honeycomb 601 | hook 602 | hoopskirt 603 | horizontal bar 604 | horse cart 605 | hourglass 606 | iPod 607 | iron 608 | jack-o'-lantern 609 | jean 610 | jeep 611 | jersey 612 | jigsaw puzzle 613 | jinrikisha 614 | joystick 615 | kimono 616 | knee pad 617 | knot 618 | lab coat 619 | ladle 620 | lampshade 621 | laptop 622 | lawn mower 623 | lens cap 624 | letter opener 625 | library 626 | lifeboat 627 | lighter 628 | limousine 629 | liner 630 | lipstick 631 | Loafer 632 | lotion 633 | loudspeaker 634 | loupe 635 | lumbermill 636 | magnetic compass 637 | mailbag 638 | mailbox 639 | maillot 640 | maillot 641 | manhole cover 642 | maraca 643 | marimba 644 | mask 645 | matchstick 646 | maypole 647 | maze 648 | measuring cup 649 | medicine chest 650 | megalith 651 | microphone 652 | microwave 653 | military uniform 654 | milk can 655 | minibus 656 | miniskirt 657 | minivan 658 | missile 659 | mitten 660 | mixing bowl 661 | mobile home 662 | Model T 663 | modem 664 | monastery 665 | monitor 666 | moped 667 | mortar 668 | mortarboard 669 | mosque 670 | mosquito net 671 | motor scooter 672 | mountain bike 673 | mountain tent 674 | mouse 675 | mousetrap 676 | moving van 677 | muzzle 678 | nail 679 | neck brace 680 | necklace 681 | nipple 682 | notebook 683 | obelisk 684 | oboe 685 | ocarina 686 | odometer 687 | oil filter 688 | organ 689 | oscilloscope 690 | overskirt 691 | oxcart 692 | oxygen mask 693 | packet 694 | paddle 695 | paddlewheel 696 | padlock 697 | paintbrush 698 | pajama 699 | palace 700 | panpipe 701 | paper towel 702 | parachute 703 | parallel bars 704 | park bench 705 | parking meter 706 | passenger car 707 | patio 708 | pay-phone 709 | pedestal 710 | pencil box 711 | pencil sharpener 712 | perfume 713 | Petri dish 714 | photocopier 715 | pick 716 | pickelhaube 717 | picket fence 718 | pickup 719 | pier 720 | piggy bank 721 | pill bottle 722 | pillow 723 | ping-pong ball 724 | pinwheel 725 | pirate 726 | pitcher 727 | plane 728 | planetarium 729 | plastic bag 730 | plate rack 731 | plow 732 | plunger 733 | Polaroid camera 734 | pole 735 | police van 736 | poncho 737 | pool table 738 | pop bottle 739 | pot 740 | potter's wheel 741 | power drill 742 | prayer rug 743 | printer 744 | prison 745 | projectile 746 | projector 747 | puck 748 | punching bag 749 | purse 750 | quill 751 | quilt 752 | racer 753 | racket 754 | radiator 755 | radio 756 | radio telescope 757 | rain barrel 758 | recreational vehicle 759 | reel 760 | reflex camera 761 | refrigerator 762 | remote control 763 | restaurant 764 | revolver 765 | rifle 766 | rocking chair 767 | rotisserie 768 | rubber eraser 769 | rugby ball 770 | rule 771 | running shoe 772 | safe 773 | safety pin 774 | saltshaker 775 | sandal 776 | sarong 777 | sax 778 | scabbard 779 | scale 780 | school bus 781 | schooner 782 | scoreboard 783 | screen 784 | screw 785 | screwdriver 786 | seat belt 787 | sewing machine 788 | shield 789 | shoe shop 790 | shoji 791 | shopping basket 792 | shopping cart 793 | shovel 794 | shower cap 795 | shower curtain 796 | ski 797 | ski mask 798 | sleeping bag 799 | slide rule 800 | sliding door 801 | slot 802 | snorkel 803 | snowmobile 804 | snowplow 805 | soap dispenser 806 | soccer ball 807 | sock 808 | solar dish 809 | sombrero 810 | soup bowl 811 | space bar 812 | space heater 813 | space shuttle 814 | spatula 815 | speedboat 816 | spider web 817 | spindle 818 | sports car 819 | spotlight 820 | stage 821 | steam locomotive 822 | steel arch bridge 823 | steel drum 824 | stethoscope 825 | stole 826 | stone wall 827 | stopwatch 828 | stove 829 | strainer 830 | streetcar 831 | stretcher 832 | studio couch 833 | stupa 834 | submarine 835 | suit 836 | sundial 837 | sunglass 838 | sunglasses 839 | sunscreen 840 | suspension bridge 841 | swab 842 | sweatshirt 843 | swimming trunks 844 | swing 845 | switch 846 | syringe 847 | table lamp 848 | tank 849 | tape player 850 | teapot 851 | teddy 852 | television 853 | tennis ball 854 | thatch 855 | theater curtain 856 | thimble 857 | thresher 858 | throne 859 | tile roof 860 | toaster 861 | tobacco shop 862 | toilet seat 863 | torch 864 | totem pole 865 | tow truck 866 | toyshop 867 | tractor 868 | trailer truck 869 | tray 870 | trench coat 871 | tricycle 872 | trimaran 873 | tripod 874 | triumphal arch 875 | trolleybus 876 | trombone 877 | tub 878 | turnstile 879 | typewriter keyboard 880 | umbrella 881 | unicycle 882 | upright 883 | vacuum 884 | vase 885 | vault 886 | velvet 887 | vending machine 888 | vestment 889 | viaduct 890 | violin 891 | volleyball 892 | waffle iron 893 | wall clock 894 | wallet 895 | wardrobe 896 | warplane 897 | washbasin 898 | washer 899 | water bottle 900 | water jug 901 | water tower 902 | whiskey jug 903 | whistle 904 | wig 905 | window screen 906 | window shade 907 | Windsor tie 908 | wine bottle 909 | wing 910 | wok 911 | wooden spoon 912 | wool 913 | worm fence 914 | wreck 915 | yawl 916 | yurt 917 | web site 918 | comic book 919 | crossword puzzle 920 | street sign 921 | traffic light 922 | book jacket 923 | menu 924 | plate 925 | guacamole 926 | consomme 927 | hot pot 928 | trifle 929 | ice cream 930 | ice lolly 931 | French loaf 932 | bagel 933 | pretzel 934 | cheeseburger 935 | hotdog 936 | mashed potato 937 | head cabbage 938 | broccoli 939 | cauliflower 940 | zucchini 941 | spaghetti squash 942 | acorn squash 943 | butternut squash 944 | cucumber 945 | artichoke 946 | bell pepper 947 | cardoon 948 | mushroom 949 | Granny Smith 950 | strawberry 951 | orange 952 | lemon 953 | fig 954 | pineapple 955 | banana 956 | jackfruit 957 | custard apple 958 | pomegranate 959 | hay 960 | carbonara 961 | chocolate sauce 962 | dough 963 | meat loaf 964 | pizza 965 | potpie 966 | burrito 967 | red wine 968 | espresso 969 | cup 970 | eggnog 971 | alp 972 | bubble 973 | cliff 974 | coral reef 975 | geyser 976 | lakeside 977 | promontory 978 | sandbar 979 | seashore 980 | valley 981 | volcano 982 | ballplayer 983 | groom 984 | scuba diver 985 | rapeseed 986 | daisy 987 | yellow lady's slipper 988 | corn 989 | acorn 990 | hip 991 | buckeye 992 | coral fungus 993 | agaric 994 | gyromitra 995 | stinkhorn 996 | earthstar 997 | hen-of-the-woods 998 | bolete 999 | ear 1000 | toilet tissue 1001 | -------------------------------------------------------------------------------- /models/caffenet-yos/settings_local.template-caffenet-yos.py: -------------------------------------------------------------------------------- 1 | # Define critical settings and/or override defaults specified in 2 | # settings.py. Copy this file to settings_local.py in the same 3 | # directory as settings.py and edit. Any settings defined here 4 | # will override those defined in settings.py 5 | 6 | 7 | 8 | # Set this to point to your compiled checkout of caffe 9 | caffevis_caffe_root = '/path/to/caffe' 10 | 11 | # Load model: caffenet-yos 12 | # Path to caffe deploy prototxt file. Minibatch size should be 1. 13 | caffevis_deploy_prototxt = '%DVT_ROOT%/models/caffenet-yos/caffenet-yos-deploy.prototxt' 14 | 15 | # Path to network weights to load. 16 | caffevis_network_weights = '%DVT_ROOT%/models/caffenet-yos/caffenet-yos-weights' 17 | 18 | # Other optional settings; see complete documentation for each in settings.py. 19 | caffevis_data_mean = '%DVT_ROOT%/models/caffenet-yos/ilsvrc_2012_mean.npy' 20 | caffevis_labels = '%DVT_ROOT%/models/caffenet-yos/ilsvrc_2012_labels.txt' 21 | caffevis_label_layers = ('fc8', 'prob') 22 | caffevis_prob_layer = 'prob' 23 | caffevis_unit_jpg_dir = '%DVT_ROOT%/models/caffenet-yos/unit_jpg_vis' 24 | caffevis_jpgvis_layers = ['conv1', 'conv2', 'conv3', 'conv4', 'conv5', 'fc6', 'fc7', 'fc8', 'prob'] 25 | caffevis_jpgvis_remap = {'pool1': 'conv1', 'pool2': 'conv2', 'pool5': 'conv5'} 26 | def caffevis_layer_pretty_name_fn(name): 27 | return name.replace('pool','p').replace('norm','n') 28 | 29 | # Use GPU? Default is True. 30 | #caffevis_mode_gpu = True 31 | # Display tweaks. 32 | # Scale all window panes in UI by this factor 33 | #global_scale = 1.0 34 | # Scale all fonts by this factor 35 | #global_font_size = 1.0 36 | -------------------------------------------------------------------------------- /models/squeezenet/deploy.prototxt: -------------------------------------------------------------------------------- 1 | input: "data" 2 | force_backward: true 3 | input_shape { 4 | dim: 1 5 | dim: 3 6 | dim: 227 7 | dim: 227 8 | } 9 | layer { 10 | name: "conv1" 11 | type: "Convolution" 12 | bottom: "data" 13 | top: "conv1" 14 | convolution_param { 15 | num_output: 96 16 | kernel_size: 7 17 | stride: 2 18 | weight_filler { 19 | type: "xavier" 20 | } 21 | } 22 | } 23 | layer { 24 | name: "relu_conv1" 25 | type: "ReLU" 26 | bottom: "conv1" 27 | top: "conv1" 28 | } 29 | layer { 30 | name: "pool1" 31 | type: "Pooling" 32 | bottom: "conv1" 33 | top: "pool1" 34 | pooling_param { 35 | pool: MAX 36 | kernel_size: 3 37 | stride: 2 38 | } 39 | } 40 | layer { 41 | name: "fire2/squeeze1x1" 42 | type: "Convolution" 43 | bottom: "pool1" 44 | top: "fire2/squeeze1x1" 45 | convolution_param { 46 | num_output: 16 47 | kernel_size: 1 48 | weight_filler { 49 | type: "xavier" 50 | } 51 | } 52 | } 53 | layer { 54 | name: "fire2/relu_squeeze1x1" 55 | type: "ReLU" 56 | bottom: "fire2/squeeze1x1" 57 | top: "fire2/squeeze1x1" 58 | } 59 | layer { 60 | name: "fire2/expand1x1" 61 | type: "Convolution" 62 | bottom: "fire2/squeeze1x1" 63 | top: "fire2/expand1x1" 64 | convolution_param { 65 | num_output: 64 66 | kernel_size: 1 67 | weight_filler { 68 | type: "xavier" 69 | } 70 | } 71 | } 72 | layer { 73 | name: "fire2/relu_expand1x1" 74 | type: "ReLU" 75 | bottom: "fire2/expand1x1" 76 | top: "fire2/expand1x1" 77 | } 78 | layer { 79 | name: "fire2/expand3x3" 80 | type: "Convolution" 81 | bottom: "fire2/squeeze1x1" 82 | top: "fire2/expand3x3" 83 | convolution_param { 84 | num_output: 64 85 | pad: 1 86 | kernel_size: 3 87 | weight_filler { 88 | type: "xavier" 89 | } 90 | } 91 | } 92 | layer { 93 | name: "fire2/relu_expand3x3" 94 | type: "ReLU" 95 | bottom: "fire2/expand3x3" 96 | top: "fire2/expand3x3" 97 | } 98 | layer { 99 | name: "fire2/concat" 100 | type: "Concat" 101 | bottom: "fire2/expand1x1" 102 | bottom: "fire2/expand3x3" 103 | top: "fire2/concat" 104 | } 105 | layer { 106 | name: "fire3/squeeze1x1" 107 | type: "Convolution" 108 | bottom: "fire2/concat" 109 | top: "fire3/squeeze1x1" 110 | convolution_param { 111 | num_output: 16 112 | kernel_size: 1 113 | weight_filler { 114 | type: "xavier" 115 | } 116 | } 117 | } 118 | layer { 119 | name: "fire3/relu_squeeze1x1" 120 | type: "ReLU" 121 | bottom: "fire3/squeeze1x1" 122 | top: "fire3/squeeze1x1" 123 | } 124 | layer { 125 | name: "fire3/expand1x1" 126 | type: "Convolution" 127 | bottom: "fire3/squeeze1x1" 128 | top: "fire3/expand1x1" 129 | convolution_param { 130 | num_output: 64 131 | kernel_size: 1 132 | weight_filler { 133 | type: "xavier" 134 | } 135 | } 136 | } 137 | layer { 138 | name: "fire3/relu_expand1x1" 139 | type: "ReLU" 140 | bottom: "fire3/expand1x1" 141 | top: "fire3/expand1x1" 142 | } 143 | layer { 144 | name: "fire3/expand3x3" 145 | type: "Convolution" 146 | bottom: "fire3/squeeze1x1" 147 | top: "fire3/expand3x3" 148 | convolution_param { 149 | num_output: 64 150 | pad: 1 151 | kernel_size: 3 152 | weight_filler { 153 | type: "xavier" 154 | } 155 | } 156 | } 157 | layer { 158 | name: "fire3/relu_expand3x3" 159 | type: "ReLU" 160 | bottom: "fire3/expand3x3" 161 | top: "fire3/expand3x3" 162 | } 163 | layer { 164 | name: "fire3/concat" 165 | type: "Concat" 166 | bottom: "fire3/expand1x1" 167 | bottom: "fire3/expand3x3" 168 | top: "fire3/concat" 169 | } 170 | layer { 171 | name: "fire4/squeeze1x1" 172 | type: "Convolution" 173 | bottom: "fire3/concat" 174 | top: "fire4/squeeze1x1" 175 | convolution_param { 176 | num_output: 32 177 | kernel_size: 1 178 | weight_filler { 179 | type: "xavier" 180 | } 181 | } 182 | } 183 | layer { 184 | name: "fire4/relu_squeeze1x1" 185 | type: "ReLU" 186 | bottom: "fire4/squeeze1x1" 187 | top: "fire4/squeeze1x1" 188 | } 189 | layer { 190 | name: "fire4/expand1x1" 191 | type: "Convolution" 192 | bottom: "fire4/squeeze1x1" 193 | top: "fire4/expand1x1" 194 | convolution_param { 195 | num_output: 128 196 | kernel_size: 1 197 | weight_filler { 198 | type: "xavier" 199 | } 200 | } 201 | } 202 | layer { 203 | name: "fire4/relu_expand1x1" 204 | type: "ReLU" 205 | bottom: "fire4/expand1x1" 206 | top: "fire4/expand1x1" 207 | } 208 | layer { 209 | name: "fire4/expand3x3" 210 | type: "Convolution" 211 | bottom: "fire4/squeeze1x1" 212 | top: "fire4/expand3x3" 213 | convolution_param { 214 | num_output: 128 215 | pad: 1 216 | kernel_size: 3 217 | weight_filler { 218 | type: "xavier" 219 | } 220 | } 221 | } 222 | layer { 223 | name: "fire4/relu_expand3x3" 224 | type: "ReLU" 225 | bottom: "fire4/expand3x3" 226 | top: "fire4/expand3x3" 227 | } 228 | layer { 229 | name: "fire4/concat" 230 | type: "Concat" 231 | bottom: "fire4/expand1x1" 232 | bottom: "fire4/expand3x3" 233 | top: "fire4/concat" 234 | } 235 | layer { 236 | name: "pool4" 237 | type: "Pooling" 238 | bottom: "fire4/concat" 239 | top: "pool4" 240 | pooling_param { 241 | pool: MAX 242 | kernel_size: 3 243 | stride: 2 244 | } 245 | } 246 | layer { 247 | name: "fire5/squeeze1x1" 248 | type: "Convolution" 249 | bottom: "pool4" 250 | top: "fire5/squeeze1x1" 251 | convolution_param { 252 | num_output: 32 253 | kernel_size: 1 254 | weight_filler { 255 | type: "xavier" 256 | } 257 | } 258 | } 259 | layer { 260 | name: "fire5/relu_squeeze1x1" 261 | type: "ReLU" 262 | bottom: "fire5/squeeze1x1" 263 | top: "fire5/squeeze1x1" 264 | } 265 | layer { 266 | name: "fire5/expand1x1" 267 | type: "Convolution" 268 | bottom: "fire5/squeeze1x1" 269 | top: "fire5/expand1x1" 270 | convolution_param { 271 | num_output: 128 272 | kernel_size: 1 273 | weight_filler { 274 | type: "xavier" 275 | } 276 | } 277 | } 278 | layer { 279 | name: "fire5/relu_expand1x1" 280 | type: "ReLU" 281 | bottom: "fire5/expand1x1" 282 | top: "fire5/expand1x1" 283 | } 284 | layer { 285 | name: "fire5/expand3x3" 286 | type: "Convolution" 287 | bottom: "fire5/squeeze1x1" 288 | top: "fire5/expand3x3" 289 | convolution_param { 290 | num_output: 128 291 | pad: 1 292 | kernel_size: 3 293 | weight_filler { 294 | type: "xavier" 295 | } 296 | } 297 | } 298 | layer { 299 | name: "fire5/relu_expand3x3" 300 | type: "ReLU" 301 | bottom: "fire5/expand3x3" 302 | top: "fire5/expand3x3" 303 | } 304 | layer { 305 | name: "fire5/concat" 306 | type: "Concat" 307 | bottom: "fire5/expand1x1" 308 | bottom: "fire5/expand3x3" 309 | top: "fire5/concat" 310 | } 311 | layer { 312 | name: "fire6/squeeze1x1" 313 | type: "Convolution" 314 | bottom: "fire5/concat" 315 | top: "fire6/squeeze1x1" 316 | convolution_param { 317 | num_output: 48 318 | kernel_size: 1 319 | weight_filler { 320 | type: "xavier" 321 | } 322 | } 323 | } 324 | layer { 325 | name: "fire6/relu_squeeze1x1" 326 | type: "ReLU" 327 | bottom: "fire6/squeeze1x1" 328 | top: "fire6/squeeze1x1" 329 | } 330 | layer { 331 | name: "fire6/expand1x1" 332 | type: "Convolution" 333 | bottom: "fire6/squeeze1x1" 334 | top: "fire6/expand1x1" 335 | convolution_param { 336 | num_output: 192 337 | kernel_size: 1 338 | weight_filler { 339 | type: "xavier" 340 | } 341 | } 342 | } 343 | layer { 344 | name: "fire6/relu_expand1x1" 345 | type: "ReLU" 346 | bottom: "fire6/expand1x1" 347 | top: "fire6/expand1x1" 348 | } 349 | layer { 350 | name: "fire6/expand3x3" 351 | type: "Convolution" 352 | bottom: "fire6/squeeze1x1" 353 | top: "fire6/expand3x3" 354 | convolution_param { 355 | num_output: 192 356 | pad: 1 357 | kernel_size: 3 358 | weight_filler { 359 | type: "xavier" 360 | } 361 | } 362 | } 363 | layer { 364 | name: "fire6/relu_expand3x3" 365 | type: "ReLU" 366 | bottom: "fire6/expand3x3" 367 | top: "fire6/expand3x3" 368 | } 369 | layer { 370 | name: "fire6/concat" 371 | type: "Concat" 372 | bottom: "fire6/expand1x1" 373 | bottom: "fire6/expand3x3" 374 | top: "fire6/concat" 375 | } 376 | layer { 377 | name: "fire7/squeeze1x1" 378 | type: "Convolution" 379 | bottom: "fire6/concat" 380 | top: "fire7/squeeze1x1" 381 | convolution_param { 382 | num_output: 48 383 | kernel_size: 1 384 | weight_filler { 385 | type: "xavier" 386 | } 387 | } 388 | } 389 | layer { 390 | name: "fire7/relu_squeeze1x1" 391 | type: "ReLU" 392 | bottom: "fire7/squeeze1x1" 393 | top: "fire7/squeeze1x1" 394 | } 395 | layer { 396 | name: "fire7/expand1x1" 397 | type: "Convolution" 398 | bottom: "fire7/squeeze1x1" 399 | top: "fire7/expand1x1" 400 | convolution_param { 401 | num_output: 192 402 | kernel_size: 1 403 | weight_filler { 404 | type: "xavier" 405 | } 406 | } 407 | } 408 | layer { 409 | name: "fire7/relu_expand1x1" 410 | type: "ReLU" 411 | bottom: "fire7/expand1x1" 412 | top: "fire7/expand1x1" 413 | } 414 | layer { 415 | name: "fire7/expand3x3" 416 | type: "Convolution" 417 | bottom: "fire7/squeeze1x1" 418 | top: "fire7/expand3x3" 419 | convolution_param { 420 | num_output: 192 421 | pad: 1 422 | kernel_size: 3 423 | weight_filler { 424 | type: "xavier" 425 | } 426 | } 427 | } 428 | layer { 429 | name: "fire7/relu_expand3x3" 430 | type: "ReLU" 431 | bottom: "fire7/expand3x3" 432 | top: "fire7/expand3x3" 433 | } 434 | layer { 435 | name: "fire7/concat" 436 | type: "Concat" 437 | bottom: "fire7/expand1x1" 438 | bottom: "fire7/expand3x3" 439 | top: "fire7/concat" 440 | } 441 | layer { 442 | name: "fire8/squeeze1x1" 443 | type: "Convolution" 444 | bottom: "fire7/concat" 445 | top: "fire8/squeeze1x1" 446 | convolution_param { 447 | num_output: 64 448 | kernel_size: 1 449 | weight_filler { 450 | type: "xavier" 451 | } 452 | } 453 | } 454 | layer { 455 | name: "fire8/relu_squeeze1x1" 456 | type: "ReLU" 457 | bottom: "fire8/squeeze1x1" 458 | top: "fire8/squeeze1x1" 459 | } 460 | layer { 461 | name: "fire8/expand1x1" 462 | type: "Convolution" 463 | bottom: "fire8/squeeze1x1" 464 | top: "fire8/expand1x1" 465 | convolution_param { 466 | num_output: 256 467 | kernel_size: 1 468 | weight_filler { 469 | type: "xavier" 470 | } 471 | } 472 | } 473 | layer { 474 | name: "fire8/relu_expand1x1" 475 | type: "ReLU" 476 | bottom: "fire8/expand1x1" 477 | top: "fire8/expand1x1" 478 | } 479 | layer { 480 | name: "fire8/expand3x3" 481 | type: "Convolution" 482 | bottom: "fire8/squeeze1x1" 483 | top: "fire8/expand3x3" 484 | convolution_param { 485 | num_output: 256 486 | pad: 1 487 | kernel_size: 3 488 | weight_filler { 489 | type: "xavier" 490 | } 491 | } 492 | } 493 | layer { 494 | name: "fire8/relu_expand3x3" 495 | type: "ReLU" 496 | bottom: "fire8/expand3x3" 497 | top: "fire8/expand3x3" 498 | } 499 | layer { 500 | name: "fire8/concat" 501 | type: "Concat" 502 | bottom: "fire8/expand1x1" 503 | bottom: "fire8/expand3x3" 504 | top: "fire8/concat" 505 | } 506 | layer { 507 | name: "pool8" 508 | type: "Pooling" 509 | bottom: "fire8/concat" 510 | top: "pool8" 511 | pooling_param { 512 | pool: MAX 513 | kernel_size: 3 514 | stride: 2 515 | } 516 | } 517 | layer { 518 | name: "fire9/squeeze1x1" 519 | type: "Convolution" 520 | bottom: "pool8" 521 | top: "fire9/squeeze1x1" 522 | convolution_param { 523 | num_output: 64 524 | kernel_size: 1 525 | weight_filler { 526 | type: "xavier" 527 | } 528 | } 529 | } 530 | layer { 531 | name: "fire9/relu_squeeze1x1" 532 | type: "ReLU" 533 | bottom: "fire9/squeeze1x1" 534 | top: "fire9/squeeze1x1" 535 | } 536 | layer { 537 | name: "fire9/expand1x1" 538 | type: "Convolution" 539 | bottom: "fire9/squeeze1x1" 540 | top: "fire9/expand1x1" 541 | convolution_param { 542 | num_output: 256 543 | kernel_size: 1 544 | weight_filler { 545 | type: "xavier" 546 | } 547 | } 548 | } 549 | layer { 550 | name: "fire9/relu_expand1x1" 551 | type: "ReLU" 552 | bottom: "fire9/expand1x1" 553 | top: "fire9/expand1x1" 554 | } 555 | layer { 556 | name: "fire9/expand3x3" 557 | type: "Convolution" 558 | bottom: "fire9/squeeze1x1" 559 | top: "fire9/expand3x3" 560 | convolution_param { 561 | num_output: 256 562 | pad: 1 563 | kernel_size: 3 564 | weight_filler { 565 | type: "xavier" 566 | } 567 | } 568 | } 569 | layer { 570 | name: "fire9/relu_expand3x3" 571 | type: "ReLU" 572 | bottom: "fire9/expand3x3" 573 | top: "fire9/expand3x3" 574 | } 575 | layer { 576 | name: "fire9/concat" 577 | type: "Concat" 578 | bottom: "fire9/expand1x1" 579 | bottom: "fire9/expand3x3" 580 | top: "fire9/concat" 581 | } 582 | layer { 583 | name: "drop9" 584 | type: "Dropout" 585 | bottom: "fire9/concat" 586 | top: "fire9/concat" 587 | dropout_param { 588 | dropout_ratio: 0.5 589 | } 590 | } 591 | layer { 592 | name: "conv10" 593 | type: "Convolution" 594 | bottom: "fire9/concat" 595 | top: "conv10" 596 | convolution_param { 597 | num_output: 1000 598 | pad: 1 599 | kernel_size: 1 600 | weight_filler { 601 | type: "gaussian" 602 | mean: 0.0 603 | std: 0.01 604 | } 605 | } 606 | } 607 | layer { 608 | name: "relu_conv10" 609 | type: "ReLU" 610 | bottom: "conv10" 611 | top: "conv10" 612 | } 613 | layer { 614 | name: "pool10" 615 | type: "Pooling" 616 | bottom: "conv10" 617 | top: "pool10" 618 | pooling_param { 619 | pool: AVE 620 | global_pooling: true 621 | } 622 | } 623 | layer { 624 | name: "prob" 625 | type: "Softmax" 626 | bottom: "pool10" 627 | top: "prob" 628 | } 629 | -------------------------------------------------------------------------------- /models/squeezenet/fetch.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Exit on first error 4 | set -e 5 | 6 | function fetch_file() { 7 | filename="$1" 8 | url="$2" 9 | if [ -e "$filename" ]; then 10 | echo "$url: file already downloaded (remove $filename to force re-download)" 11 | else 12 | echo "$url: fetching..." 13 | wget -O "$filename" "$url" 14 | echo "$url: done." 15 | fi 16 | } 17 | 18 | function fetch_and_extract() { 19 | filename="$1" 20 | url="$2" 21 | dir="$3" 22 | example_filename="$4" 23 | example_path="$dir/$example_filename" 24 | if [ -e "$example_path" ]; then 25 | echo "$url: $example_path already exists, skipping." 26 | else 27 | fetch_file "$filename" "$url" 28 | echo "$url: extracting..." 29 | mkdir -p "$dir" 30 | tar -C "$dir" -xzf "$filename" 31 | echo "$url: done." 32 | fi 33 | } 34 | 35 | fetch_file squeezenet_v1.0.caffemodel https://github.com/DeepScale/SqueezeNet/raw/master/SqueezeNet_v1.0/squeezenet_v1.0.caffemodel 36 | 37 | -------------------------------------------------------------------------------- /models/squeezenet/ilsvrc_2012_labels.txt: -------------------------------------------------------------------------------- 1 | tench 2 | goldfish 3 | great white shark 4 | tiger shark 5 | hammerhead 6 | electric ray 7 | stingray 8 | cock 9 | hen 10 | ostrich 11 | brambling 12 | goldfinch 13 | house finch 14 | junco 15 | indigo bunting 16 | robin 17 | bulbul 18 | jay 19 | magpie 20 | chickadee 21 | water ouzel 22 | kite 23 | bald eagle 24 | vulture 25 | great grey owl 26 | European fire salamander 27 | common newt 28 | eft 29 | spotted salamander 30 | axolotl 31 | bullfrog 32 | tree frog 33 | tailed frog 34 | loggerhead 35 | leatherback turtle 36 | mud turtle 37 | terrapin 38 | box turtle 39 | banded gecko 40 | common iguana 41 | American chameleon 42 | whiptail 43 | agama 44 | frilled lizard 45 | alligator lizard 46 | Gila monster 47 | green lizard 48 | African chameleon 49 | Komodo dragon 50 | African crocodile 51 | American alligator 52 | triceratops 53 | thunder snake 54 | ringneck snake 55 | hognose snake 56 | green snake 57 | king snake 58 | garter snake 59 | water snake 60 | vine snake 61 | night snake 62 | boa constrictor 63 | rock python 64 | Indian cobra 65 | green mamba 66 | sea snake 67 | horned viper 68 | diamondback 69 | sidewinder 70 | trilobite 71 | harvestman 72 | scorpion 73 | black and gold garden spider 74 | barn spider 75 | garden spider 76 | black widow 77 | tarantula 78 | wolf spider 79 | tick 80 | centipede 81 | black grouse 82 | ptarmigan 83 | ruffed grouse 84 | prairie chicken 85 | peacock 86 | quail 87 | partridge 88 | African grey 89 | macaw 90 | sulphur-crested cockatoo 91 | lorikeet 92 | coucal 93 | bee eater 94 | hornbill 95 | hummingbird 96 | jacamar 97 | toucan 98 | drake 99 | red-breasted merganser 100 | goose 101 | black swan 102 | tusker 103 | echidna 104 | platypus 105 | wallaby 106 | koala 107 | wombat 108 | jellyfish 109 | sea anemone 110 | brain coral 111 | flatworm 112 | nematode 113 | conch 114 | snail 115 | slug 116 | sea slug 117 | chiton 118 | chambered nautilus 119 | Dungeness crab 120 | rock crab 121 | fiddler crab 122 | king crab 123 | American lobster 124 | spiny lobster 125 | crayfish 126 | hermit crab 127 | isopod 128 | white stork 129 | black stork 130 | spoonbill 131 | flamingo 132 | little blue heron 133 | American egret 134 | bittern 135 | crane 136 | limpkin 137 | European gallinule 138 | American coot 139 | bustard 140 | ruddy turnstone 141 | red-backed sandpiper 142 | redshank 143 | dowitcher 144 | oystercatcher 145 | pelican 146 | king penguin 147 | albatross 148 | grey whale 149 | killer whale 150 | dugong 151 | sea lion 152 | Chihuahua 153 | Japanese spaniel 154 | Maltese dog 155 | Pekinese 156 | Shih-Tzu 157 | Blenheim spaniel 158 | papillon 159 | toy terrier 160 | Rhodesian ridgeback 161 | Afghan hound 162 | basset 163 | beagle 164 | bloodhound 165 | bluetick 166 | black-and-tan coonhound 167 | Walker hound 168 | English foxhound 169 | redbone 170 | borzoi 171 | Irish wolfhound 172 | Italian greyhound 173 | whippet 174 | Ibizan hound 175 | Norwegian elkhound 176 | otterhound 177 | Saluki 178 | Scottish deerhound 179 | Weimaraner 180 | Staffordshire bullterrier 181 | American Staffordshire terrier 182 | Bedlington terrier 183 | Border terrier 184 | Kerry blue terrier 185 | Irish terrier 186 | Norfolk terrier 187 | Norwich terrier 188 | Yorkshire terrier 189 | wire-haired fox terrier 190 | Lakeland terrier 191 | Sealyham terrier 192 | Airedale 193 | cairn 194 | Australian terrier 195 | Dandie Dinmont 196 | Boston bull 197 | miniature schnauzer 198 | giant schnauzer 199 | standard schnauzer 200 | Scotch terrier 201 | Tibetan terrier 202 | silky terrier 203 | soft-coated wheaten terrier 204 | West Highland white terrier 205 | Lhasa 206 | flat-coated retriever 207 | curly-coated retriever 208 | golden retriever 209 | Labrador retriever 210 | Chesapeake Bay retriever 211 | German short-haired pointer 212 | vizsla 213 | English setter 214 | Irish setter 215 | Gordon setter 216 | Brittany spaniel 217 | clumber 218 | English springer 219 | Welsh springer spaniel 220 | cocker spaniel 221 | Sussex spaniel 222 | Irish water spaniel 223 | kuvasz 224 | schipperke 225 | groenendael 226 | malinois 227 | briard 228 | kelpie 229 | komondor 230 | Old English sheepdog 231 | Shetland sheepdog 232 | collie 233 | Border collie 234 | Bouvier des Flandres 235 | Rottweiler 236 | German shepherd 237 | Doberman 238 | miniature pinscher 239 | Greater Swiss Mountain dog 240 | Bernese mountain dog 241 | Appenzeller 242 | EntleBucher 243 | boxer 244 | bull mastiff 245 | Tibetan mastiff 246 | French bulldog 247 | Great Dane 248 | Saint Bernard 249 | Eskimo dog 250 | malamute 251 | Siberian husky 252 | dalmatian 253 | affenpinscher 254 | basenji 255 | pug 256 | Leonberg 257 | Newfoundland 258 | Great Pyrenees 259 | Samoyed 260 | Pomeranian 261 | chow 262 | keeshond 263 | Brabancon griffon 264 | Pembroke 265 | Cardigan 266 | toy poodle 267 | miniature poodle 268 | standard poodle 269 | Mexican hairless 270 | timber wolf 271 | white wolf 272 | red wolf 273 | coyote 274 | dingo 275 | dhole 276 | African hunting dog 277 | hyena 278 | red fox 279 | kit fox 280 | Arctic fox 281 | grey fox 282 | tabby 283 | tiger cat 284 | Persian cat 285 | Siamese cat 286 | Egyptian cat 287 | cougar 288 | lynx 289 | leopard 290 | snow leopard 291 | jaguar 292 | lion 293 | tiger 294 | cheetah 295 | brown bear 296 | American black bear 297 | ice bear 298 | sloth bear 299 | mongoose 300 | meerkat 301 | tiger beetle 302 | ladybug 303 | ground beetle 304 | long-horned beetle 305 | leaf beetle 306 | dung beetle 307 | rhinoceros beetle 308 | weevil 309 | fly 310 | bee 311 | ant 312 | grasshopper 313 | cricket 314 | walking stick 315 | cockroach 316 | mantis 317 | cicada 318 | leafhopper 319 | lacewing 320 | dragonfly 321 | damselfly 322 | admiral 323 | ringlet 324 | monarch 325 | cabbage butterfly 326 | sulphur butterfly 327 | lycaenid 328 | starfish 329 | sea urchin 330 | sea cucumber 331 | wood rabbit 332 | hare 333 | Angora 334 | hamster 335 | porcupine 336 | fox squirrel 337 | marmot 338 | beaver 339 | guinea pig 340 | sorrel 341 | zebra 342 | hog 343 | wild boar 344 | warthog 345 | hippopotamus 346 | ox 347 | water buffalo 348 | bison 349 | ram 350 | bighorn 351 | ibex 352 | hartebeest 353 | impala 354 | gazelle 355 | Arabian camel 356 | llama 357 | weasel 358 | mink 359 | polecat 360 | black-footed ferret 361 | otter 362 | skunk 363 | badger 364 | armadillo 365 | three-toed sloth 366 | orangutan 367 | gorilla 368 | chimpanzee 369 | gibbon 370 | siamang 371 | guenon 372 | patas 373 | baboon 374 | macaque 375 | langur 376 | colobus 377 | proboscis monkey 378 | marmoset 379 | capuchin 380 | howler monkey 381 | titi 382 | spider monkey 383 | squirrel monkey 384 | Madagascar cat 385 | indri 386 | Indian elephant 387 | African elephant 388 | lesser panda 389 | giant panda 390 | barracouta 391 | eel 392 | coho 393 | rock beauty 394 | anemone fish 395 | sturgeon 396 | gar 397 | lionfish 398 | puffer 399 | abacus 400 | abaya 401 | academic gown 402 | accordion 403 | acoustic guitar 404 | aircraft carrier 405 | airliner 406 | airship 407 | altar 408 | ambulance 409 | amphibian 410 | analog clock 411 | apiary 412 | apron 413 | ashcan 414 | assault rifle 415 | backpack 416 | bakery 417 | balance beam 418 | balloon 419 | ballpoint 420 | Band Aid 421 | banjo 422 | bannister 423 | barbell 424 | barber chair 425 | barbershop 426 | barn 427 | barometer 428 | barrel 429 | barrow 430 | baseball 431 | basketball 432 | bassinet 433 | bassoon 434 | bathing cap 435 | bath towel 436 | bathtub 437 | beach wagon 438 | beacon 439 | beaker 440 | bearskin 441 | beer bottle 442 | beer glass 443 | bell cote 444 | bib 445 | bicycle-built-for-two 446 | bikini 447 | binder 448 | binoculars 449 | birdhouse 450 | boathouse 451 | bobsled 452 | bolo tie 453 | bonnet 454 | bookcase 455 | bookshop 456 | bottlecap 457 | bow 458 | bow tie 459 | brass 460 | brassiere 461 | breakwater 462 | breastplate 463 | broom 464 | bucket 465 | buckle 466 | bulletproof vest 467 | bullet train 468 | butcher shop 469 | cab 470 | caldron 471 | candle 472 | cannon 473 | canoe 474 | can opener 475 | cardigan 476 | car mirror 477 | carousel 478 | carpenter's kit 479 | carton 480 | car wheel 481 | cash machine 482 | cassette 483 | cassette player 484 | castle 485 | catamaran 486 | CD player 487 | cello 488 | cellular telephone 489 | chain 490 | chainlink fence 491 | chain mail 492 | chain saw 493 | chest 494 | chiffonier 495 | chime 496 | china cabinet 497 | Christmas stocking 498 | church 499 | cinema 500 | cleaver 501 | cliff dwelling 502 | cloak 503 | clog 504 | cocktail shaker 505 | coffee mug 506 | coffeepot 507 | coil 508 | combination lock 509 | computer keyboard 510 | confectionery 511 | container ship 512 | convertible 513 | corkscrew 514 | cornet 515 | cowboy boot 516 | cowboy hat 517 | cradle 518 | crane 519 | crash helmet 520 | crate 521 | crib 522 | Crock Pot 523 | croquet ball 524 | crutch 525 | cuirass 526 | dam 527 | desk 528 | desktop computer 529 | dial telephone 530 | diaper 531 | digital clock 532 | digital watch 533 | dining table 534 | dishrag 535 | dishwasher 536 | disk brake 537 | dock 538 | dogsled 539 | dome 540 | doormat 541 | drilling platform 542 | drum 543 | drumstick 544 | dumbbell 545 | Dutch oven 546 | electric fan 547 | electric guitar 548 | electric locomotive 549 | entertainment center 550 | envelope 551 | espresso maker 552 | face powder 553 | feather boa 554 | file 555 | fireboat 556 | fire engine 557 | fire screen 558 | flagpole 559 | flute 560 | folding chair 561 | football helmet 562 | forklift 563 | fountain 564 | fountain pen 565 | four-poster 566 | freight car 567 | French horn 568 | frying pan 569 | fur coat 570 | garbage truck 571 | gasmask 572 | gas pump 573 | goblet 574 | go-kart 575 | golf ball 576 | golfcart 577 | gondola 578 | gong 579 | gown 580 | grand piano 581 | greenhouse 582 | grille 583 | grocery store 584 | guillotine 585 | hair slide 586 | hair spray 587 | half track 588 | hammer 589 | hamper 590 | hand blower 591 | hand-held computer 592 | handkerchief 593 | hard disc 594 | harmonica 595 | harp 596 | harvester 597 | hatchet 598 | holster 599 | home theater 600 | honeycomb 601 | hook 602 | hoopskirt 603 | horizontal bar 604 | horse cart 605 | hourglass 606 | iPod 607 | iron 608 | jack-o'-lantern 609 | jean 610 | jeep 611 | jersey 612 | jigsaw puzzle 613 | jinrikisha 614 | joystick 615 | kimono 616 | knee pad 617 | knot 618 | lab coat 619 | ladle 620 | lampshade 621 | laptop 622 | lawn mower 623 | lens cap 624 | letter opener 625 | library 626 | lifeboat 627 | lighter 628 | limousine 629 | liner 630 | lipstick 631 | Loafer 632 | lotion 633 | loudspeaker 634 | loupe 635 | lumbermill 636 | magnetic compass 637 | mailbag 638 | mailbox 639 | maillot 640 | maillot 641 | manhole cover 642 | maraca 643 | marimba 644 | mask 645 | matchstick 646 | maypole 647 | maze 648 | measuring cup 649 | medicine chest 650 | megalith 651 | microphone 652 | microwave 653 | military uniform 654 | milk can 655 | minibus 656 | miniskirt 657 | minivan 658 | missile 659 | mitten 660 | mixing bowl 661 | mobile home 662 | Model T 663 | modem 664 | monastery 665 | monitor 666 | moped 667 | mortar 668 | mortarboard 669 | mosque 670 | mosquito net 671 | motor scooter 672 | mountain bike 673 | mountain tent 674 | mouse 675 | mousetrap 676 | moving van 677 | muzzle 678 | nail 679 | neck brace 680 | necklace 681 | nipple 682 | notebook 683 | obelisk 684 | oboe 685 | ocarina 686 | odometer 687 | oil filter 688 | organ 689 | oscilloscope 690 | overskirt 691 | oxcart 692 | oxygen mask 693 | packet 694 | paddle 695 | paddlewheel 696 | padlock 697 | paintbrush 698 | pajama 699 | palace 700 | panpipe 701 | paper towel 702 | parachute 703 | parallel bars 704 | park bench 705 | parking meter 706 | passenger car 707 | patio 708 | pay-phone 709 | pedestal 710 | pencil box 711 | pencil sharpener 712 | perfume 713 | Petri dish 714 | photocopier 715 | pick 716 | pickelhaube 717 | picket fence 718 | pickup 719 | pier 720 | piggy bank 721 | pill bottle 722 | pillow 723 | ping-pong ball 724 | pinwheel 725 | pirate 726 | pitcher 727 | plane 728 | planetarium 729 | plastic bag 730 | plate rack 731 | plow 732 | plunger 733 | Polaroid camera 734 | pole 735 | police van 736 | poncho 737 | pool table 738 | pop bottle 739 | pot 740 | potter's wheel 741 | power drill 742 | prayer rug 743 | printer 744 | prison 745 | projectile 746 | projector 747 | puck 748 | punching bag 749 | purse 750 | quill 751 | quilt 752 | racer 753 | racket 754 | radiator 755 | radio 756 | radio telescope 757 | rain barrel 758 | recreational vehicle 759 | reel 760 | reflex camera 761 | refrigerator 762 | remote control 763 | restaurant 764 | revolver 765 | rifle 766 | rocking chair 767 | rotisserie 768 | rubber eraser 769 | rugby ball 770 | rule 771 | running shoe 772 | safe 773 | safety pin 774 | saltshaker 775 | sandal 776 | sarong 777 | sax 778 | scabbard 779 | scale 780 | school bus 781 | schooner 782 | scoreboard 783 | screen 784 | screw 785 | screwdriver 786 | seat belt 787 | sewing machine 788 | shield 789 | shoe shop 790 | shoji 791 | shopping basket 792 | shopping cart 793 | shovel 794 | shower cap 795 | shower curtain 796 | ski 797 | ski mask 798 | sleeping bag 799 | slide rule 800 | sliding door 801 | slot 802 | snorkel 803 | snowmobile 804 | snowplow 805 | soap dispenser 806 | soccer ball 807 | sock 808 | solar dish 809 | sombrero 810 | soup bowl 811 | space bar 812 | space heater 813 | space shuttle 814 | spatula 815 | speedboat 816 | spider web 817 | spindle 818 | sports car 819 | spotlight 820 | stage 821 | steam locomotive 822 | steel arch bridge 823 | steel drum 824 | stethoscope 825 | stole 826 | stone wall 827 | stopwatch 828 | stove 829 | strainer 830 | streetcar 831 | stretcher 832 | studio couch 833 | stupa 834 | submarine 835 | suit 836 | sundial 837 | sunglass 838 | sunglasses 839 | sunscreen 840 | suspension bridge 841 | swab 842 | sweatshirt 843 | swimming trunks 844 | swing 845 | switch 846 | syringe 847 | table lamp 848 | tank 849 | tape player 850 | teapot 851 | teddy 852 | television 853 | tennis ball 854 | thatch 855 | theater curtain 856 | thimble 857 | thresher 858 | throne 859 | tile roof 860 | toaster 861 | tobacco shop 862 | toilet seat 863 | torch 864 | totem pole 865 | tow truck 866 | toyshop 867 | tractor 868 | trailer truck 869 | tray 870 | trench coat 871 | tricycle 872 | trimaran 873 | tripod 874 | triumphal arch 875 | trolleybus 876 | trombone 877 | tub 878 | turnstile 879 | typewriter keyboard 880 | umbrella 881 | unicycle 882 | upright 883 | vacuum 884 | vase 885 | vault 886 | velvet 887 | vending machine 888 | vestment 889 | viaduct 890 | violin 891 | volleyball 892 | waffle iron 893 | wall clock 894 | wallet 895 | wardrobe 896 | warplane 897 | washbasin 898 | washer 899 | water bottle 900 | water jug 901 | water tower 902 | whiskey jug 903 | whistle 904 | wig 905 | window screen 906 | window shade 907 | Windsor tie 908 | wine bottle 909 | wing 910 | wok 911 | wooden spoon 912 | wool 913 | worm fence 914 | wreck 915 | yawl 916 | yurt 917 | web site 918 | comic book 919 | crossword puzzle 920 | street sign 921 | traffic light 922 | book jacket 923 | menu 924 | plate 925 | guacamole 926 | consomme 927 | hot pot 928 | trifle 929 | ice cream 930 | ice lolly 931 | French loaf 932 | bagel 933 | pretzel 934 | cheeseburger 935 | hotdog 936 | mashed potato 937 | head cabbage 938 | broccoli 939 | cauliflower 940 | zucchini 941 | spaghetti squash 942 | acorn squash 943 | butternut squash 944 | cucumber 945 | artichoke 946 | bell pepper 947 | cardoon 948 | mushroom 949 | Granny Smith 950 | strawberry 951 | orange 952 | lemon 953 | fig 954 | pineapple 955 | banana 956 | jackfruit 957 | custard apple 958 | pomegranate 959 | hay 960 | carbonara 961 | chocolate sauce 962 | dough 963 | meat loaf 964 | pizza 965 | potpie 966 | burrito 967 | red wine 968 | espresso 969 | cup 970 | eggnog 971 | alp 972 | bubble 973 | cliff 974 | coral reef 975 | geyser 976 | lakeside 977 | promontory 978 | sandbar 979 | seashore 980 | valley 981 | volcano 982 | ballplayer 983 | groom 984 | scuba diver 985 | rapeseed 986 | daisy 987 | yellow lady's slipper 988 | corn 989 | acorn 990 | hip 991 | buckeye 992 | coral fungus 993 | agaric 994 | gyromitra 995 | stinkhorn 996 | earthstar 997 | hen-of-the-woods 998 | bolete 999 | ear 1000 | toilet tissue 1001 | -------------------------------------------------------------------------------- /models/squeezenet/settings_local.template-squeezenet.py: -------------------------------------------------------------------------------- 1 | # Define critical settings and/or override defaults specified in 2 | # settings.py. Copy this file to settings_local.py in the same 3 | # directory as settings.py and edit. Any settings defined here 4 | # will override those defined in settings.py 5 | 6 | 7 | 8 | # Set this to point to your compiled checkout of caffe 9 | caffevis_caffe_root = '/path/to/caffe' 10 | 11 | # Load model: squeezenet 12 | # Path to caffe deploy prototxt file. Minibatch size should be 1. 13 | caffevis_deploy_prototxt = '%DVT_ROOT%/models/squeezenet/deploy.prototxt' 14 | 15 | # Path to network weights to load. 16 | caffevis_network_weights = '%DVT_ROOT%/models/squeezenet/squeezenet_v1.0.caffemodel' 17 | 18 | 19 | 20 | # Other optional settings; see complete documentation for each in settings.py. 21 | caffevis_data_mean = (104, 117, 123) # per-channel mean 22 | caffevis_labels = '%DVT_ROOT%/models/squeezenet/ilsvrc_2012_labels.txt' 23 | caffevis_jpgvis_layers = [] 24 | caffevis_prob_layer = 'prob' 25 | caffevis_label_layers = ('conv10', 'relu_conv10', 'pool10', 'prob') 26 | def caffevis_layer_pretty_name_fn(name): 27 | name = name.replace('fire','f').replace('relu_expand','re').replace('expand','e').replace('concat','c').replace('squeeze','s') 28 | name = name.replace('1x1_','').replace('1x1','') 29 | return name 30 | # Don't display duplicate *_split_* layers 31 | caffevis_filter_layers = lambda name: '_split_' in name 32 | 33 | # Window panes for squeezenet (no caffevis_jpgvis pane, larger control pane to fit many layer names). 34 | _control_height = 45 35 | window_panes = ( 36 | # (i, j, i_size, j_size) 37 | ('input', ( 0, 0, 300, 300)), 38 | ('caffevis_aux', (300, 0, 300, 300)), 39 | ('caffevis_back', (600, 0, 300, 300)), 40 | ('caffevis_status', (900, 0, 30, 1500)), 41 | ('caffevis_control', ( 0, 300, _control_height, 1200)), 42 | ('caffevis_layers', ( _control_height, 300, 900 - _control_height, 1200)), 43 | ) 44 | caffevis_layers_aspect_ratio = float(window_panes[-1][1][3])/window_panes[-1][1][2] # Actual ratio from caffevis_layers 45 | 46 | # Use GPU? Default is True. 47 | #caffevis_mode_gpu = True 48 | # Display tweaks. 49 | # Scale all window panes in UI by this factor 50 | #global_scale = 1.0 51 | # Scale all fonts by this factor 52 | #global_font_size = 1.0 53 | 54 | # Wider spacing 55 | caffevis_control_line_spacing = 10 56 | 57 | -------------------------------------------------------------------------------- /numpy_cache.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from collections import OrderedDict 3 | from threading import RLock 4 | 5 | class FIFOLimitedArrayCache(object): 6 | '''Threadsafe cache that stores numpy arrays (or any other object that 7 | defines obj.nbytes) and limits total memory. Items are ejected, if 8 | necessary, in the same order in which they were added. 9 | ''' 10 | 11 | def __init__(self, max_bytes = 1e7): 12 | self._store = OrderedDict() 13 | self._store_bytes = 0 14 | self._max_bytes = max_bytes 15 | self._lock = RLock() 16 | 17 | def get(self, key, default = None): 18 | with self._lock: 19 | if key in self._store: 20 | return self._store[key] 21 | else: 22 | return default 23 | 24 | def set(self, key, val): 25 | with self._lock: 26 | if key in self._store: 27 | self._store_bytes -= self._store[key].nbytes 28 | self._store[key] = val 29 | self._store_bytes += self._store[key].nbytes 30 | self._trim() 31 | 32 | def _trim(self): 33 | while len(self._store) > 0 and self._store_bytes > self._max_bytes: 34 | key,val = self._store.popitem(last = False) 35 | self._store_bytes -= val.nbytes 36 | 37 | def delete(self, key, raise_if_missing = False): 38 | with self._lock: 39 | if key in self._store: 40 | self._store_bytes -= val.nbytes 41 | del self._store[key] 42 | elif raise_if_missing: 43 | raise Exception('key %s not found in cache' % repr(key)) 44 | 45 | def get_size(self): 46 | return self._store_bytes 47 | 48 | def __str__(self): 49 | with self._lock: 50 | return 'FIFOLimitedArrayCache<%d items, bytes used/max %g/%g >' % (len(self._store), self._store_bytes, self._max_bytes) 51 | -------------------------------------------------------------------------------- /optimize/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/yosinski/deep-visualization-toolbox/4e739ae9045fb20a9c8a7a4cf57590139f6b4585/optimize/__init__.py -------------------------------------------------------------------------------- /optimize_image.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import sys 5 | import argparse 6 | import numpy as np 7 | 8 | import settings 9 | from optimize.gradient_optimizer import GradientOptimizer, FindParams 10 | from caffevis.caffevis_helper import check_force_backward_true, read_label_file 11 | 12 | LR_POLICY_CHOICES = ('constant', 'progress', 'progress01') 13 | 14 | 15 | 16 | def get_parser(): 17 | parser = argparse.ArgumentParser(description='Script to find, with or without regularization, images that cause high or low activations of specific neurons in a network via numerical optimization. Settings are read from settings.py, overridden in settings_local.py, and may be further overridden on the command line.', 18 | formatter_class=lambda prog: argparse.ArgumentDefaultsHelpFormatter(prog, width=100) 19 | ) 20 | 21 | # Network and data options 22 | parser.add_argument('--caffe-root', type = str, default = settings.caffevis_caffe_root, 23 | help = 'Path to caffe root directory.') 24 | parser.add_argument('--deploy-proto', type = str, default = settings.caffevis_deploy_prototxt, 25 | help = 'Path to caffe network prototxt.') 26 | parser.add_argument('--net-weights', type = str, default = settings.caffevis_network_weights, 27 | help = 'Path to caffe network weights.') 28 | parser.add_argument('--mean', type = str, default = repr(settings.caffevis_data_mean), 29 | help = '''Mean. The mean may be None, a tuple of one mean value per channel, or a string specifying the path to a mean image to load. Because of the multiple datatypes supported, this argument must be specified as a string that evaluates to a valid Python object. For example: "None", "(10,20,30)", and "'mean.npy'" are all valid values. Note that to specify a string path to a mean file, it must be passed with quotes, which usually entails passing it with double quotes in the shell! Alternately, just provide the mean in settings_local.py.''') 30 | parser.add_argument('--channel-swap-to-rgb', type = str, default = '(2,1,0)', 31 | help = 'Permutation to apply to channels to change to RGB space for plotting. Hint: (0,1,2) if your network is trained for RGB, (2,1,0) if it is trained for BGR.') 32 | parser.add_argument('--data-size', type = str, default = '(227,227)', 33 | help = 'Size of network input.') 34 | 35 | #### FindParams 36 | 37 | # Where to start 38 | parser.add_argument('--start-at', type = str, default = 'mean_plus_rand', choices = ('mean_plus_rand', 'randu', 'mean'), 39 | help = 'How to generate x0, the initial point used in optimization.') 40 | parser.add_argument('--rand-seed', type = int, default = 0, 41 | help = 'Random seed used for generating the start-at image (use different seeds to generate different images).') 42 | 43 | # What to optimize 44 | parser.add_argument('--push-layer', type = str, default = 'fc8', 45 | help = 'Name of layer that contains the desired neuron whose value is optimized.') 46 | parser.add_argument('--push-channel', type = int, default = '130', 47 | help = 'Channel number for desired neuron whose value is optimized (channel for conv, neuron index for FC).') 48 | parser.add_argument('--push-spatial', type = str, default = 'None', 49 | help = 'Which spatial location to push for conv layers. For FC layers, set this to None. For conv layers, set it to a tuple, e.g. when using `--push-layer conv5` on AlexNet, --push-spatial (6,6) will maximize the center unit of the 13x13 spatial grid.') 50 | parser.add_argument('--push-dir', type = float, default = 1, 51 | help = 'Which direction to push the activation of the selected neuron, that is, the value used to begin backprop. For example, use 1 to maximize the selected neuron activation and -1 to minimize it.') 52 | 53 | # Use regularization? 54 | parser.add_argument('--decay', type = float, default = 0, 55 | help = 'Amount of L2 decay to use.') 56 | parser.add_argument('--blur-radius', type = float, default = 0, 57 | help = 'Radius in pixels of blur to apply after each BLUR_EVERY steps. If 0, perform no blurring. Blur sizes between 0 and 0.3 work poorly.') 58 | parser.add_argument('--blur-every', type = int, default = 0, 59 | help = 'Blur every BLUR_EVERY steps. If 0, perform no blurring.') 60 | parser.add_argument('--small-val-percentile', type = float, default = 0, 61 | help = 'Induce sparsity by setting pixels with absolute value under SMALL_VAL_PERCENTILE percentile to 0. Not discussed in paper. 0 to disable.') 62 | parser.add_argument('--small-norm-percentile', type = float, default = 0, 63 | help = 'Induce sparsity by setting pixels with norm under SMALL_NORM_PERCENTILE percentile to 0. \\theta_{n_pct} from the paper. 0 to disable.') 64 | parser.add_argument('--px-benefit-percentile', type = float, default = 0, 65 | help = 'Induce sparsity by setting pixels with contribution under PX_BENEFIT_PERCENTILE percentile to 0. Mentioned briefly in paper but not used. 0 to disable.') 66 | parser.add_argument('--px-abs-benefit-percentile', type = float, default = 0, 67 | help = 'Induce sparsity by setting pixels with contribution under PX_BENEFIT_PERCENTILE percentile to 0. \\theta_{c_pct} from the paper. 0 to disable.') 68 | 69 | # How much to optimize 70 | parser.add_argument('--lr-policy', type = str, default = 'constant', choices = LR_POLICY_CHOICES, 71 | help = 'Learning rate policy. See description in lr-params.') 72 | parser.add_argument('--lr-params', type = str, default = '{"lr": 1}', 73 | help = 'Learning rate params, specified as a string that evalutes to a Python dict. Params that must be provided dependon which lr-policy is selected. The "constant" policy requires the "lr" key and uses the constant given learning rate. The "progress" policy requires the "max_lr" and "desired_prog" keys and scales the learning rate such that the objective function will change by an amount equal to DESIRED_PROG under a linear objective assumption, except the LR is limited to MAX_LR. The "progress01" policy requires the "max_lr", "early_prog", and "late_prog_mult" keys and is tuned for optimizing neurons with outputs in the [0,1] range, e.g. neurons on a softmax layer. Under this policy optimization slows down as the output approaches 1 (see code for details).') 74 | parser.add_argument('--max-iter', type = int, default = 500, 75 | help = 'Number of iterations of the optimization loop.') 76 | 77 | # Where to save results 78 | parser.add_argument('--output-prefix', type = str, default = 'optimize_results/opt', 79 | help = 'Output path and filename prefix (default: optimize_results/opt)') 80 | parser.add_argument('--output-template', type = str, default = '%(p.push_layer)s_%(p.push_channel)04d_%(p.rand_seed)d', 81 | help = 'Output filename template; see code for details (default: "%%(p.push_layer)s_%%(p.push_channel)04d_%%(p.rand_seed)d"). ' 82 | 'The default output-prefix and output-template produce filenames like "optimize_results/opt_prob_0278_0_best_X.jpg"') 83 | parser.add_argument('--brave', action = 'store_true', help = 'Allow overwriting existing results files. Default: off, i.e. cowardly refuse to overwrite existing files.') 84 | parser.add_argument('--skipbig', action = 'store_true', help = 'Skip outputting large *info_big.pkl files (contains pickled version of x0, last x, best x, first x that attained max on the specified layer.') 85 | 86 | return parser 87 | 88 | 89 | 90 | def parse_and_validate_lr_params(parser, lr_policy, lr_params): 91 | assert lr_policy in LR_POLICY_CHOICES 92 | 93 | try: 94 | lr_params = eval(lr_params) 95 | except (SyntaxError,NameError) as _: 96 | err = 'Tried to parse the following lr_params value\n%s\nas a Python expression, but it failed. lr_params should evaluate to a valid Python dict.' % lr_params 97 | parser.error(err) 98 | 99 | if lr_policy == 'constant': 100 | if not 'lr' in lr_params: 101 | parser.error('Expected lr_params to be dict with at least "lr" key, but dict is %s' % repr(lr_params)) 102 | elif lr_policy == 'progress': 103 | if not ('max_lr' in lr_params and 'desired_prog' in lr_params): 104 | parser.error('Expected lr_params to be dict with at least "max_lr" and "desired_prog" keys, but dict is %s' % repr(lr_params)) 105 | elif lr_policy == 'progress01': 106 | if not ('max_lr' in lr_params and 'early_prog' in lr_params and 'late_prog_mult' in lr_params): 107 | parser.error('Expected lr_params to be dict with at least "max_lr", "early_prog", and "late_prog_mult" keys, but dict is %s' % repr(lr_params)) 108 | 109 | return lr_params 110 | 111 | 112 | 113 | def parse_and_validate_push_spatial(parser, push_spatial): 114 | '''Returns tuple of length 2.''' 115 | try: 116 | push_spatial = eval(push_spatial) 117 | except (SyntaxError,NameError) as _: 118 | err = 'Tried to parse the following push_spatial value\n%s\nas a Python expression, but it failed. push_spatial should be a valid Python expression.' % push_spatial 119 | parser.error(err) 120 | 121 | if push_spatial == None: 122 | push_spatial = (0,0) # Convert to tuple format 123 | elif isinstance(push_spatial, tuple) and len(push_spatial) == 2: 124 | pass 125 | else: 126 | err = 'push_spatial should be None or a valid tuple of indices of length 2, but it is: %s' % push_spatial 127 | parser.error(err) 128 | 129 | return push_spatial 130 | 131 | 132 | 133 | def main(): 134 | parser = get_parser() 135 | args = parser.parse_args() 136 | 137 | # Finish parsing args 138 | channel_swap_to_rgb = eval(args.channel_swap_to_rgb) 139 | assert isinstance(channel_swap_to_rgb, tuple) and len(channel_swap_to_rgb) > 0, 'channel_swap_to_rgb should be a tuple' 140 | data_size = eval(args.data_size) 141 | assert isinstance(data_size, tuple) and len(data_size) == 2, 'data_size should be a length 2 tuple' 142 | #channel_swap_inv = tuple([net_channel_swap.index(ii) for ii in range(len(net_channel_swap))]) 143 | 144 | lr_params = parse_and_validate_lr_params(parser, args.lr_policy, args.lr_params) 145 | push_spatial = parse_and_validate_push_spatial(parser, args.push_spatial) 146 | 147 | # Load mean 148 | data_mean = eval(args.mean) 149 | 150 | if isinstance(data_mean, basestring): 151 | # If the mean is given as a filename, load the file 152 | try: 153 | data_mean = np.load(data_mean) 154 | except IOError: 155 | print '\n\nCound not load mean file:', data_mean 156 | print 'To fetch a default model and mean file, use:\n' 157 | print ' $ cd models/caffenet-yos/' 158 | print ' $ cp ./fetch.sh\n\n' 159 | print 'Or to use your own mean, change caffevis_data_mean in settings_local.py or override by running with `--mean MEAN_FILE` (see --help).\n' 160 | raise 161 | # Crop center region (e.g. 227x227) if mean is larger (e.g. 256x256) 162 | excess_h = data_mean.shape[1] - data_size[0] 163 | excess_w = data_mean.shape[2] - data_size[1] 164 | assert excess_h >= 0 and excess_w >= 0, 'mean should be at least as large as %s' % repr(data_size) 165 | data_mean = data_mean[:, (excess_h/2):(excess_h/2+data_size[0]), (excess_w/2):(excess_w/2+data_size[1])] 166 | elif data_mean is None: 167 | pass 168 | else: 169 | # The mean has been given as a value or a tuple of values 170 | data_mean = np.array(data_mean) 171 | # Promote to shape C,1,1 172 | while len(data_mean.shape) < 3: 173 | data_mean = np.expand_dims(data_mean, -1) 174 | 175 | print 'Using mean:', repr(data_mean) 176 | 177 | # Load network 178 | sys.path.insert(0, os.path.join(args.caffe_root, 'python')) 179 | import caffe 180 | net = caffe.Classifier( 181 | args.deploy_proto, 182 | args.net_weights, 183 | mean = data_mean, 184 | raw_scale = 1.0, 185 | ) 186 | check_force_backward_true(settings.caffevis_deploy_prototxt) 187 | 188 | labels = None 189 | if settings.caffevis_labels: 190 | labels = read_label_file(settings.caffevis_labels) 191 | 192 | optimizer = GradientOptimizer(net, data_mean, labels = labels, 193 | label_layers = settings.caffevis_label_layers, 194 | channel_swap_to_rgb = channel_swap_to_rgb) 195 | 196 | params = FindParams( 197 | start_at = args.start_at, 198 | rand_seed = args.rand_seed, 199 | push_layer = args.push_layer, 200 | push_channel = args.push_channel, 201 | push_spatial = push_spatial, 202 | push_dir = args.push_dir, 203 | decay = args.decay, 204 | blur_radius = args.blur_radius, 205 | blur_every = args.blur_every, 206 | small_val_percentile = args.small_val_percentile, 207 | small_norm_percentile = args.small_norm_percentile, 208 | px_benefit_percentile = args.px_benefit_percentile, 209 | px_abs_benefit_percentile = args.px_abs_benefit_percentile, 210 | lr_policy = args.lr_policy, 211 | lr_params = lr_params, 212 | max_iter = args.max_iter, 213 | ) 214 | 215 | prefix_template = '%s_%s_' % (args.output_prefix, args.output_template) 216 | im = optimizer.run_optimize(params, prefix_template = prefix_template, 217 | brave = args.brave, skipbig = args.skipbig) 218 | 219 | 220 | 221 | if __name__ == '__main__': 222 | main() 223 | -------------------------------------------------------------------------------- /run_toolbox.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | from live_vis import LiveVis 5 | from bindings import bindings 6 | try: 7 | import settings 8 | except: 9 | print '\nError importing settings.py. Check the error message below for more information.' 10 | print "If you haven't already, you'll want to copy one of the settings_local.template-*.py files" 11 | print 'to settings_local.py and edit it to point to your caffe checkout. E.g. via:' 12 | print 13 | print ' $ cp models/caffenet-yos/settings_local.template-caffenet-yos.py settings_local.py' 14 | print ' $ < edit settings_local.py >\n' 15 | raise 16 | 17 | if not os.path.exists(settings.caffevis_caffe_root): 18 | raise Exception('ERROR: Set caffevis_caffe_root in settings.py first.') 19 | 20 | 21 | 22 | def main(): 23 | lv = LiveVis(settings) 24 | 25 | help_keys, _ = bindings.get_key_help('help_mode') 26 | quit_keys, _ = bindings.get_key_help('quit') 27 | print '\n\nRunning toolbox. Push %s for help or %s to quit.\n\n' % (help_keys[0], quit_keys[0]) 28 | lv.run_loop() 29 | 30 | 31 | 32 | if __name__ == '__main__': 33 | main() 34 | -------------------------------------------------------------------------------- /test_keys.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | # Different platforms give different codes for keys. This can cause 4 | # bindings to be messed up. To test the keymapping on your system, run 5 | # this script. 6 | # 7 | # On Mac OS X 10.8, this produces: 8 | # $ ./test_keys.py 9 | # Click on the picture and then carefully push the following keys: 10 | # Press key j: got code 106 = j 11 | # Press key k: got code 107 = k 12 | # Press key J: got code 74 = J 13 | # Press key K: got code 75 = K 14 | # Press key 1: got code 49 = 1 15 | # Press key 2: got code 50 = 2 16 | # Press key left: got code 63234 = left 17 | # Press key right: got code 63235 = right 18 | # Press key esc: got code 27 = esc 19 | 20 | import sys 21 | import cv2 22 | import keys 23 | from bindings import bindings 24 | img = cv2.imread('input_images/ILSVRC2012_val_00000610.jpg') # load example image 25 | 26 | 27 | 28 | def check_key(key_str): 29 | print ' Press key %5s: ' % key_str, 30 | sys.stdout.flush() 31 | while True: 32 | keycode = cv2.waitKey(0) 33 | label, masked_vals = bindings.get_key_label_from_keycode(keycode, extra_info = True) 34 | if label and ('shift' in label or 'ctrl' in label): 35 | print '(ignoring modifier %s)' % label, 36 | sys.stdout.flush() 37 | else: 38 | break 39 | masked_vals_pp = ', '.join(['%d (%s)' % (mv, hex(mv)) for mv in masked_vals]) 40 | if label == key_str: 41 | print ' %d (%s) matched %s' % (keycode, hex(keycode), label) 42 | elif label is not None: 43 | print '* %d (%s) failed, matched key %s (masked vals tried: %s)' % (keycode, hex(keycode), label, masked_vals_pp) 44 | else: 45 | print '* %d (%s) failed, no match found (masked vals tried: %s)' % (keycode, hex(keycode), masked_vals_pp) 46 | #print 'Got:', label 47 | #found = False 48 | #for k,v in keys.Key.__dict__.iteritems(): 49 | # if '__' in k: continue # Skip __module__, etc. 50 | # num,st = v 51 | # if num == key: 52 | # print 'got code %d = %s' % (num,st) 53 | # found = True 54 | # break 55 | #if not found: 56 | # print 'code not found:', key 57 | 58 | 59 | 60 | def main(): 61 | print 'Click on the picture and then carefully push the following keys:' 62 | cv2.imshow('img',img) 63 | check_key('j') 64 | check_key('k') 65 | check_key('J') 66 | check_key('K') 67 | check_key('1') 68 | check_key('2') 69 | check_key('left') 70 | check_key('right') 71 | check_key('esc') 72 | 73 | 74 | 75 | if __name__ == '__main__': 76 | main() 77 | --------------------------------------------------------------------------------