├── .gitignore
├── .zenodo.json
├── CONTRIBUTING.md
├── LICENSE.txt
├── MANIFEST.in
├── README.md
├── README.rst
├── TeLL
├── __init__.py
├── activations.py
├── config.py
├── dataprocessing.py
├── datareaders.py
├── datasets.py
├── evaluation.py
├── initializations.py
├── layers.py
├── loss.py
├── normalizations.py
├── regularization.py
├── scripts
│ ├── __init__.py
│ ├── dropoutmask.py
│ └── resume.py
├── session.py
└── utility
│ ├── __init__.py
│ ├── gputools.py
│ ├── misc.py
│ ├── misc_tensorflow.py
│ ├── plotting.py
│ ├── plotting_daemons.py
│ ├── timer.py
│ └── workingdir.py
├── __init__.py
├── samples
├── config_convlstm.json
├── config_convlstm_mnist.json
├── config_dense.json
├── config_lstm.json
├── config_lstm3.json
├── main_convlstm.py
├── main_convlstm_mnist.py
├── main_lstm.py
├── mnist
│ ├── architectures.py
│ ├── config.json
│ ├── main.py
│ └── main_datapreprocessing.py
├── moving_mnist
│ ├── architectures.py
│ ├── config.json
│ ├── config_gauss.json
│ ├── config_lstm.json
│ ├── dataset.py
│ └── main.py
└── sample_architectures.py
├── setup.cfg
└── setup.py
/.gitignore:
--------------------------------------------------------------------------------
1 | # python
2 | *.pyc
3 | *.pyo
4 |
5 | # temporary files from text editors
6 | *~
7 |
8 | # interim latex files
9 | *.log
10 | *.out
11 | *.aux
12 | *.tmp
13 |
14 | # pycharm project files
15 | *.iml
16 | #*.xml
--------------------------------------------------------------------------------
/.zenodo.json:
--------------------------------------------------------------------------------
1 | {
2 | "license": "MIT",
3 | "access_right": "open",
4 | "title": "Tensorflow Layer Library (TeLL): v1.0.0",
5 | "version": "v1.0.0",
6 | "upload_type": "software",
7 | "publication_date": "2018-06-14",
8 | "description": "Tensorflow Layer Library (TeLL)
",
9 | "creators": [
10 | {
11 | "name": "Michael Widrich"
12 | },
13 | {
14 | "name": "Markus Hofmarcher"
15 | }
16 | ]
17 | }
18 |
--------------------------------------------------------------------------------
/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | # Coding Conventions (Python)
2 |
3 | In general, stick to the [PEP standards](https://www.python.org/dev/peps/pep-0008) as good as possible.
4 |
5 | When adding new functions/modules, use the existing directory structure or include/describe the newly added folders or files in the README.md.
6 |
7 | Keep consistent with existing file, function, and class docstrings. Use and update comments, so that others can quickly see what your code is about.
8 |
9 | Thank you for contributing! :)
10 |
11 | ## Comments for File Structure
12 |
13 | Top-level comment:
14 | ```python
15 | code
16 |
17 |
18 | # ----------------------------------------------------------------------------------------------------------------------
19 | # Comment
20 | # ----------------------------------------------------------------------------------------------------------------------
21 |
22 | code
23 | ```
24 |
25 | or
26 |
27 | ```python
28 | code
29 |
30 |
31 | # ----------------------------------------------------------------------------------------------------------------------
32 | # Comment
33 | # Description of following section
34 | # ----------------------------------------------------------------------------------------------------------------------
35 |
36 | code
37 | ```
38 |
39 | Mid-level comment:
40 | ```python
41 | code
42 |
43 | #
44 | # Comment
45 | #
46 | code
47 | ```
48 |
49 | or
50 |
51 | ```python
52 | code
53 |
54 | #
55 | # Comment
56 | # Description of following section
57 | #
58 | code
59 | ```
60 |
61 | Low-level comment:
62 |
63 | ```python
64 | code
65 |
66 | # Comment
67 | code
68 | ```
69 |
70 |
71 | ## PyCharm Configuration
72 |
73 | If working with PyCharm, please use the provided [configuration file](https://gitlab.markushofmarcher.at/markus.hofmarcher/tools/blob/b2556cd8a097377eb5bf55beedc21b4992a91724/misc/codestyle_cs.xml). Import via
74 |
75 | File->Settings->Editor->Code Style->Manage->Import...
76 |
77 | File->Settings->Editor->File and Code Templates->Python
78 |
79 | ## Docstring conventions
80 |
81 | [Numpy style](https://github.com/numpy/numpy/blob/master/doc/HOWTO_DOCUMENT.rst.txt) is interpretable by sphinx and recommended:
82 |
83 | ## File Template
84 |
85 | ```python
86 | # -*- coding: utf-8 -*-
87 | """
88 | Short discription of contents and purpose of file
89 |
90 | """
91 | ```
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2017, Michael Widrich and Markus Hofmarcher, Institute of Bioinformatics, Johannes Kepler University Linz, Austria.
2 | Free for academic use, commercial license on request.
3 |
4 | The MIT License (MIT)
5 |
6 | Copyright (c) 2017 Michael Widrich and Markus Hofmarcher, Institute of Bioinformatics, Johannes Kepler University Linz, Austria.
7 |
8 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the
9 | Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
10 | and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
15 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
16 | ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH
17 | THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
--------------------------------------------------------------------------------
/MANIFEST.in:
--------------------------------------------------------------------------------
1 | # Include the license file
2 | include LICENSE.txt
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Tensorflow Layer Library (TeLL)
2 | [](https://zenodo.org/badge/latestdoi/87196662)
3 |
4 | Provides a variety of tensorflow-based network layers, flexible (recurrent) network designs, convenience routines for saving and resuming networks, and more!
5 |
6 | Copyright (c) Michael Widrich and Markus Hofmarcher, Institute of Bioinformatics, Johannes Kepler University Linz, Austria.
7 |
8 | If you use TeLL or parts of the code in your work, please cite us as
9 |
10 | @misc{tell,
11 | author = {Michael Widrich and Markus Hofmarcher},
12 | title = {{Tensorflow Layer Library (TeLL)}},
13 | publisher = {Zenodo},
14 | journal = {GitHub repository},
15 | howpublished = {\url{https://github.com/ml-jku/tensorflow-layer-library}},
16 | year = {2018},
17 | doi = {10.5281/zenodo.1292055},
18 | url = {https://doi.org/10.5281/zenodo.1292055},
19 | }
20 |
21 | ## Setup
22 | You can either install TeLL via pip, use TeLL as a git-submodule in your git project, or download it as a static Python package.
23 |
24 | If you intend on using multiple versions of TeLL in different projects, we recommend to use the [git-submodule](#tell-as-git-submodule) approach.
25 | If you use the same TeLL version with all of your projects, a [pip installation](#tell-as-pip-package) or [static Python package](#tell-as-static-python-package) is sufficient.
26 |
27 | TeLL will run with tensorflow version 1.0.
28 |
29 | ### TeLL as Pip Package
30 | Download the TeLL package and install it via
31 |
32 | ```
33 | pip install yourpath/tensorflow-layer-library
34 | ```
35 |
36 | If you want to install the tensorflow dependencies as well, specify "tensorflow" for CPU only or "tensorflow-gpu" for GPU support in brackets after the package, e.g.
37 |
38 | ```
39 | pip install yourpath/tensorflow-layer-library[tensorflow-gpu]
40 | ```
41 |
42 | Continue with section [Usage](#usage).
43 |
44 | ### TeLL as Static Python Package
45 | Download TeLL from GitLab or clone it to your disk. Continue with section [Usage](#usage).
46 |
47 | ### TeLL as Git-Submodule
48 | If you want to keep TeLL as a subfolder in your git-project, with different git-projects having different TeLL versions, it may be best to add TeLL as a git-submodule to your project.
49 | This will create a subfolder "tensorflow-layer-library" in your project folder, which can be separately updated to the last version of TeLL.
50 | Let us assume that your project folder has the following structure:
51 | ``` ruby
52 | myproject/
53 | |
54 | +-- my_main_file.
55 | |
56 | +-- my_other_file.py
57 | ```
58 |
59 | As described in [this](https://git-scm.com/book/en/v2/Git-Tools-Submodules) guide, you will have to move into your project directory and add the submodule:
60 |
61 | ```
62 | cd myproject/
63 | git submodule add https://the-git-repository-address.git
64 | ```
65 |
66 | This will add the submodule and a .gitmodules file to your directory, resulting in the following structure:
67 |
68 | ``` ruby
69 | myproject/
70 | |
71 | +-- my_main_file.py
72 | |
73 | +-- tensorflow-layer-library/
74 | | |
75 | | +-- TeLL/
76 | |
77 | +-- .gitmodules
78 | ```
79 |
80 | Now you have to change the path in the .gitmodules file to a relative path, if your project is hosted on the same server as the submodule:
81 |
82 | ```
83 | [submodule "tensorflow-layer-library"]
84 | path = tensorflow-layer-library
85 | url = ../../TeLL/tensorflow-layer-library.git
86 | ```
87 |
88 | Sources: [https://git-scm.com/book/en/v2/Git-Tools-Submodules](https://git-scm.com/book/en/v2/Git-Tools-Submodules),
89 | [https://docs.gitlab.com/ce/ci/git_submodules.html](https://docs.gitlab.com/ce/ci/git_submodules.html)
90 |
91 | ### Run Example Main-File
92 | Try to run one of the examples [samples/main_lstm.py](samples/main_lstm.py) or [samples/main_convlstm.py](samples/main_convlstm.py) provided in the tensorflow-layer-library folder.
93 | The following should start the computations and create a working_dir folder in the tensorflow-layer-library/samples folder:
94 |
95 | ```
96 | cd tensorflow-layer-library/samples/
97 | python3 main_lstm_example.py --config lstm_example.json
98 | ```
99 |
100 | ## Usage:
101 | A focus of this project is to provide easy and fast usability while keeping the design flexible.
102 | There are three basic steps to perform to create and run your architecture:
103 |
104 | ### Design Dataloader
105 | In order to access/create your dataset, a reader/loader class should be used. This class has to contain a batch_loader() function to yield the minibatches.
106 | Examples for creator-classes are ShortLongDataset and MovingDotDataset in [TeLL/datasets.py](TeLL/datasets.py), which can be adapted for your needs.
107 | For reading data, [TeLL/datasets.py](TeLL/datasets.py) provides the classes DatareaderSimpleFiles and DatareaderAdvancedImageFiles, from which readers can be derived from.
108 | DatareaderSimpleFiles and DatareaderAdvancedImageFiles provide support for automatic loading of data in background processes, search for datafiles, etc..
109 |
110 | ### Design Network Architecture
111 | It is advised to create a new class for each network architecture, e.g. in a file my_architectures.py in your project folder.
112 | In general, the layers can simply be stacked as follows:
113 |
114 | ```python
115 | # define some placeholder for the input and target
116 | X = tf.placeholder(tf.float32, shape=input_shape)
117 | y_ = tf.placeholder(tf.float32, shape=target_shape)
118 |
119 | # stack some layers
120 | layer1 = Layer(incoming=X, ...)
121 | layer2 = Layer(incoming=layer1, ...)
122 | outputlayer = Layer(incoming=layer2, ...)
123 |
124 | # calculate the output of the last layer
125 | output = outputlayer.get_output()
126 | ```
127 |
128 | A collection of forward- and recurrent network sample architectures can be found in [TeLL/samples/sample_architectures.py](samples/sample_architectures.py).
129 |
130 | ### Adapt Main-File
131 | To adapt the main-file to your needs, copy the example file [samples/main_lstm.py](samples/main_lstm.py) or [samples/main_convlstm.py](samples/main_convlstm.py) and modify the loss calculations, starting at line 246, and the dataloader.
132 |
133 | Finally, you will need to create your configuration file (examples can be found in [samples/](samples)) and you are good to go!
134 |
135 | ### Utility Features
136 |
137 | #### Storage/Resumption
138 | By default, TeLL will create checkpoints for each run in the working_dir folder.
139 | These checkpoints contain a .zip of the directory the main file is located in, so that the code base is at the correct version when the run is resumed.
140 |
141 | To resume an experiment run the following command:
142 |
143 | ```
144 | tell-resume --epochs --gpu --path
145 | ```
146 |
147 | #### Plotting
148 | tba
149 |
150 |
151 | ## Directory Structure
152 | The project directory is structured as follows:
153 | ``` ruby
154 | tensorflow-layer-library/
155 | | '''the TeLL project, including example scripts'''
156 | +-- TeLL/
157 | | | '''the TeLL package'''
158 | | +-- architectures/
159 | | | +-- sample_architectures.py
160 | | | '''some example network architectures'''
161 | | +-- configs/
162 | | | +-- examples/
163 | | | | '''example configuration files for usage with sample_architectures.py'''
164 | | | +-- config.py
165 | | | '''default configuration settings'''
166 | | +-- network_modules/
167 | | | '''holds modules for network'''
168 | | | +-- datareader.py
169 | | | | '''base class for dataset readers'''
170 | | | +-- datasets.py
171 | | | | '''classes for dataset loaders and creators'''
172 | | | +-- initializations.py
173 | | | | '''initializers for variables'''
174 | | | +-- layers.py
175 | | | | '''network layer classes'''
176 | | | +-- loss.py
177 | | | | '''loss functions'''
178 | | | +-- regularization.py
179 | | | '''regularization functions'''
180 | | +-- utility/
181 | | '''holds convenience functions'''
182 | | +-- misc.py
183 | | | '''unclassified convenience functions'''
184 | | +-- plotting.py
185 | | | '''functions for plotting and saving images/videos'''
186 | | +-- plotting_daemons.py
187 | | '''functions for creating and starting (sub)processes for plotting'''
188 | +-- README.md
189 | | '''this file'''
190 | +-- main_lstm_example.py
191 | | '''example main file for LSTM architectures'''
192 | +-- main_convlstm_example.py
193 | | '''example main file for convLSTM architectures'''
194 | +-- main_convlstm_advanced_example.py
195 | | '''example main file for advanced convLSTM architectures'''
196 | +-- todo.py
197 | '''todo-list: indicate on what you are working and strikethrough when you are done'''
198 | ```
199 |
200 | ## Contributing
201 |
202 | If you want to contribute to TeLL, please read the [guidelines](CONTRIBUTING.md), create a branch or fork, and send merge-requests.
203 | For contribution to this project, your have to assign the copyright of the contribution to the TeLL project.
204 | Please include the statement "I hereby assign copyright in this code to the TeLL project, to be licensed under the same terms as the rest of the code." in your merge requests.
205 |
--------------------------------------------------------------------------------
/README.rst:
--------------------------------------------------------------------------------
1 | Tensorflow Layer Library (TeLL)
2 | ===============================
3 |
4 | Provides a variety of tensorflow-based network layers, flexible
5 | (recurrent) network designs, convenience routines for saving and
6 | resuming networks, and more!
7 |
8 | Copyright (c) 2016-2017 Michael Widrich and Markus Hofmarcher, Institute
9 | of Bioinformatics, Johannes Kepler University Linz, Austria
10 |
11 | Setup
12 | -----
13 |
14 | You can either use TeLL as a git-submodule in your git project or as a
15 | static Python package.
16 |
17 | If you intend on using multiple versions of TeLL in different projects,
18 | we recommend to use the `git-submodule <#tell-as-git-submodule>`__
19 | approach. If you use the same TeLL version with all of your projects, a
20 | `static Python package <#tell-as-static-python-package>`__ is
21 | sufficient.
22 |
23 | TeLL will run with tensorflow version 1.0.
24 |
25 | TeLL as Pip Package
26 | ~~~~~~~~~~~~~~~~~~~
27 |
28 | Download the TeLL package and install it via
29 |
30 | ::
31 |
32 | pip install
33 |
34 | If you want to install the tensorflow dependencies as well specify
35 | "tensorflow" or "tensorflow-gpu" in brackets after the package, e.g.
36 |
37 | ::
38 |
39 | pip install [tensorflow-gpu]
40 |
41 | TeLL as Static Python Package
42 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
43 |
44 | Download TeLL from GitLab or clone it to your disk. Continue with
45 | section `Usage <#usage>`__.
46 |
47 | TeLL as Git-Submodule
48 | ~~~~~~~~~~~~~~~~~~~~~
49 |
50 | If you want to keep TeLL as a subfolder in your git-project, with
51 | different git-projects having different TeLL versions, it may be best to
52 | add TeLL as a git-submodule to your project. This will create a
53 | subfolder "tensorflow-layer-library" in your project folder, which can
54 | be separately updated to the last version of TeLL. Let us assume that
55 | your project folder has the following structure:
56 |
57 | .. code:: ruby
58 |
59 | myproject/
60 | |
61 | +-- my_main_file.
62 | |
63 | +-- my_other_file.py
64 |
65 | As described in
66 | `this `__ guide,
67 | you will have to move into your project directory and add the submodule:
68 |
69 | ::
70 |
71 | cd myproject/
72 | git submodule add https://the-git-repository-address.git
73 |
74 | This will add the submodule and a .gitmodules file to your directory,
75 | resulting in the following structure:
76 |
77 | .. code:: ruby
78 |
79 | myproject/
80 | |
81 | +-- my_main_file.py
82 | |
83 | +-- tensorflow-layer-library/
84 | | |
85 | | +-- TeLL/
86 | |
87 | +-- .gitmodules
88 |
89 | Now you have to change the path in the .gitmodules file to a relative
90 | path, if your project is hosted on the same server as the submodule:
91 |
92 | ::
93 |
94 | [submodule "tensorflow-layer-library"]
95 | path = tensorflow-layer-library
96 | url = ../../TeLL/tensorflow-layer-library.git
97 |
98 | Sources: https://git-scm.com/book/en/v2/Git-Tools-Submodules,
99 | https://docs.gitlab.com/ce/ci/git_submodules.html
100 |
101 | Run Example Main-File
102 | ~~~~~~~~~~~~~~~~~~~~~
103 |
104 | Try to run one of the examples
105 | `main\_lstm\_example.py `__
106 | or
107 | `main\_convlstm\_example.py `__
108 | provided in the tensorflow-layer-library folder. The following should
109 | start the computations and create a working\_dir folder in the
110 | tensorflow-layer-library folder:
111 |
112 | ::
113 |
114 | cd tensorflow-layer-library/
115 | python3 main_lstm_example.py --config TeLL/configs/examples/lstm_example.json
116 |
117 | Usage:
118 | ------
119 |
120 | A focus of this project is to provide easy and fast usability while
121 | keeping the design flexible. There are three basic steps to perform to
122 | create and run your architecture:
123 |
124 | Design Dataloader
125 | ~~~~~~~~~~~~~~~~~
126 |
127 | In order to access/create your dataset, a reader/loader class should be
128 | used. This class has to contain a batch\_loader() function to yield the
129 | minibatches. Examples for creator-classes are ShortLongDataset and
130 | MovingDotDataset in
131 | `TeLL/network\_modules/datasets.py `__,
132 | which can be adapted for your needs. For reading data,
133 | `TeLL/network\_modules/datasets.py `__
134 | provides the classes DatareaderSimpleFiles and
135 | DatareaderAdvancedImageFiles, from which readers can be derived from.
136 | DatareaderSimpleFiles and DatareaderAdvancedImageFiles provide support
137 | for automatic loading of data in background processes, search for
138 | datafiles, etc..
139 |
140 | Design Network Architecture
141 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~
142 |
143 | It is advised to create a new class for each network architecture, e.g.
144 | in a file my\_architectures.py in your project folder. In general, the
145 | layers can simply be stacked as follows:
146 |
147 | .. code:: python
148 |
149 | # define some placeholder for the input and target
150 | X = tf.placeholder(tf.float32, shape=input_shape)
151 | y_ = tf.placeholder(tf.float32, shape=target_shape)
152 |
153 | # stack some layers
154 | layer1 = Layer(incoming=X, ...)
155 | layer2 = Layer(incoming=layer1, ...)
156 | outputlayer = Layer(incoming=layer2, ...)
157 |
158 | # calculate the output of the last layer
159 | output = outputlayer.get_output()
160 |
161 | A collection of forward- and recurrent network sample architectures can
162 | be found in
163 | `TeLL/architectures/sample\_architectures.py `__.
164 |
165 | Adapt Main-File
166 | ~~~~~~~~~~~~~~~
167 |
168 | To adapt the main-file to your needs, copy the example file
169 | `TeLL/architectures/main\_lstm\_example.py `__
170 | or
171 | `architectures/main\_lstm\_example.py `__
172 | and modify the loss calculations, starting at line 246, and the
173 | dataloader.
174 |
175 | You will probably also have to adapt the path in variable
176 | tell\_library\_path (first code line in the main files) to your path
177 | '/somepath/tensorflow-layer-library/'. Alternatively, you may also add
178 | the path to the system's PYTHONPATH.
179 |
180 | Finally, you will need to create your configuration file (examples can
181 | be found in
182 | `TeLL/configs/examples `__)
183 | and you are good to go!
184 |
185 | Utility Features
186 | ~~~~~~~~~~~~~~~~
187 |
188 | Storage/Resumption
189 | ^^^^^^^^^^^^^^^^^^
190 |
191 | By default, TeLL will create checkpoints for each run in the
192 | working\_dir folder. These checkpoints contain a .zip of the directory
193 | the main file is located in, so that the code base is at the correct
194 | version when the run is resumed.
195 |
196 | To resume an experiment run the following command:
197 |
198 | ::
199 |
200 | tell-resume --epochs --gpu --path
201 |
202 | Plotting
203 | ^^^^^^^^
204 |
205 | Directory Structure
206 | -------------------
207 |
208 | The project directory is structured as follows:
209 |
210 | .. code:: ruby
211 |
212 | tensorflow-layer-library/
213 | | '''the TeLL project, including example scripts'''
214 | +-- TeLL/
215 | | | '''the TeLL package'''
216 | | +-- architectures/
217 | | | +-- sample_architectures.py
218 | | | '''some example network architectures'''
219 | | +-- configs/
220 | | | +-- examples/
221 | | | | '''example configuration files for usage with sample_architectures.py'''
222 | | | +-- config.py
223 | | | '''default configuration settings'''
224 | | +-- network_modules/
225 | | | '''holds modules for network'''
226 | | | +-- datareader.py
227 | | | | '''base class for dataset readers'''
228 | | | +-- datasets.py
229 | | | | '''classes for dataset loaders and creators'''
230 | | | +-- initializations.py
231 | | | | '''initializers for variables'''
232 | | | +-- layers.py
233 | | | | '''network layer classes'''
234 | | | +-- loss.py
235 | | | | '''loss functions'''
236 | | | +-- regularization.py
237 | | | '''regularization functions'''
238 | | +-- utility/
239 | | '''holds convenience functions'''
240 | | +-- misc.py
241 | | | '''unclassified convenience functions'''
242 | | +-- plotting.py
243 | | | '''functions for plotting and saving images/videos'''
244 | | +-- plotting_daemons.py
245 | | '''functions for creating and starting (sub)processes for plotting'''
246 | +-- README.md
247 | | '''this file'''
248 | +-- main_lstm_example.py
249 | | '''example main file for LSTM architectures'''
250 | +-- main_convlstm_example.py
251 | | '''example main file for convLSTM architectures'''
252 | +-- main_convlstm_advanced_example.py
253 | | '''example main file for advanced convLSTM architectures'''
254 | +-- todo.py
255 | '''todo-list: indicate on what you are working and strikethrough when you are done'''
--------------------------------------------------------------------------------
/TeLL/__init__.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """__init__.py.py: short description
3 |
4 |
5 | Author -- Michael Widrich
6 | Created on -- 2017-03-01
7 | Contact -- michael.widrich@jku.at
8 |
9 | long description
10 |
11 |
12 | ======= ========== ================= ================================
13 | Version Date Author Description
14 | 0.1 2017-03-01 Michael Widrich -
15 | ======= ========== ================= ================================
16 |
17 | """
--------------------------------------------------------------------------------
/TeLL/activations.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Functions for regularization and convenience wrappers for tensorflow regularization functions
6 |
7 | """
8 | import tensorflow as tf
9 |
10 | '''
11 | Tensorflow Implementation of the Scaled ELU function and Dropout
12 | '''
13 |
14 |
15 | def selu(x):
16 | """ When using SELUs you have to keep the following in mind:
17 | # (1) scale inputs to zero mean and unit variance
18 | # (2) use SELUs
19 | # (3) initialize weights with stddev sqrt(1/n)
20 | # (4) use SELU dropout
21 | """
22 | with tf.name_scope('selu'):
23 | alpha = 1.6732632423543772848170429916717
24 | scale = 1.0507009873554804934193349852946
25 | return scale * tf.where(x >= 0.0, x, alpha * tf.nn.elu(x))
26 |
27 | def selu_offset(offset=None):
28 | """selu with constant offset, offset defaults to alpha*scale, i.e. only positive activations"""
29 | with tf.name_scope('selu_offset'):
30 | alpha = 1.6732632423543772848170429916717+1e-7
31 | scale = 1.0507009873554804934193349852946
32 | if offset is None:
33 | offset = tf.constant(alpha*scale, dtype=tf.float32)
34 | return lambda *args, **kwargs: selu(*args, **kwargs) + offset
35 |
--------------------------------------------------------------------------------
/TeLL/config.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Default configuration settings
6 |
7 | """
8 | import json
9 | import os
10 |
11 | from TeLL.utility.misc import import_object, extract_named_args, try_to_number, parse_args
12 |
13 |
14 | class Config(object):
15 | def __init__(self, filename: str = None):
16 | """Create config object from json file.
17 |
18 | filename : optional;
19 | If passed read config from specified file, otherwise parse command line for config parameter and optionally
20 | override arguments.
21 | """
22 | if filename is None:
23 | args, override_args = parse_args()
24 | config_file = args.config
25 | else:
26 | args = None
27 | override_args = None
28 | config_file = filename
29 |
30 | # Read config and override with args if passed
31 | if os.path.exists(config_file):
32 | with open(config_file) as f:
33 | self.initialize_from_json(json.loads(f.read()).items())
34 | # set restore path if passed
35 | if args is not None and args.restore is not None:
36 | self.restore = args.restore
37 | # override if necessary
38 | if override_args is not None:
39 | self.override_from_commandline(override_args)
40 | else:
41 | raise Exception("Configuration file does not exist!")
42 |
43 | def override(self, name, value):
44 | if value is not None:
45 | setattr(self, name, value)
46 | print("CONFIG: {}={}".format(name, getattr(self, name)))
47 |
48 | def has_value(self, name):
49 | return hasattr(self, name)
50 |
51 | def get_value(self, name, default=None):
52 | return getattr(self, name, default)
53 |
54 | def import_architecture(self):
55 | if hasattr(self, "architecture"):
56 | return import_object(self.architecture)
57 | else:
58 | return None
59 |
60 | def initialize_from_json(self, nv_pairs=None):
61 | if nv_pairs:
62 | for i, (name, value) in enumerate(nv_pairs):
63 | self.override(name, value)
64 |
65 | def override_from_commandline(self, override_args=None):
66 | if override_args is not None:
67 | override = extract_named_args(override_args)
68 | for k, v in override.items():
69 | name = k[2:]
70 | value = v if v.startswith('"') or v.startswith("'") else try_to_number(v)
71 | if "." in name:
72 | names = name.split(".")
73 | name = names[0]
74 | if len(names) == 2:
75 | if hasattr(self, names[0]):
76 | curdict = getattr(self, names[0])
77 | else:
78 | curdict = dict()
79 | curdict[names[1]] = value
80 | value = curdict
81 | elif len(names) == 3:
82 | if hasattr(self, names[0]):
83 | curdict = getattr(self, names[0])
84 | else:
85 | curdict = dict()
86 |
87 | if names[1] in curdict:
88 | subdict = curdict[names[1]]
89 | else:
90 | curdict[names[1]] = dict()
91 | subdict = curdict[names[1]]
92 |
93 | subdict[names[2]] = value
94 | value = curdict
95 | else:
96 | raise Exception("Unsupported command line option (can only override dicts with 1 or 2 levels)")
97 | self.override(name, value)
98 |
99 | #
100 | # Default settings
101 | #
102 | specs = 'default'
103 |
104 | # logging and plotting
105 | plot_at = 100 # plot at each xth weight update
106 | score_at = 100 # calculate score on validation set at each xth weight update
107 |
108 | # GPU and CPU usage
109 | cuda_gpu = "0"
110 | inter_op_parallelism_threads = 1
111 | intra_op_parallelism_threads = 1
112 | log_device_placement = False
113 |
114 | # Default paths
115 | restore = None
116 | working_dir = "working_dir"
117 |
--------------------------------------------------------------------------------
/TeLL/evaluation.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 | """
4 |
5 | import sys
6 | import progressbar
7 | import numpy as np
8 | import tensorflow as tf
9 | from collections import OrderedDict
10 |
11 | from TeLL.utility.misc import custom_tensorflow_histogram, check_kill_file
12 | from TeLL.utility.timer import Timer
13 | from TeLL.utility.workingdir import Workspace
14 |
15 |
16 | class Evaluation(object):
17 | def __init__(self, dataset, session, model, workspace: Workspace, summary_tensor_dict=None, scope=None):
18 | """Evaluate model on dataset"""
19 |
20 | #
21 | # Create tensors and tf operations
22 | #
23 | if summary_tensor_dict is None:
24 | summary_tensor_dict = {}
25 |
26 | summary_tensors = [tens[0] if isinstance(tens, tuple) else tens for tens in summary_tensor_dict.values()]
27 | summary_tensor_is_op = [True if isinstance(tens, tuple) else False for tens in summary_tensor_dict.values()]
28 | summary_ops = [tens[1] for tens in summary_tensor_dict.values() if isinstance(tens, tuple)]
29 |
30 | self.dataset = dataset
31 | self.session = session
32 | self.model = model
33 | self.workspace = workspace
34 | self.scope = scope
35 |
36 | self.summary_tensor_dict = summary_tensor_dict
37 | self.summary_tensors = summary_tensors
38 | self.summary_tensor_is_op = summary_tensor_is_op
39 | self.summary_ops = summary_ops
40 |
41 | resetable_tensors = tf.get_collection(tf.GraphKeys.LOCAL_VARIABLES, scope=self.scope.name)
42 | self.variables_initializer = tf.variables_initializer(resetable_tensors)
43 | self.reset_tensors()
44 |
45 | def reset_tensors(self):
46 | if self.scope is not None:
47 | self.session.run([self.variables_initializer])
48 |
49 | def evaluate(self, step: int, summary_writer, prefix='validation ', num_cached=5, num_threads=3, rnd_gen=None,
50 | plotter=None, model=None):
51 | # Reset streaming measures
52 | self.reset_tensors()
53 |
54 | # Get tensors to evaluate for plotting
55 | if plotter is not None:
56 | plot_tensors = plotter.get_tensors()
57 |
58 | # Set up progress bar
59 | _pbw = ['Evaluating on {}set:'.format(prefix), progressbar.ETA()]
60 | progress = progressbar.ProgressBar(widgets=_pbw, maxval=self.dataset.n_mbs - 1).start()
61 |
62 | #
63 | # Iterate over dataset minibatches
64 | #
65 | mb_validation = self.dataset.batch_loader(num_cached=num_cached, num_threads=num_threads, rnd_gen=rnd_gen)
66 | with Timer(verbose=True, name="Evaluate on {}set".format(prefix)):
67 | summary_values_filled = None
68 |
69 | for mb_i, mb in enumerate(mb_validation):
70 |
71 | # Abort if indicated by file
72 | check_kill_file(self.workspace)
73 |
74 | if mb.get('pixel_weights', None) is None:
75 | feed_dict = {self.model.X: mb['X'], self.model.y_: mb['y']}
76 | else:
77 | feed_dict = {self.model.X: mb['X'], self.model.y_: mb['y'],
78 | self.model.pixel_weights: mb['pixel_weights']}
79 |
80 | if plotter is not None:
81 | evaluated_tensors = self.session.run([*self.summary_ops, *self.summary_tensors, *plot_tensors],
82 | feed_dict=feed_dict)
83 | else:
84 | evaluated_tensors = self.session.run([*self.summary_ops, *self.summary_tensors],
85 | feed_dict=feed_dict)
86 |
87 | # Discard return values from summary_ops (=update operations)
88 | evaluated_tensors = evaluated_tensors[len(self.summary_ops):]
89 | summary_values = evaluated_tensors[:len(self.summary_tensors)]
90 |
91 | # Perform plotting
92 | if plotter is not None:
93 | plotter.set_tensor_values(evaluated_tensors[len(self.summary_tensors):len(self.plot_tensors) +
94 | len(plot_tensors)])
95 | plotter.plot(evaluate_tensors=False)
96 |
97 | # Re-associate returned tensorflow values to keys and incorporate new minibatch values
98 | if summary_values_filled is None:
99 | # Fill summary_values_filled for the first time
100 | summary_values_filled = OrderedDict(zip(list(self.summary_tensor_dict.keys()), summary_values))
101 | for key_i, key in enumerate(summary_values_filled.keys()):
102 | if not self.summary_tensor_is_op[key_i]:
103 | if isinstance(summary_values_filled[key], np.ndarray):
104 | summary_values_filled[key] = [summary_values_filled[key]]
105 | elif np.isfinite(summary_values_filled[key]):
106 | summary_values_filled[key] = [summary_values_filled[key]]
107 | else:
108 | summary_values_filled[key] = []
109 | else:
110 | for key_i, key in enumerate(summary_values_filled.keys()):
111 | if not self.summary_tensor_is_op[key_i]:
112 | if isinstance(summary_values[key_i], np.ndarray):
113 | summary_values_filled[key].append(summary_values[key_i])
114 | elif np.isfinite(summary_values[key_i]):
115 | summary_values_filled[key].append(summary_values[key_i])
116 | else:
117 | summary_values_filled[key] = summary_values[key_i]
118 |
119 | # Update progress bar and clear minibatch
120 | progress.update(mb_i)
121 | mb.clear()
122 | del mb
123 |
124 | progress.finish()
125 |
126 | #
127 | # Divide sums by number of samples for tensors that do not have an update function
128 | #
129 | if len(summary_values_filled):
130 | for key_i, key in enumerate(summary_values_filled.keys()):
131 | if not self.summary_tensor_is_op[key_i]:
132 | if len(summary_values_filled[key]):
133 | if not isinstance(summary_values_filled[key][0], np.ndarray):
134 | summary_values_filled[key] = np.mean(summary_values_filled[key])
135 | else:
136 | summary_values_filled[key] = np.concatenate(summary_values_filled[key])
137 | else:
138 | summary_values_filled[key] = np.nan
139 |
140 | #
141 | # Go through values to use as summaries, create histograms if values are not scalars
142 | #
143 | values_to_print = OrderedDict()
144 | if len(summary_values_filled):
145 | for key_i, key in enumerate(summary_values_filled.keys()):
146 | if not isinstance(summary_values_filled[key], np.ndarray):
147 | values_to_print.update({key: summary_values_filled[key]})
148 | summary = tf.Summary(value=[tf.Summary.Value(tag=prefix + key,
149 | simple_value=float(summary_values_filled[key]))])
150 | else:
151 | hist = custom_tensorflow_histogram(summary_values_filled[key], bins=100)
152 | summary = tf.Summary(value=[tf.Summary.Value(tag=prefix + key, histo=hist)])
153 |
154 | summary_writer.add_summary(summary, step)
155 |
156 | print("{}scores:\n\tstep {}, {}".format(prefix, step, values_to_print))
157 | summary_writer.flush()
158 | sys.stdout.flush()
159 |
--------------------------------------------------------------------------------
/TeLL/initializations.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Functions for initializing tensorflow variables and wrappers for tensorflow initializers
6 |
7 | """
8 |
9 | import numpy as np
10 | import tensorflow as tf
11 |
12 |
13 | def constant(shape=None, init=0.1, dtype=tf.float32):
14 | if isinstance(init, (float, int)):
15 | initial = tf.constant(init, shape=shape, dtype=dtype)
16 | else:
17 | initial = tf.constant(init, dtype=dtype)
18 |
19 | return initial
20 |
21 |
22 | def orthogonal(shape, gain=np.sqrt(2)):
23 | """ orthogonal initialization method
24 |
25 | Parameters
26 | -------
27 | shape : array
28 | the shape of the weight matrix. Frist dimension contains the width of the layer below.
29 | """
30 | return tf.initializers.orthogonal(gain=gain)(shape=shape)
31 |
32 |
33 | def scaled_elu_initialization(shape, truncated=True):
34 | """ Preferred variable initialization method for the scaled ELU activation function.
35 |
36 | Parameters
37 | -------
38 | shape : array
39 | the shape of the weight matrix. Frist dimension contains the width of the layer below.
40 | truncated : boolean
41 | Whether the truncated normal distribution should be used.
42 | """
43 | if len(shape) == 4:
44 | f_in = int(np.prod(shape[:-1]))
45 | else:
46 | f_in = shape[0]
47 |
48 | if truncated:
49 | return tf.truncated_normal(shape=shape, stddev=tf.cast(tf.sqrt(1 / f_in), tf.float32))
50 | else:
51 | return tf.random_normal(shape=shape, stddev=tf.cast(tf.sqrt(1 / f_in), tf.float32))
52 |
53 |
54 | def scaled_elu_initialization_rec(shape, truncated=True):
55 | """ Preferred variable initialization method for the scaled ELU activation function.
56 |
57 | Parameters
58 | -------
59 | shape : array
60 | the shape of the weight matrix. Frist dimension contains the width of the layer below.
61 | truncated : boolean
62 | Whether the truncated normal distribution should be used.
63 | """
64 | if len(shape) == 4:
65 | f_in = int(np.prod(shape[:-1]))
66 | else:
67 | f_in = shape[0]
68 |
69 | if truncated:
70 | return tf.truncated_normal(shape=shape, stddev=tf.cast(tf.sqrt(1 / f_in / 10), tf.float32))
71 | else:
72 | return tf.random_normal(shape=shape, stddev=tf.cast(tf.sqrt(1 / f_in / 10), tf.float32))
73 |
74 |
75 | def weight_klambauer_elu(shape, seed=None):
76 | """ Preferred variable initialization method for the non-scaled ELU activation function.
77 |
78 | Parameters
79 | -------
80 | shape : array
81 | the shape of the weight matrix. First dimension contains the width of the layer below.
82 | seed : integer
83 | seed for the initialization
84 | """
85 |
86 | klambauer_constat = 1.5505188080679277
87 | initial = tf.contrib.layers.variance_scaling_initializer(factor=klambauer_constat, mode='FAN_IN',
88 | uniform=False, seed=seed, dtype=tf.float32)
89 | return initial(shape)
90 |
91 |
92 | def gaussian(x, mu, std):
93 | return (1. / (np.sqrt(2 * np.pi) * std)) * (np.exp(- (x - mu) * (x - mu) / (2 * std * std)))
94 |
95 |
96 | def weight_xavier(shape, uniform=False, seed=None):
97 | initial = tf.contrib.layers.xavier_initializer(uniform=uniform, seed=seed, dtype=tf.float32)
98 | return initial(shape)
99 |
100 |
101 | def weight_xavier_conv2d(shape, uniform=False, seed=None):
102 | initial = tf.contrib.layers.xavier_initializer_conv2d(uniform=uniform, seed=seed, dtype=tf.float32)
103 | return initial(shape)
104 |
105 |
106 | def weight_he(shape, seed=None):
107 | initial = tf.contrib.layers.variance_scaling_initializer(factor=2.0, mode='FAN_IN', uniform=False, seed=seed,
108 | dtype=tf.float32)
109 | return initial(shape)
110 |
111 |
112 | weight_he_conv2d = weight_he
113 |
114 |
115 | def weight_truncated_normal(shape, stddev=0.1):
116 | initial = tf.truncated_normal(shape, stddev=stddev)
117 | return initial
118 |
119 |
120 | def weight_gauss_conv2d(shape, dtype=np.float32):
121 | # Use gauss PDF with center at middle of axes to initialize weights
122 | x = np.arange(np.ceil(shape[0] / 2))
123 | if (shape[0] % 2) == 0:
124 | x = np.append(x, x[::-1])
125 | else:
126 | x = np.append(x, x[-2::-1])
127 |
128 | p_x = gaussian(x, mu=np.ceil(shape[0] / 2), std=np.ceil(shape[0] / 2) / 4)
129 |
130 | y = np.arange(np.ceil(shape[1] / 2))
131 | if (shape[1] % 2) == 0:
132 | y = np.append(y, y[::-1])
133 | else:
134 | y = np.append(y, y[-2::-1])
135 |
136 | p_y = gaussian(y, mu=np.ceil(shape[1] / 2), std=np.ceil(shape[1] / 2) / 4)
137 |
138 | p_xy = np.outer(p_x, p_y)
139 |
140 | W = np.zeros(shape, dtype=dtype)
141 | W[:, :, :, :] = p_xy[:, :, None, None]
142 |
143 | return constant(init=W)
144 |
--------------------------------------------------------------------------------
/TeLL/normalizations.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 | Functions for normalization
5 |
6 | """
7 |
8 | import tensorflow as tf
9 |
10 |
11 | def max_norm(tensor):
12 | """Simple normalization by maximum"""
13 | maximum = tf.reduce_max(tf.abs(tensor))
14 | tensor /= maximum
15 | return tensor
16 |
17 |
18 | def max_norm_all_tensors(tensor_list, clip: bool = True, max_val=tf.constant(1.0)):
19 | """Normalization of list of tensors by maximum of tensors"""
20 | maxima = [tf.reduce_max(tf.abs(tensor)) for tensor in tensor_list]
21 | maxima = tf.stack(maxima)
22 | if clip:
23 | maximum = tf.reduce_max(maxima) + 1e-16
24 | else:
25 | maximum = tf.reduce_max(maxima)
26 | return [tf.divide(tensor, maximum) * max_val for tensor in tensor_list]
27 |
28 |
29 | def euclid_norm(tensor):
30 | """Normalization by euclidean distance"""
31 | summation = tf.reduce_sum(tf.square(tensor))
32 | tensor /= tf.sqrt(summation)
33 | return tensor
34 |
--------------------------------------------------------------------------------
/TeLL/regularization.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Functions for regularization and convenience wrappers for tensorflow regularization functions
6 |
7 | """
8 | import numbers
9 |
10 | import tensorflow as tf
11 | from tensorflow.python.framework import ops
12 | from tensorflow.python.framework import tensor_shape
13 | from tensorflow.python.framework import tensor_util
14 | from tensorflow.python.layers import utils
15 | from tensorflow.python.ops import array_ops
16 | from tensorflow.python.ops import math_ops
17 | from tensorflow.python.ops import random_ops
18 |
19 | from .layers import get_input
20 |
21 |
22 | def regularize(layers, l1=1e-6, l2=1e-3, regularize_weights=True, regularize_biases=True):
23 | """ Regularize weights and/or biases of given layers if they offer
24 | a function getWeigthts/getBiases respectively to retrieve them.
25 |
26 | Parameters
27 | -------
28 | layers : objects implementing getWeights and/or getBiases
29 | Array of layers to regularize
30 | l1 : float
31 | Weight of L1 regularization (default = 1e-6)
32 | l2 : float
33 | Weight of L2 regularization (default = 1e-3)
34 | regularize_weights : bool
35 | Regularize only layer weights (default = True)
36 | regularize_biases : bool
37 | Regularize only layer biases (default = True)
38 |
39 | Returns
40 | -------
41 | Returns combined regularization penalty.
42 |
43 | """
44 | penalty = 0
45 | for layer in layers:
46 | get_weights = getattr(layer, "get_weights", None)
47 | get_biases = getattr(layer, "get_biases", None)
48 | if regularize_weights and callable(get_weights):
49 | weights = get_weights()
50 | for w in weights:
51 | if l1 != 0:
52 | penalty += l1 * tf.reduce_sum(tf.abs(w))
53 | if l2 != 0:
54 | penalty += l2 * tf.nn.l2_loss(w)
55 | if regularize_biases and callable(get_biases):
56 | biases = get_biases()
57 | for b in biases:
58 | if l1 != 0:
59 | penalty += l1 * tf.reduce_sum(tf.abs(b))
60 | if l2 != 0:
61 | penalty += l2 * tf.nn.l2_loss(b)
62 | return penalty
63 |
64 |
65 | def __covar_l1norm(hiddens, enum_dims, feature_dims, n_features):
66 | """ Compute the L1 norm of the covariance matrix of hiddens. """
67 | enum_dims = list(set(range(len(hiddens.shape.as_list()))) - set(feature_dims))
68 | centered = hiddens - tf.reduce_mean(hiddens, enum_dims, keep_dims=True)
69 |
70 | # quick-fix
71 | # data2d = tf.reshape(tf.transpose(centered, enum_dims + feature_dims), [-1, n_features])
72 | data2d = tf.reshape(centered, [-1, n_features])
73 |
74 | covar_scaled_matrix = tf.matmul(data2d, data2d, transpose_a=True)
75 | covar_l1norm = tf.reduce_sum(tf.abs(covar_scaled_matrix))
76 | covar_num = tf.to_float(tf.shape(data2d)[0])
77 |
78 | return covar_l1norm, covar_num
79 |
80 |
81 | def decor_penalty(layer, labels, n_classes, feature_dims, weight=1e-3, is_one_hot=True):
82 | """ Compute layer's decorrelation penalty conditioned on labels.
83 |
84 | For every class in labels, this function computes the L1-norm of the covariance
85 | matrix of the hiddens belonging to that class and returns the sum of these norms
86 | multiplied by weight.
87 |
88 | layer : The layer to regularize.
89 | labels : An integer tensor representing the labels of layer's output on which the
90 | decorrelation should be conditioned. The first tf.rank(labels) dimensions
91 | of labels and layer's output must match. If not one-hot encoded, the values of
92 | labels must be in [0 .. n_classes-1].
93 | n_classes : The number of distinct classes in labels.
94 | feature_dims : The dimensions (as list) of layer's output representing the features
95 | which are to be decorrelated.
96 | weight : A factor by which to multiply the final score.
97 | is_one_hot : Whether the labels are one-hot or integer encoded.
98 | """
99 |
100 | # TODO: maybe enable for a list of layers
101 | # TODO: make default arg for feature_dims
102 |
103 | get_hiddens, h_shape = get_input(layer)
104 | hiddens = get_hiddens()
105 |
106 | labels_rank = len(labels.shape.as_list())
107 | hiddens_rank = len(h_shape)
108 |
109 | if is_one_hot:
110 | labels_rank -= 1
111 |
112 | # ensure label dims come first, feature dims come later
113 | assert (all(labels_rank <= i and i < hiddens_rank for i in feature_dims))
114 |
115 | # initial score
116 | score = tf.zeros([1])
117 | denom = tf.zeros([1])
118 |
119 | # all non-feature dims enumerate the feature instances
120 | enum_dims = list(set(range(len(h_shape))) - set(feature_dims))
121 |
122 | # compute the total number of features
123 | n_features = 1
124 | for i in feature_dims:
125 | n_features *= h_shape[i]
126 |
127 | # make sure there was no -1 in the product
128 | assert (n_features > 0)
129 |
130 | for i in range(n_classes):
131 | # make boolean mask indicating membership to class i
132 | if is_one_hot:
133 | class_i_mask = tf.equal(labels[:, i], 1)
134 | else:
135 | class_i_mask = tf.equal(labels, i)
136 | # determine if class i is empty
137 | empty = tf.equal(tf.reduce_sum(tf.to_float(class_i_mask)), 0)
138 |
139 | # add _covar_l1norm to score if not empty
140 | l1norm, num = tf.cond(empty, lambda: (tf.zeros([1]), tf.zeros([1])), lambda: __covar_l1norm(
141 | tf.boolean_mask(hiddens, class_i_mask), enum_dims, feature_dims, n_features))
142 |
143 | score += l1norm
144 | denom += num
145 |
146 | score /= denom
147 | return tf.reduce_sum((score / tf.to_float(n_classes)) * weight)
148 |
149 |
150 | def dropout_selu(x, rate, alpha=-1.7580993408473766, fixedPointMean=0.0, fixedPointVar=1.0,
151 | noise_shape=None, seed=None, name=None, training=False):
152 | """Dropout to a value with rescaling."""
153 |
154 | def dropout_selu_impl(x, rate, alpha, noise_shape, seed, name):
155 | keep_prob = 1.0 - rate
156 | x = ops.convert_to_tensor(x, name="x")
157 | if isinstance(keep_prob, numbers.Real) and not 0 < keep_prob <= 1:
158 | raise ValueError("keep_prob must be a scalar tensor or a float in the "
159 | "range (0, 1], got %g" % keep_prob)
160 | keep_prob = ops.convert_to_tensor(keep_prob, dtype=x.dtype, name="keep_prob")
161 | keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
162 |
163 | alpha = ops.convert_to_tensor(alpha, dtype=x.dtype, name="alpha")
164 | keep_prob.get_shape().assert_is_compatible_with(tensor_shape.scalar())
165 |
166 | if tensor_util.constant_value(keep_prob) == 1:
167 | return x
168 |
169 | noise_shape = noise_shape if noise_shape is not None else array_ops.shape(x)
170 | random_tensor = keep_prob
171 | random_tensor += random_ops.random_uniform(noise_shape, seed=seed, dtype=x.dtype)
172 | binary_tensor = math_ops.floor(random_tensor)
173 | ret = x * binary_tensor + alpha * (1 - binary_tensor)
174 |
175 | a = tf.sqrt(fixedPointVar / (keep_prob * ((1 - keep_prob) * tf.pow(alpha - fixedPointMean, 2) + fixedPointVar)))
176 |
177 | b = fixedPointMean - a * (keep_prob * fixedPointMean + (1 - keep_prob) * alpha)
178 | ret = a * ret + b
179 | ret.set_shape(x.get_shape())
180 | return ret
181 |
182 | with ops.name_scope(name, "dropout", [x]) as name:
183 | return utils.smart_cond(training,
184 | lambda: dropout_selu_impl(x, rate, alpha, noise_shape, seed, name),
185 | lambda: array_ops.identity(x))
186 |
--------------------------------------------------------------------------------
/TeLL/scripts/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ml-jku/tensorflow-layer-library/ae2bf342e174e8ad34bb96e4058a18f61f03b13f/TeLL/scripts/__init__.py
--------------------------------------------------------------------------------
/TeLL/scripts/dropoutmask.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Functions for dropout masks
6 |
7 | """
8 |
9 | import tensorflow as tf
10 | import numpy as np
11 | import os
12 | import argparse
13 | from PIL import Image
14 | from TeLL.utility.timer import Timer
15 | import logging
16 |
17 |
18 | def make_ising_mask(shape, keep_prob, num_steps, beta, beta_step=1.01):
19 | """
20 | Create x ising patterns, return 4 x by flipping the patterns
21 | """
22 |
23 | # start off with random multinomial samples in range {-1,1}
24 | samples = 2 * tf.multinomial(tf.log([[1 - keep_prob, keep_prob]]), np.prod(shape)) - 1
25 | samples = tf.to_float(tf.reshape(samples, shape))
26 | ising_filter = tf.constant([[0, 1, 0], [1, 0, 1], [0, 1, 0]], dtype=tf.float32)
27 | ising_filter = tf.reshape(ising_filter, [3, 3, 1, 1])
28 |
29 | i = tf.constant(0)
30 | beta = tf.constant(beta)
31 | beta_step = tf.constant(beta_step)
32 |
33 | condition = lambda i_, beta_, samples_: tf.less(i_, num_steps)
34 |
35 | def body(i_, beta_, samples_):
36 | # only consider a small subset of bits to flip
37 |
38 | # We've got to find the average of the 4 nearest pixels
39 | # Ideas on how to do this:
40 | # - hand written convolutional kernel
41 | # - shift the matrix
42 | conv = tf.nn.conv2d(samples_, ising_filter, [1, 1, 1, 1], 'SAME')
43 |
44 | # Calculate the energy difference if we flip this pixel
45 | # energy = -1 * mul(conv, samples)
46 | # ed = -1 * mul(conv, -1 * samples) + mul(conv, samples) = 2 * mul(conv, samples)
47 | flip_bit = 2 * tf.multiply(conv, samples_)
48 |
49 | # ok, so here if everything is the same [low energy] this guy is really small.
50 | flip_bit = tf.exp(-beta_ * flip_bit)
51 | flip_bit = tf.to_float(tf.greater(flip_bit, tf.random_uniform(shape)))
52 | bits_to_flip = tf.multinomial(tf.log([[9., 1.]]), np.prod(shape))
53 | bits_to_flip = tf.to_float(tf.reshape(bits_to_flip, shape))
54 | flip_bit = tf.to_float(-2 * tf.multiply(flip_bit, bits_to_flip) + 1)
55 | samples_ = tf.multiply(flip_bit, samples_)
56 | if 0:
57 | beta_ = tf.Print(beta_, [tf.reduce_sum(samples_[0, :, :, 0])], message="This is the sum: ")
58 | i_ = tf.add(i_, 1)
59 | beta_ = tf.multiply(beta_, beta_step)
60 | return [i_, beta_, samples_]
61 |
62 | _, _, samples_out = tf.while_loop(condition, body, [i, beta, samples])
63 | samples_out = (samples_out + 1) / 2
64 | return samples_out
65 |
66 |
67 | def boosted_ising_mask(shape, keep_prob, num_steps, beta, beta_step=1.01):
68 | """
69 | Create x ising patterns, return 4 x by flipping the patterns
70 | """
71 |
72 | assert len(shape) == 4
73 | assert shape[0] % 4 == 0
74 | c = shape[3]
75 | shape[3] = 1
76 | if keep_prob == 1:
77 | return tf.constant(1, dtype=tf.float32, shape=shape)
78 |
79 | shape[0] = int(shape[0] / 4)
80 | img_shape = shape[1:]
81 | samples = make_ising_mask(shape, keep_prob, num_steps, beta, beta_step=beta_step)
82 | my_image_list = []
83 |
84 | for s in tf.split(axis=0, num_or_size_splits=shape[0], value=samples):
85 | s = tf.reshape(s, img_shape)
86 | my_image_list.append(s)
87 | my_image_list.append(tf.image.flip_up_down(s))
88 | my_image_list.append(tf.image.flip_left_right(s))
89 | my_image_list.append(tf.image.flip_up_down(my_image_list[-1]))
90 |
91 | for i in range(len(my_image_list)):
92 | my_image_list[i] = tf.reshape(my_image_list[i], [1] + img_shape)
93 | # Create multiple color channels, each with the same values
94 | my_image_list[i] = tf.concat(axis=3, values=c * [my_image_list[i]])
95 |
96 | return tf.concat(axis=0, values=my_image_list)
97 |
98 |
99 | def parse_args():
100 | parser = argparse.ArgumentParser()
101 | # parser.add_argument("--config", type=str, default=None, help="Path to the config file")
102 | parser.add_argument("--path", type=str, default="./", help="Path to output the images")
103 | parser.add_argument("--num_images", type=int, default=4, help="Number of images (must be dividable by 4)")
104 | parser.add_argument("--keep_prob", type=float, default=.5, help="Probability for pattern")
105 | parser.add_argument("--num_steps", type=int, default=400, help="Iterations")
106 | parser.add_argument("--width", type=int, default=2000, help="Width of the picture to create")
107 | parser.add_argument("--height", type=int, default=1000, help="Height of the picture to create")
108 | parser.add_argument("--random_num", type=int, default=12345, help="Random number initialization")
109 | parser.add_argument("--gpu", type=str, default=None, help="CUDA_VISIBLE_DEVICES string. Used for resumed runs.")
110 | args = parser.parse_args()
111 | # --
112 | logging.info("Creating images in {}; keep_prob {}; num_steps {}; gpu {}".format(
113 | args.path, args.keep_prob, args.num_steps, args.gpu))
114 | return args
115 |
116 |
117 | def main():
118 | logging.basicConfig(level=logging.DEBUG, format=' %(asctime)s - %(levelname)s - %(message)s')
119 | args = parse_args()
120 |
121 | out_path = args.path
122 | os.makedirs(out_path, exist_ok=True)
123 |
124 | if not args.num_images % 4 == 0:
125 | raise ValueError("Number of images must be a multiple of 4 due to data augmentation")
126 |
127 | if args.num_images >= 200:
128 | print("Many images are going to be created, an overflow on the GPU could occur")
129 |
130 | if args.gpu is not None:
131 | os.environ['CUDA_VISIBLE_DEVICES'] = args.gpu
132 | else:
133 | os.environ['CUDA_VISIBLE_DEVICES'] = ''
134 |
135 | tf.set_random_seed(args.random_num)
136 |
137 | samples = boosted_ising_mask(shape=[args.num_images, 400, int(400 * (args.width / args.height)), 3],
138 | keep_prob=args.keep_prob, num_steps=args.num_steps, beta=.5, beta_step=1.1)
139 | # samples = tf.image.resize_images(samples, (args.height, args.width), method=tf.image.ResizeMethod.NEAREST_NEIGHBOR)
140 |
141 | logging.info('Samples' + str(samples))
142 |
143 | with Timer(verbose=True, name="Ising") as t:
144 | gpu_options = tf.GPUOptions(allow_growth=True)
145 | sess = tf.Session(config=tf.ConfigProto(gpu_options=gpu_options))
146 | sess.run(tf.global_variables_initializer())
147 | samples_np = sess.run(samples)
148 |
149 |
150 | # Append parameters to log file
151 | params = ""
152 | for arg in vars(args):
153 | if arg not in ["path", "gpu"]:
154 | params += arg + ":" + str(getattr(args, arg)) + "-"
155 |
156 | params = params[:-1]
157 |
158 | logfile = os.path.join(out_path, "log.txt")
159 | if os.path.isfile(logfile):
160 | with open(logfile, "r") as f:
161 | x = f.read().split("\n")
162 | offset = int(x[-1].split("\t")[1])
163 | else:
164 | offset = 0
165 | with open(logfile, "a") as myfile:
166 | myfile.write("\n" + params + "\t" + str(offset + getattr(args, "num_images")))
167 |
168 | for i in range(samples_np.shape[0]):
169 | out = Image.fromarray(samples_np[i, :, :, 0] * 255)
170 | out = out.convert("1")
171 | out.save(out_path + '/image' + '_' + str(offset + i + 1) + '.png')
172 |
173 | sess.close()
174 |
175 |
176 | if __name__ == "__main__":
177 | main()
178 |
--------------------------------------------------------------------------------
/TeLL/scripts/resume.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Script for resuming from saved checkpoint
6 |
7 | """
8 | # ----------------------------------------------------------------------------------------------------------------------
9 | # Imports
10 | # ----------------------------------------------------------------------------------------------------------------------
11 |
12 | import os
13 | import sys
14 | import signal
15 | import subprocess
16 | import argparse
17 | import shlex
18 |
19 | from TeLL.config import Config
20 | from TeLL.utility.misc import extract_to_tmp, rmdir, extract_named_args, extract_unnamed_args
21 |
22 | # ----------------------------------------------------------------------------------------------------------------------
23 | # Globals
24 | # ----------------------------------------------------------------------------------------------------------------------
25 | process_handle = None
26 | working_dir = None
27 | kill_retry_max = 10
28 | kill_retry_count = 0
29 |
30 |
31 | # ----------------------------------------------------------------------------------------------------------------------
32 | # Functions
33 | # ----------------------------------------------------------------------------------------------------------------------
34 |
35 | def read_config(config: Config, epochs: int = None, gpu: str = None):
36 | """Get config either from file or use default config"""
37 | if os.path.isfile(config):
38 | config = Config.from_file(config)
39 | else:
40 | config = Config()
41 |
42 | config.override("n_epochs", epochs)
43 | config.override("cuda_gpu", gpu)
44 |
45 | if epochs is not None:
46 | config.n_epochs = epochs
47 |
48 | return config
49 |
50 |
51 | def parse_args():
52 | parser = argparse.ArgumentParser()
53 | parser.add_argument("path", type=str, default=None,
54 | help="Path to previous working directory of run that should be resumed")
55 | args, unknown = parser.parse_known_args()
56 | if args.path is None:
57 | parser.print_help()
58 | sys.exit(1)
59 | # --
60 | print("Resuming {}".format(args.path))
61 | return args, unknown
62 |
63 |
64 | def parse_and_merge_args(command, override_args):
65 | parts = shlex.split(command)
66 | result = [parts[0]]
67 | orig_args = extract_named_args(parts[1:])
68 | override_args = extract_named_args(override_args)
69 | merged = {**orig_args, **override_args}
70 | for k, v in merged.items():
71 | if v is None:
72 | result.append(k)
73 | else:
74 | result.extend([k, v])
75 | result.extend(extract_unnamed_args(parts[1:]))
76 | return " ".join(result)
77 |
78 |
79 | def resume(directory: str, unknown_args: str = None):
80 | if os.path.isdir(directory):
81 | results = os.path.join(directory, "results")
82 | archive = os.path.join(directory, "00-script.zip")
83 | if os.path.exists(archive) and os.path.exists(results):
84 | global working_dir
85 | working_dir = extract_to_tmp(archive)
86 | # parse used config
87 | with open(os.path.join(working_dir, "00-INFO")) as f:
88 | command = f.readline().strip()
89 | command = parse_and_merge_args(command, unknown_args)
90 | # start
91 | cmd_sep = " &&" if sys.platform == "win32" else "; "
92 | cmd = ["cd \"{}\"{}".format(working_dir, cmd_sep),
93 | '"{}"'.format(sys.executable),
94 | command,
95 | "--restore \"{}\"".format(directory)]
96 | cmd = " ".join(cmd)
97 | print("Resuming with command '{}' in directory '{}'".format(cmd, working_dir))
98 | initial_working_dir = os.getcwd()
99 | os.chdir(working_dir)
100 | global process_handle
101 | process_handle = subprocess.Popen(cmd, cwd=working_dir, shell=True, start_new_session=True)
102 | process_handle.wait()
103 | # clean up
104 | print("Cleaning up temp directory...")
105 | os.chdir(initial_working_dir)
106 | rmdir(working_dir)
107 | print("Done!")
108 | else:
109 | print("Can't resume from {}".format(directory))
110 |
111 |
112 | def sigint_handler(sig, frame):
113 | print("Killing sub-process...")
114 | if process_handle is not None:
115 | global kill_retry_count
116 | while process_handle.returncode is None and kill_retry_count < kill_retry_max:
117 | kill_retry_count += 1
118 | print("Killing sub-process ({})...".format(kill_retry_count))
119 | try:
120 | os.killpg(os.getpgid(process_handle.pid), signal.SIGTERM)
121 | os.waitpid(process_handle.pid, os.WNOHANG)
122 | except ProcessLookupError:
123 | break
124 |
125 | try:
126 | process_handle.wait(1)
127 | except subprocess.TimeoutExpired:
128 | pass
129 |
130 | if working_dir is not None:
131 | rmdir(working_dir)
132 |
133 | sys.exit(0)
134 |
135 |
136 | def main():
137 | signal.signal(signal.SIGINT, sigint_handler)
138 | args, unknown_args = parse_args()
139 |
140 | # If resume option specified resume from snapshot and exit here
141 | if args.path is not None:
142 | resume(args.path, unknown_args)
143 |
144 |
145 | if __name__ == "__main__":
146 | main()
147 |
--------------------------------------------------------------------------------
/TeLL/session.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 | """
4 | import os
5 | import numpy as np
6 | import tensorflow as tf
7 | from TeLL.config import Config
8 | from TeLL.utility.workingdir import Workspace
9 | from TeLL.utility.timer import Timer
10 |
11 |
12 | def set_seed(seed: int = 12345):
13 | tf.set_random_seed(seed)
14 |
15 |
16 | class TeLLSession(object):
17 | def __init__(self, config: Config = None, summaries: list = ["training"], model_params=None):
18 | """
19 | Take care of initializing a TeLL environment.
20 | Creates working directory, instantiates network architecture, configures tensorflow and tensorboard.
21 | Furthermore takes care of resuming runs from an existing workspace.
22 |
23 | :param config: Config
24 | config object or None; if None config will be initialized from command line parameter
25 | :param summaries: list
26 | List of names for summary writers, by default one writer named "training" is opened
27 | :param model_params:
28 | Optional dictionary of parameters unpacked and passed to model upon initialization if not None
29 |
30 | :returns:
31 |
32 | tf_session: Tensorflow session
33 |
34 | tf_saver: Tensorflow checkpoint saver
35 |
36 | tf_summaries: dictionary containing tensorflow summary writers, accessible via the names passed upon creation
37 |
38 | model: TeLL model
39 |
40 | step: current global step (0 for new runs otherwise step stored in checkpoint file)
41 |
42 | workspace: TeLL workspace instance
43 |
44 | config: TeLL config object
45 | """
46 | if config is None:
47 | config = Config()
48 |
49 | # Setup working dir
50 | workspace = Workspace(config.working_dir, config.specs, config.restore)
51 | print("TeLL workspace: {}".format(workspace.working_dir))
52 | # Import configured architecture
53 | architecture = config.import_architecture()
54 | # Set GPU
55 | os.environ["CUDA_VISIBLE_DEVICES"] = str(config.get_value("cuda_gpu", "0"))
56 | # Some Tensorflow configuration
57 | tf_config = tf.ConfigProto(
58 | inter_op_parallelism_threads=config.get_value("inter_op_parallelism_threads", 1),
59 | intra_op_parallelism_threads=config.get_value("intra_op_parallelism_threads", 1),
60 | log_device_placement=config.get_value("log_device_placement", False)
61 | )
62 | tf_config.gpu_options.allow_growth = config.get_value("tf_allow_growth", True)
63 | # Start Tensorflow session
64 | print("Starting session...")
65 | tf_session = tf.Session(config=tf_config)
66 | # Set tensorflow random seed
67 | set_seed(config.get_value("random_seed", 12345))
68 | #
69 | # Init Tensorboard
70 | #
71 | print("Initializing summaries...")
72 | summary_instances = {}
73 | for summary in summaries:
74 | summary_instances[summary] = tf.summary.FileWriter(os.path.join(workspace.get_tensorboard_dir(), summary),
75 | tf_session.graph)
76 | # Initialize Model
77 | if model_params is None:
78 | model = architecture(config=config)
79 | else:
80 | model = architecture(config=config, **model_params)
81 |
82 | # Print number of trainable parameters
83 | trainable_vars = np.sum([np.prod(t.get_shape()) for t in tf.trainable_variables()])
84 | print("Number of trainable variables: {}".format(trainable_vars))
85 |
86 | with tf.name_scope("TeLL") as tell_namescope:
87 | # Store global step in checkpoint
88 | tf_global_step = tf.Variable(initial_value=tf.constant(0, dtype=tf.int64), name="tell_global_step",
89 | dtype=tf.int64, trainable=False)
90 | # Define placeholder and operation to dynamically update tf_global_step with a python integer
91 | global_step_placeholder = tf.placeholder_with_default(tf_global_step, shape=tf_global_step.get_shape())
92 | set_global_step = tf_global_step.assign(global_step_placeholder)
93 |
94 | #
95 | # Add ops to save and restore all the variables
96 | #
97 | tf_saver = tf.train.Saver(max_to_keep=config.get_value("max_checkpoints", 10), sharded=False)
98 | # Expose members
99 | self.tf_session = tf_session
100 | self.tf_saver = tf_saver
101 | self.tf_summaries = summary_instances
102 |
103 | if config.get_value('optimizer', None) is not None:
104 | if isinstance(config.optimizer, list):
105 | self.tf_optimizer = [getattr(tf.train, config.optimizer[i])(**config.optimizer_params[i])
106 | for i in range(len(config.optimizer))]
107 | else:
108 | self.tf_optimizer = getattr(tf.train, config.optimizer)(**config.optimizer_params)
109 |
110 | self.model = model
111 | self.workspace = workspace
112 | self.config = config
113 | self.global_step = 0
114 | self.__global_step_placeholder = global_step_placeholder
115 | self.__global_step_update = set_global_step
116 | self.__tell_namescope = tell_namescope
117 |
118 | def initialize_tf_variables(self, reset_optimizer_on_restore=False):
119 | """
120 | Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint).
121 |
122 | :param reset_optimizer_on_restore: Flag indicating whether to reset the optimizer(s) given that this
123 | function call includes a restore operation.
124 |
125 | :return: updated TeLL session
126 | """
127 |
128 | session = self.tf_session
129 | checkpoint = self.workspace.get_checkpoint()
130 | #
131 | # Initialize or load variables
132 | #
133 | with Timer(name="Initializing variables"):
134 | session.run(tf.global_variables_initializer())
135 | session.run(tf.local_variables_initializer())
136 |
137 | if checkpoint is not None:
138 | # restore from checkpoint
139 | self.tf_saver.restore(session, checkpoint)
140 | # get step number from checkpoint
141 | step = session.run(self.__global_step_placeholder) + 1
142 | self.global_step = step
143 | # reopen summaries
144 | for _, summary in self.tf_summaries.items():
145 | summary.reopen()
146 | summary.add_session_log(tf.SessionLog(status=tf.SessionLog.START), global_step=step)
147 | print("Resuming from checkpoint '{}' at iteration {}".format(checkpoint, step))
148 |
149 | if self.config.get_value('optimizer', None) is not None:
150 | if reset_optimizer_on_restore:
151 | if isinstance(self.tf_optimizer, list):
152 | for optimizer in self.tf_optimizer:
153 | self.reset_optimizer(optimizer)
154 | else:
155 | self.reset_optimizer(self.tf_optimizer)
156 | else:
157 | for _, summary in self.tf_summaries.items():
158 | summary.add_graph(session.graph)
159 |
160 | return self
161 |
162 | def reset_optimizer(self, optimizer, trainables=None):
163 | if trainables is None:
164 | trainables = tf.trainable_variables()
165 |
166 | slots = [optimizer.get_slot(var, name)
167 | for name in optimizer.get_slot_names()
168 | for var in trainables]
169 | self.tf_session.run(tf.initialize_variables(filter(None, slots)))
170 |
171 | def save_checkpoint(self, global_step: int):
172 | """
173 | Store current state in checkpoint.
174 |
175 | :param global_step: current global step variable; has to be stored for resuming later on.
176 | """
177 | self.global_step = global_step
178 | tf_session = self.tf_session
179 | # Update global step variable
180 | tf_session.run(self.__global_step_update, feed_dict={self.__global_step_placeholder: global_step})
181 | # Store checkpoint
182 | self.tf_saver.save(tf_session,
183 | os.path.join(self.workspace.get_result_dir(), "checkpoint-{}.ckpt".format(global_step)))
184 |
185 | def close(self, save_checkpoint: bool = True, global_step: int = 0):
186 | """
187 | Close all tensorflow related stuff and store checkpoint if requested.
188 |
189 | :param save_checkpoint: bool
190 | flag indicating whether current state should be stored in checkpoint
191 | :param global_step: int
192 | if save_checkpoint is True this value is required to store the step in the checkpoint
193 | """
194 | tf_session = self.tf_session
195 | # Close Summarywriters
196 | for _, summary in self.tf_summaries.items():
197 | summary.close()
198 | # Save checkpoint
199 | if save_checkpoint:
200 | self.save_checkpoint(global_step)
201 | # Close session
202 | tf_session.close()
203 | # Close and terminate plotting queue and subprocesses
204 | from TeLL.utility.plotting import terminate_plotting_daemon
205 | terminate_plotting_daemon()
206 |
--------------------------------------------------------------------------------
/TeLL/utility/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ml-jku/tensorflow-layer-library/ae2bf342e174e8ad34bb96e4058a18f61f03b13f/TeLL/utility/__init__.py
--------------------------------------------------------------------------------
/TeLL/utility/gputools.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | """
6 | from tensorflow.python.client import device_lib
7 | import itertools as it
8 |
9 |
10 | class ResourceManager(object):
11 | def __init__(self):
12 | self.devices = device_lib.list_local_devices()
13 | self.cpus = [x.name for x in self.devices if x.device_type == 'CPU']
14 | self.gpus = [x.name for x in self.devices if x.device_type == 'GPU']
15 | self.iterate_cpus = it.cycle(self.cpus)
16 | self.iterate_gpus = it.cycle(self.gpus)
17 |
18 | def next_device(self):
19 | """
20 | Returns the id of the next available device. If GPUs are present will
21 | cycle through GPUs, otherwise returns CPU ids.
22 |
23 | :return: next available device id
24 | """
25 | try:
26 | return self.get_next_gpu()
27 | except StopIteration:
28 | return self.get_next_cpu()
29 |
30 | def get_available_cpus(self):
31 | """
32 | :return: indices of available CPUs
33 | """
34 | return self.cpus
35 |
36 | def get_available_gpus(self):
37 | """
38 | :return: indices of available GPUs
39 | """
40 | return self.gpus
41 |
42 | def get_next_gpu(self):
43 | """
44 | :return: next available gpu id
45 | """
46 | return self.iterate_gpus.__next__()
47 |
48 | def get_next_cpu(self):
49 | """
50 | :return: next available cpu id
51 | """
52 | return self.iterate_cpus.__next__()
53 |
--------------------------------------------------------------------------------
/TeLL/utility/misc.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Misc. objects that do not import tensorflow
6 | """
7 | import argparse
8 | import errno
9 | import importlib
10 | import os
11 | import tempfile
12 | import zipfile
13 | from shutil import copyfile
14 |
15 |
16 | # ----------------------------------------------------------------------------------------------------------------------
17 | # Exceptions
18 | # ----------------------------------------------------------------------------------------------------------------------
19 |
20 | class AbortRun(Exception):
21 | pass
22 |
23 |
24 | # ----------------------------------------------------------------------------------------------------------------------
25 | # Functions
26 | # ----------------------------------------------------------------------------------------------------------------------
27 |
28 | def check_kill_file(workspace):
29 | """raise an AbortRun error if the kill file exists"""
30 | if os.path.isfile(workspace.get_kill_file()):
31 | print("Detected kill file, aborting...")
32 | raise AbortRun
33 |
34 |
35 | def make_sure_path_exists(path):
36 | try:
37 | os.makedirs(path)
38 | except OSError as exception:
39 | if exception.errno != errno.EEXIST:
40 | raise
41 |
42 |
43 | def parse_args():
44 | parser = argparse.ArgumentParser()
45 | parser.add_argument("--config", type=str, default="config.json", help="JSON file with the model params")
46 | parser.add_argument("--restore", type=str, default=None, help=argparse.SUPPRESS)
47 | args, override_args = parser.parse_known_args()
48 | if len(override_args) == 0:
49 | override_args = None
50 | return args, override_args
51 |
52 |
53 | def touch(path):
54 | with open(path, 'a'):
55 | os.utime(path, None)
56 |
57 |
58 | def chmod(path: str, permissions=0o775, recursive=False):
59 | if recursive:
60 | for root, dirs, files in os.walk(path):
61 | for d in dirs:
62 | try:
63 | os.chmod(os.path.join(root, d), permissions)
64 | except PermissionError:
65 | continue
66 | for f in files:
67 | try:
68 | os.chmod(os.path.join(root, f), permissions)
69 | except PermissionError:
70 | continue
71 | else:
72 | os.chmod(path, permissions)
73 |
74 |
75 | def copydir(src: str, dst: str, exclude: list = None):
76 | if os.path.isdir(dst):
77 | rmdir(dst)
78 | for root, dirs, files in os.walk(src):
79 | for file in files:
80 | f = os.path.realpath(os.path.join(root, file))
81 | # don't add files / folders in exclude
82 | exclude_file = False
83 | if exclude:
84 | for entry in exclude:
85 | if os.path.realpath(entry) in f:
86 | exclude_file = True
87 | break
88 |
89 | if not exclude_file:
90 | d = f.replace(os.path.realpath(src), os.path.realpath(dst))
91 | if not os.path.exists(os.path.dirname(d)):
92 | os.makedirs(os.path.dirname(d))
93 | copyfile(f, d)
94 |
95 |
96 | def rmdir(path: str):
97 | if os.path.exists(path):
98 | for root, dirs, files in os.walk(path):
99 | for d in dirs:
100 | current = os.path.join(root, d)
101 | try:
102 | rmdir(current)
103 | except PermissionError:
104 | continue
105 | for f in files:
106 | try:
107 | os.remove(os.path.join(root, f))
108 | except PermissionError:
109 | continue
110 | os.rmdir(path)
111 |
112 |
113 | def zipdir(dir, zip, info=None, exclude=None):
114 | # canonicalize paths
115 | dir = os.path.realpath(dir)
116 | zip = os.path.realpath(zip)
117 | # open zipfile
118 | zipf = zipfile.ZipFile(zip, 'w', zipfile.ZIP_DEFLATED)
119 |
120 | if info:
121 | zipf.writestr("00-INFO", info)
122 |
123 | # add dir
124 | for root, dirs, files in os.walk(dir):
125 | for file in files:
126 | f = os.path.realpath(os.path.join(root, file))
127 | # don't add zip archive to itself
128 | if zip == os.path.join(root, file) or "__pycache__" in f:
129 | continue
130 | # don't add files / folders in exclude
131 | exclude_file = False
132 | if exclude:
133 | for entry in exclude:
134 | if os.path.realpath(entry) in f:
135 | exclude_file = True
136 | break
137 |
138 | if not exclude_file:
139 | zipf.write(filename=f, arcname=f[len(dir):])
140 | zipf.close()
141 |
142 |
143 | def import_object(objname):
144 | objmodule = importlib.import_module(objname.split('.', maxsplit=1)[0])
145 | return get_rec_attr(objmodule, objname.split('.', maxsplit=1)[-1])
146 |
147 |
148 | def get_rec_attr(module, attrstr):
149 | """Get attributes and do so recursively if needed"""
150 | if attrstr is None:
151 | return None
152 | attrs = attrstr.split('.')
153 | for attr in attrs:
154 | module = getattr(module, attr)
155 | return module
156 |
157 |
158 | def load_architecture(arch_name: str = None):
159 | """Import an architecture defined as a class in a file in the current namespace.
160 |
161 | :param arch_name
162 | name of architecture to load; format: .
163 |
164 | :returns architecture handle or none if not found
165 | """
166 | architecture = importlib.import_module(arch_name.split('.', maxsplit=1)[0])
167 | return get_rec_attr(architecture, arch_name.split('.', maxsplit=1)[-1])
168 |
169 |
170 | def extract_to_tmp(file: str):
171 | tempdir = tempfile.mkdtemp("tell")
172 | ziphandle = zipfile.ZipFile(file, "r")
173 | ziphandle.extractall(tempdir)
174 | ziphandle.close()
175 | return tempdir
176 |
177 |
178 | def get_tensor_shape_list(tensor):
179 | """get shape of a tf tensor as list of integers usable as conventional shape"""
180 | return [d if isinstance(d, int) else -1 for d in tensor.shape.as_list()]
181 |
182 |
183 | def load_files_in_dir(directory: str, suffix: str = ''):
184 | """Search for all files in sample_directory (optionally) with suffix sample_suffix an return them as pandas
185 | dataframe
186 |
187 | Parameters
188 | -------
189 | directory : str
190 | Directory to search for files in
191 | suffix : str
192 | If a string is provided as sample_suffix, the file names without this suffix will be ignored
193 |
194 | Returns
195 | -------
196 | : pd.DataFrame
197 | Pandas dataframe with base filenames (=filenames without path or suffix) as keys and full filenames as values;
198 | Dataframe is sorted by full filenames;
199 | """
200 | from os import path
201 | import glob.glob
202 | import pandas as pd
203 |
204 | sample_pattern = "**/*{}".format(suffix)
205 |
206 | # Collect files in path, sort them by name, and store them into dictionary
207 | samples = glob.glob(path.join(directory, sample_pattern))
208 | samples.sort()
209 |
210 | # Extract base filenames without path or suffix and store them as keys for the pandas dataframe
211 | keys = [path.basename(file)[:-len(suffix)] for file in samples]
212 |
213 | # Store in data frame for easy indexing and fast key->value access
214 | samples = pd.DataFrame(index=keys, data=samples)
215 |
216 | return samples
217 |
218 |
219 | def custom_tensorflow_histogram(values, bins=100):
220 | import numpy as np
221 | import tensorflow as tf
222 |
223 | # Create histogram using numpy
224 | counts, bin_edges = np.histogram(values, bins=bins)
225 |
226 | # Fill fields of histogram proto
227 | hist = tf.HistogramProto()
228 | if len(values):
229 | hist.min = float(np.min(values))
230 | hist.max = float(np.max(values))
231 | hist.num = int(np.prod(values.shape))
232 | hist.sum = float(np.sum(values))
233 | hist.sum_squares = float(np.sum(values ** 2))
234 | else:
235 | hist.min = 0
236 | hist.max = 0
237 | hist.num = 0
238 | hist.sum = 0
239 | hist.sum_squares = 0
240 | bin_edges = bin_edges[1:]
241 |
242 | # Add bin edges and counts
243 | for edge in bin_edges:
244 | hist.bucket_limit.append(edge)
245 | for c in counts:
246 | hist.bucket.append(int(c))
247 |
248 | return hist
249 |
250 |
251 | def extract_named_args(arglist):
252 | result = {}
253 | for i in range(0, len(arglist)):
254 | a = arglist[i]
255 | if a.startswith("--"):
256 | if i + 1 < len(arglist) and not arglist[i + 1].startswith("--"):
257 | result[a] = arglist[i + 1]
258 | else:
259 | result[a] = None
260 | return result
261 |
262 |
263 | def extract_unnamed_args(arglist):
264 | result = []
265 | for i in range(1, len(arglist)):
266 | a = arglist[i]
267 | p = arglist[i - 1]
268 | if not a.startswith("--") and not p.startswith("--"):
269 | result.append(a)
270 | return result
271 |
272 |
273 | def try_to_number(value):
274 | try:
275 | value = int(value)
276 | except ValueError:
277 | try:
278 | value = float(value)
279 | except ValueError:
280 | pass
281 | return value
282 |
283 |
284 | # ----------------------------------------------------------------------------------------------------------------------
285 | # Classes
286 | # ----------------------------------------------------------------------------------------------------------------------
287 |
288 | class Tee(object):
289 | """Created from snippets on stackoverflow"""
290 | def __init__(self, original_stdout, *files):
291 | self.files = files
292 | self.original_stdout = original_stdout
293 |
294 | def write(self, obj):
295 | for f in self.files:
296 | f.write(obj)
297 |
298 | def flush(self):
299 | try:
300 | self.original_stdout.flush()
301 | except AttributeError:
302 | # if original_stdout does not support flush()
303 | pass
304 |
--------------------------------------------------------------------------------
/TeLL/utility/misc_tensorflow.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2018
4 |
5 | Misc. objects that import tensorflow
6 | """
7 | import tensorflow as tf
8 | import TeLL.layers
9 | from TeLL.utility.misc import get_rec_attr, import_object
10 |
11 |
12 | # ----------------------------------------------------------------------------------------------------------------------
13 | # Functions
14 | # ----------------------------------------------------------------------------------------------------------------------
15 |
16 | def layers_from_specs(incoming, layerspecs, verbose=True):
17 | """Create a single-branch-network from a list of layer specifications; beta-state convenience function;
18 |
19 | Create a network from a list of layer specifications (dictionaries); does only work for simple
20 | single-branch-networks, for more flexible designs, please create your network directly; layer in list will be
21 | created one-by-one, each with previous layer as input;
22 | If weight initializers ('W' or 'weight_initializer') are iterables with length>1, the first element is treated as
23 | function and the others as arguments for the function; otherwise, weight initializers is treated as function;
24 | This function has not been tested for all layers, use carefully/adapt to your needs;
25 |
26 |
27 | Parameters
28 | ----------
29 | incoming : layer, tensorflow tensor, or placeholder
30 | Incoming layer
31 | layerspecs : list of dicts or dict-likes
32 | Network design as list of dictionaries, where each dict represents the layer parameters as kwargs and requires
33 | the additional key "type" with a string value that is the TeLL layer-class name.
34 |
35 | Returns
36 | ----------
37 | list of TeLL Layer objects
38 | Network as list of connected TeLL layers
39 |
40 | Example
41 | -------
42 | >>> # Creating a network containing a convolutional layer, followed by maxpooling and another convolutional layer
43 | >>> batchsize, x_dim, y_dim, channels = (5, 28, 28, 3)
44 | >>> network_input = tf.placeholder(shape=(batchsize, x_dim, y_dim, channels), dtype=tf.float32)
45 | >>> layerspecs = [{"type": "ConvLayer", "name": "c1", "num_outputs": 32, "ksize": 3, "a": "tensorflow.nn.relu",
46 | >>> "weight_initializer": "0.1:tensorflow.orthogonal_initializer"},
47 | >>> {"type": "MaxPoolingLayer", "name": "mp1", "ksize": [1, 2, 2, 1], "strides": [1, 2, 2, 1]},
48 | >>> {"type": "ConvLayer", "name": "c2", "num_outputs": 64, "ksize": 3, "a": "tensorflow.nn.relu",
49 | >>> "weight_initializer": "tensorflow.orthogonal_initializer"}]
50 | >>> tell_network = layers_from_specs(incoming=network_input, layerspecs=layerspecs)
51 | >>> output_tensor = tell_network[-1].get_output()
52 | """
53 | layers = [incoming]
54 | for l_i, layerspec in enumerate(layerspecs):
55 | layertype = layerspec['type']
56 | if verbose:
57 | print("\t\tbuilding {}...".format(layertype), end='')
58 | layer = get_rec_attr(TeLL.layers, layertype)
59 |
60 | # Replace strings with corresponding functions and remove the "type" field
61 | layerspec = layerspec.copy()
62 | if 'a' in layerspec.keys():
63 | layerspec['a'] = import_object(layerspec['a'])
64 | if 'W' in layerspec.keys():
65 | W = layerspec['W'].split(':')
66 | if len(W) == 1:
67 | layerspec['W'] = import_object(W[0])()
68 | else:
69 | winit_fct = import_object(W[-1])
70 | winit_args = [float(a) for a in W[:-1]]
71 | layerspec['W'] = winit_fct(*winit_args)
72 |
73 | if 'weight_initializer' in layerspec.keys():
74 | weight_initializer = layerspec['weight_initializer'].split(':')
75 | if len(weight_initializer) == 1:
76 | layerspec['weight_initializer'] = import_object(weight_initializer[0])()
77 | else:
78 | winit_fct = import_object(weight_initializer[-1])
79 | winit_args = [float(a) for a in weight_initializer[:-1]]
80 | layerspec['weight_initializer'] = winit_fct(*winit_args)
81 |
82 | if layerspec.get('name', None) is None:
83 | layerspec['name'] = "{}_{}".format(l_i, layerspec['type'])
84 | del layerspec['type']
85 |
86 | layers.append(layer(layers[-1], **layerspec))
87 | if verbose:
88 | print(" in {} / out {}".format(layers[-2].get_output_shape(), layers[-1].get_output_shape()))
89 | return layers[1:]
90 |
91 |
92 | def tensor_shape_with_flexible_dim(tensor, dim):
93 | """Return shape of tensor with dimension dim as a flexible dimension
94 |
95 | Parameters
96 | ----------
97 | tensor : tensor
98 | Tensor of which to return shape with a flexible dimension
99 |
100 | dim : int
101 | Index of dimension to replace with a flexible dimension
102 | """
103 | shape_list = tensor.shape.as_list()
104 | shape_list = [s if s_i != dim else None for s_i, s in enumerate(shape_list)]
105 | return tf.TensorShape(shape_list)
106 |
107 |
108 | # ----------------------------------------------------------------------------------------------------------------------
109 | # Classes
110 | # ----------------------------------------------------------------------------------------------------------------------
111 |
112 | class TriangularValueEncoding(object):
113 | def __init__(self, max_value: int, triangle_span: int):
114 | """Encodes an integer value with range [0, max_value] as multiple activations between 0 and 1 via triangles of
115 | width triangle_span;
116 |
117 | LSTM profits from having an integer input with large range split into multiple input nodes; This class encodes
118 | an integer as multiple nodes with activations of range [0,1]; Each node represents a triangle of width
119 | triangle_span; These triangles are distributed equidistantly over the integer range such that 2 triangles
120 | overlap by 1/2 width and the whole integer range is covered; For each integer to encode, the high of the
121 | triangle at this integer position is taken as node activation, i.e. max. 2 nodes have an activation > 0 for each
122 | integer value;
123 |
124 | Values are encoded via encode_value(value) and returned as float32 tensorflow tensor of length self.n_nodes;
125 |
126 | Parameters
127 | ----------
128 | max_value : int
129 | Maximum value to encode
130 | triangle_span : int
131 | Width of each triangle
132 | """
133 | # round max_value up to a multiple of triangle_span
134 | if max_value % triangle_span != 0:
135 | max_value = ((max_value // triangle_span) + 1) * triangle_span
136 |
137 | # Calculate number of overlapping triangle nodes
138 | n_nodes_half = int(max_value / triangle_span)
139 | n_nodes = n_nodes_half * 2 + 1
140 |
141 | # Template for tensor
142 | coding = tf.zeros((n_nodes,), dtype=tf.float32, name='ingametime')
143 |
144 | self.n_nodes_python = n_nodes
145 | self.n_nodes = tf.constant(n_nodes, dtype=tf.int32)
146 | self.n_nodes_half = tf.constant(n_nodes_half, dtype=tf.int32)
147 | self.max_value = tf.constant(max_value, dtype=tf.int32)
148 | self.triangle_span = tf.constant(triangle_span, dtype=tf.int32)
149 | self.triangle_span_float = tf.constant(triangle_span, dtype=tf.float32)
150 | self.half_triangle_span = tf.cast(self.triangle_span / 2, dtype=tf.int32)
151 | self.half_triangle_span_float = tf.cast(self.triangle_span / 2, dtype=tf.float32)
152 | self.coding = coding
153 |
154 | def encode_value(self, value):
155 | """Encode value as multiple triangle node activations
156 |
157 | Parameters
158 | ----------
159 | value : int tensor
160 | Value to encode as integer tensorflow tensor
161 |
162 | Returns
163 | ----------
164 | float32 tensor
165 | Encoded value as float32 tensor of length self.n_nodes
166 | """
167 | value_float = tf.cast(value, dtype=tf.float32)
168 |
169 | index = tf.cast(value / self.triangle_span, dtype=tf.int32)
170 | act = (tf.constant(0.5, dtype=tf.float32)
171 | - (tf.mod(tf.abs(value_float - self.half_triangle_span_float), self.triangle_span_float)
172 | / self.triangle_span_float)) * tf.constant(2, dtype=tf.float32)
173 | coding1 = tf.one_hot(index, on_value=act, depth=self.n_nodes, dtype=tf.float32)
174 |
175 | index = tf.cast((value + self.half_triangle_span) / self.triangle_span, dtype=tf.int32) + self.n_nodes_half
176 | act = (tf.mod(tf.abs(value_float - self.half_triangle_span_float), self.triangle_span_float)
177 | / self.triangle_span_float) * tf.constant(2, dtype=tf.float32)
178 | coding2 = tf.one_hot(index, on_value=act, depth=self.n_nodes, dtype=tf.float32)
179 |
180 | return coding1 + coding2
181 |
--------------------------------------------------------------------------------
/TeLL/utility/plotting_daemons.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Handling of worker subprocesses and threads for plotting
6 |
7 | """
8 |
9 | from multiprocess import Pool
10 |
11 |
12 | def stop_plotting_daemon(plotting_queue, plotting_proc):
13 | """End plotting daemon process properly"""
14 | # tell process to finish
15 | plotting_queue.put(0)
16 | # reap process
17 | plotting_proc.join()
18 |
19 |
20 | def launch_proc(target, arguments, wait=False, daemon=True):
21 | import multiprocessing as mp
22 | proc = mp.Process(target=target, args=arguments)
23 | proc.daemon = daemon
24 | proc.start()
25 | if wait:
26 | proc.wait()
27 | return proc
28 |
29 |
30 | def plotting_demon(plotting_queue, multicore):
31 | print("Starting plotting daemon...", end=" ")
32 | pool = Pool(processes=multicore)
33 | print("Done!")
34 | while True:
35 | rec = plotting_queue.get()
36 | if rec == 0:
37 | break
38 | func, arguments = rec
39 | if isinstance(arguments, tuple):
40 | pool.apply_async(func, *arguments)
41 | else:
42 | pool.apply_async(func, arguments)
43 |
44 | pool.close()
45 | pool.join()
46 | del pool
47 | print("Plotting daemon terminated.")
48 | exit(0)
49 |
50 |
51 | def start_plotting_daemon(wait=False, multicore=3):
52 | """ Launch programms with plotting_queue.put(function, (args, kwargs))"""
53 | import multiprocessing as mp
54 | plotting_queue = mp.Queue()
55 | proc = launch_proc(target=plotting_demon, arguments=[plotting_queue, multicore], daemon=False, wait=wait)
56 | return (plotting_queue, proc)
--------------------------------------------------------------------------------
/TeLL/utility/timer.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | """
6 | import time
7 |
8 |
9 | class Timer(object):
10 | def __init__(self, name="", verbose=True, precision='msec'):
11 | self.verbose = verbose
12 | self.name = name
13 | self.precision = precision
14 | self.start = time.time()
15 | self.end = self.start
16 | self.secs = 0
17 | self.msecs = 0
18 |
19 | def __enter__(self):
20 | self.start = time.time()
21 | return self
22 |
23 | def __exit__(self, *args):
24 | if self.verbose:
25 | self.print()
26 |
27 | def print(self):
28 | self.end = time.time()
29 | self.secs = self.end - self.start
30 | self.msecs = self.secs * 1000 # millisecs
31 | if self.precision == 'msec':
32 | print('Timer ({0}): {1:.2f} ms'.format(self.name, self.msecs))
33 | else:
34 | print('Timer ({0}): {1:.3f} s'.format(self.name, self.secs))
35 |
--------------------------------------------------------------------------------
/TeLL/utility/workingdir.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | """
6 | import os
7 | import sys
8 | import datetime as dt
9 | import glob
10 | import tempfile
11 | import TeLL
12 | from natsort import natsorted
13 | from TeLL.utility.misc import make_sure_path_exists, touch, zipdir, chmod, copydir, rmdir
14 | import __main__
15 |
16 |
17 | class Workspace(object):
18 | def __init__(self, workspace: str, specs: str, restore: str = None):
19 | """
20 |
21 | :param workspace: str
22 | path to general workspace directory
23 | :param specs: str
24 | short description of specs for a specific test run;
25 | will be part of the run specific working directory so don't use spaces or special chars
26 | """
27 | self.workspace = os.path.realpath(workspace)
28 | self.specs = specs
29 |
30 | if restore is None:
31 | self.timestamp = dt.datetime.now().strftime("%Y-%m-%dT%H-%M-%S")
32 | self.working_dir, self.result_dir, self.tensorboard_dir, self.kill_file, self.plot_file, self.checkpoint = self.__setup_working_dir__()
33 | else:
34 | self.working_dir, self.result_dir, self.tensorboard_dir, self.kill_file, self.plot_file, self.checkpoint = self.__resume_from_dir__(
35 | restore)
36 |
37 | def get_result_dir(self):
38 | return self.result_dir
39 |
40 | def get_tensorboard_dir(self):
41 | return self.tensorboard_dir
42 |
43 | def get_kill_file(self):
44 | return self.kill_file
45 |
46 | def get_plot_file(self):
47 | return self.plot_file
48 |
49 | def get_timestamp(self):
50 | return self.timestamp
51 |
52 | def get_checkpoint(self):
53 | return self.checkpoint
54 |
55 | def __setup_working_dir__(self):
56 | # fix permissions of workspace root
57 | make_sure_path_exists(self.workspace)
58 | try:
59 | chmod(self.workspace, 0o775)
60 | except PermissionError:
61 | print("PermissionError when trying to change permissions of workspace to 775")
62 |
63 | # setup working directory
64 | specs_dir = os.path.realpath("{}/{}".format(self.workspace, self.specs))
65 | working_dir = os.path.realpath("{}/{}".format(specs_dir, self.timestamp))
66 | # Set up result folder structure
67 | results_path = "{}/results".format(working_dir, self.timestamp)
68 | make_sure_path_exists(results_path)
69 |
70 | # Set up tensorboard directory
71 | tensorboard = "{}/tensorboard".format(working_dir, self.timestamp)
72 | make_sure_path_exists(tensorboard)
73 |
74 | # set path to kill file (if this file exists abort run)
75 | kill_file_name = "ABORT_RUN"
76 | kill_file = os.path.join(working_dir, kill_file_name)
77 |
78 | # create plot file to plot by default
79 | plot_file_name = "PLOT_ON"
80 | plot_file = os.path.join(working_dir, plot_file_name)
81 | touch(plot_file)
82 |
83 | # remove kill file before starting the run (should not exist anyway)
84 | if os.path.isfile(kill_file):
85 | os.remove(kill_file)
86 |
87 | # fix permissions to grant group write access (to allow kill_file creation and plot control)
88 | try:
89 | chmod(self.workspace, 0o775, recursive=False)
90 | chmod(specs_dir, 0o775, recursive=False)
91 | chmod(working_dir, 0o775, recursive=True)
92 | chmod(plot_file, 0o664)
93 | except PermissionError:
94 | print("PermissionError when trying to change permissions of workspace to 775")
95 |
96 | # compress and copy current script and dependencies to results dir
97 | command = " ".join(sys.argv)
98 | # copy current code to temp dir
99 | script_dir = os.path.dirname(os.path.realpath(__main__.__file__))
100 | tempdir = tempfile.mkdtemp("tell")
101 | copydir(script_dir, tempdir,
102 | exclude=[self.workspace, os.path.join(script_dir, ".git"), os.path.join(script_dir, ".idea"),
103 | os.path.join(script_dir, "__pycache__")])
104 | # also copy currently used TeLL library so it can be used for resuming runs
105 | copydir(TeLL.__path__[0], os.path.join(tempdir, os.path.basename(TeLL.__path__[0])))
106 | rmdir(os.path.join(os.path.join(tempdir, os.path.basename(TeLL.__path__[0])), "__pycache__"))
107 | zipdir(dir=tempdir, zip=os.path.join(working_dir, '00-script.zip'), info=command,
108 | exclude=[self.workspace, '.git'])
109 | rmdir(tempdir)
110 | return [working_dir, results_path, tensorboard, kill_file, plot_file, None]
111 |
112 | def __resume_from_dir__(self, dir):
113 | # setup working directory
114 | working_dir = os.path.realpath(dir)
115 | self.timestamp = os.path.basename(working_dir)
116 |
117 | # Set up result folder structure
118 | results_path = "{}/results".format(working_dir, self.timestamp)
119 |
120 | # Set up tensorboard directory
121 | tensorboard = "{}/tensorboard".format(working_dir, self.timestamp)
122 |
123 | # set path to kill file (if this file exists abort run)
124 | kill_file_name = "ABORT_RUN"
125 | kill_file = os.path.join(working_dir, kill_file_name)
126 | if os.path.exists(kill_file):
127 | os.remove(kill_file)
128 |
129 | # create plot file to plot by default
130 | plot_file_name = "PLOT_ON"
131 | plot_file = os.path.join(working_dir, plot_file_name)
132 |
133 | if not (os.path.exists(results_path) or not os.path.exists(tensorboard)):
134 | raise Exception("can not resume from given directory")
135 |
136 | checkpoints = glob.glob1(results_path, "*.ckpt*")
137 | checkpoints = [checkpoint for checkpoint in checkpoints if
138 | not (".meta" in checkpoint or ".index" in checkpoint)]
139 | checkpoints = natsorted(checkpoints)
140 | checkpoint = os.path.join(results_path, checkpoints[-1])
141 |
142 | if not os.path.exists(checkpoint):
143 | raise Exception("could not find checkpoint in given directory")
144 |
145 | if not checkpoint.endswith("ckpt"):
146 | checkpoint = checkpoint[:checkpoint.index(".ckpt.") + 5]
147 |
148 | return [working_dir, results_path, tensorboard, kill_file, plot_file, checkpoint]
149 |
--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ml-jku/tensorflow-layer-library/ae2bf342e174e8ad34bb96e4058a18f61f03b13f/__init__.py
--------------------------------------------------------------------------------
/samples/config_convlstm.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "ArchitectureConvLSTM",
3 | "cuda_gpu": "0",
4 | "architecture": "sample_architectures.ArchitectureConvLSTM",
5 | "dataset": {
6 | "reader": "TeLL.datareaders.MovingDots",
7 | "train": {
8 | "n_timesteps": 5,
9 | "n_samples": 50,
10 | "random_seed": 12345
11 | },
12 | "val": {
13 | "n_timesteps": 5,
14 | "n_samples": 25,
15 | "random_seed": 54321
16 | }
17 | },
18 | "n_epochs": 100,
19 | "score_at": 1000,
20 | "plot_at": 50,
21 | "batchsize": 10,
22 | "optimizer_params": {
23 | "learning_rate": 1e-2
24 | },
25 | "optimizer": "AdamOptimizer",
26 | "l1": 1e-6,
27 | "l2": 1e-5,
28 | "n_lstm": 3,
29 | "kernel_lstm_fwd": 5,
30 | "kernel_lstm_bwd": 5,
31 | "reduced_rec_lstm": false,
32 | "tickersteps": 3,
33 | "lstm_act": "nn.elu",
34 | "store_states": true,
35 | "kernel_conv": 3,
36 | "working_dir": "working_dir"
37 | }
--------------------------------------------------------------------------------
/samples/config_convlstm_mnist.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "ArchitectureConvLSTM",
3 | "cuda_gpu": "0",
4 | "architecture": "sample_architectures.ArchitectureConvLSTMMNIST",
5 | "dataset": {
6 | "reader": "TeLL.datareaders.MovingMNIST",
7 | "scaling_factor": [
8 | 1.0,
9 | 1.1
10 | ],
11 | "scaling_velocity": [
12 | 0,
13 | 0.1
14 | ],
15 | "velocity": [
16 | 5.6,
17 | 14
18 | ],
19 | "rotation_angle": [
20 | 0,
21 | 360
22 | ],
23 | "rotation_velocity": [
24 | 0,
25 | 5
26 | ],
27 | "n_objects": 2,
28 | "train": {
29 | "dset": "train",
30 | "n_timesteps": 20,
31 | "n_samples": 5000,
32 | "random_seed": 12345
33 | },
34 | "val": {
35 | "dset": "validation",
36 | "n_timesteps": 20,
37 | "n_samples": 250,
38 | "random_seed": 54321
39 | }
40 | },
41 | "n_epochs": 100,
42 | "score_at": 1000,
43 | "plot_at": 50,
44 | "batchsize": 5,
45 | "optimizer_params": {
46 | "learning_rate": 1e-4
47 | },
48 | "optimizer": "AdamOptimizer",
49 | "l1": 1e-6,
50 | "l2": 1e-5,
51 | "enc_dec_depth": 2,
52 | "n_lstm": 64,
53 | "kernel_lstm_fwd": 3,
54 | "kernel_lstm_bwd": 3,
55 | "forgetgate": true,
56 | "lstm_output_dropout": false,
57 | "reduced_rec_lstm": false,
58 | "tickersteps": 0,
59 | "lstm_act": "nn.elu",
60 | "store_states": true,
61 | "kernel_conv": 3,
62 | "kernel_conv_out": 1,
63 | "working_dir": "working_dir"
64 | }
--------------------------------------------------------------------------------
/samples/config_dense.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "ArchitectureDense",
3 | "cuda_gpu": "0",
4 | "architecture": "sample_architectures.ArchitectureDense",
5 | "dataset": {
6 | "reader": "TeLL.datareaders.ShortLongDataset",
7 | "train": {
8 | "n_timesteps": 250,
9 | "n_samples": 3000,
10 | "random_seed": 12345
11 | },
12 | "val": {
13 | "n_timesteps": 250,
14 | "n_samples": 300,
15 | "random_seed": 54321
16 | }
17 | },
18 | "n_epochs": 100,
19 | "score_at": 1000,
20 | "batchsize": 15,
21 | "optimizer_params": {
22 | "learning_rate": 1e-3
23 | },
24 | "optimizer": "AdamOptimizer",
25 | "l1": 1e-6,
26 | "l2": 1e-5,
27 | "n_dense": 25,
28 | "working_dir": "working_dir"
29 | }
--------------------------------------------------------------------------------
/samples/config_lstm.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "ArchitectureLSTM",
3 | "cuda_gpu": "0",
4 | "architecture": "sample_architectures.ArchitectureLSTM",
5 | "dataset": {
6 | "reader": "TeLL.datareaders.ShortLongDataset",
7 | "train": {
8 | "n_timesteps": 250,
9 | "n_samples": 3000,
10 | "random_seed": 12345
11 | },
12 | "val": {
13 | "n_timesteps": 250,
14 | "n_samples": 300,
15 | "random_seed": 54321
16 | }
17 | },
18 | "n_epochs": 100,
19 | "score_at": 1000,
20 | "batchsize": 15,
21 | "optimizer_params": {
22 | "learning_rate": 1e-3
23 | },
24 | "optimizer": "AdamOptimizer",
25 | "l1": 1e-6,
26 | "l2": 1e-5,
27 | "n_lstm": 5,
28 | "tickersteps": 0,
29 | "working_dir": "working_dir"
30 | }
--------------------------------------------------------------------------------
/samples/config_lstm3.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "ArchitectureLSTM3",
3 | "cuda_gpu": "0",
4 | "architecture": "sample_architectures.ArchitectureLSTM3",
5 | "n_epochs": 100,
6 | "dataset": {
7 | "reader": "TeLL.datareaders.ShortLongDataset",
8 | "train": {
9 | "n_timesteps": 250,
10 | "n_samples": 3000,
11 | "random_seed": 12345
12 | },
13 | "val": {
14 | "n_timesteps": 250,
15 | "n_samples": 300,
16 | "random_seed": 54321
17 | }
18 | },
19 | "score_at": 1000,
20 | "batchsize": 15,
21 | "optimizer_params": {
22 | "learning_rate": 1e-3
23 | },
24 | "optimizer": "AdamOptimizer",
25 | "l1": 1e-6,
26 | "l2": 1e-5,
27 | "n_lstm": 5,
28 | "tickersteps": 0,
29 | "n_dense_units": [
30 | 20,
31 | 20
32 | ],
33 | "working_dir": "working_dir"
34 | }
--------------------------------------------------------------------------------
/samples/main_convlstm.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Main file for simple convLSTM example
6 |
7 | Main file for simple convLSTM example to be used with convLSTM architecture in sample_architectures and config file
8 | config_convlstm.json; Plots input, output, weights, ConvLSTM output (of 2 ConvLSTM units), and cell states in
9 | working_dir;
10 |
11 | Goal: Given a video sequence with moving dots, predict 1 frame into the future after the sequence end
12 |
13 | """
14 | # ----------------------------------------------------------------------------------------------------------------------
15 | # Imports
16 | # ----------------------------------------------------------------------------------------------------------------------
17 |
18 | #
19 | # Imports before spawning workers (do NOT import tensorflow or matplotlib here)
20 | #
21 | import os
22 | import sys
23 | from collections import OrderedDict
24 |
25 | import numpy as np
26 | import progressbar
27 |
28 | from TeLL.config import Config
29 | from TeLL.dataprocessing import Normalize
30 | from TeLL.datareaders import initialize_datareaders, DataLoader
31 | from TeLL.utility.misc import AbortRun, check_kill_file
32 | from TeLL.utility.plotting import launch_plotting_daemon, save_subplots
33 | from TeLL.utility.timer import Timer
34 | from TeLL.utility.workingdir import Workspace
35 |
36 | if __name__ == "__main__":
37 | #
38 | # Start subprocess for plotting workers
39 | # Due to a garbage-collector bug with matplotlib/GPU, launch_plotting_daemon needs so be called before tensorflow
40 | # import
41 | #
42 | launch_plotting_daemon(num_workers=3)
43 | from TeLL.session import TeLLSession
44 |
45 | import tensorflow as tf
46 | from TeLL.regularization import regularize
47 |
48 |
49 | # ----------------------------------------------------------------------------------------------------------------------
50 | # Functions
51 | # ----------------------------------------------------------------------------------------------------------------------
52 |
53 | def update_step(loss, config, clip_gradient=1., scope='optimizer'):
54 | """Computation of gradients and application of weight updates
55 |
56 | Optimizer can be supplied via config file, e.g. as
57 | "optimizer_params": {"learning_rate": 1e-3},
58 | "optimizer": "'AdamOptimizer'"
59 |
60 | Parameters
61 | -------
62 | loss : tensor
63 | Tensor representing the
64 | config : config file
65 | Configuration file
66 | clip_gradient : (positive) float or False
67 | Clip gradient at +/- clip_gradient or don't clip gradient if clip_gradient=False
68 |
69 | Returns
70 | -------
71 | : tensor
72 | Application of gradients via optimizer
73 | """
74 | # Define weight update
75 | with tf.variable_scope(scope):
76 | trainables = tf.trainable_variables()
77 | # Set optimizer (one learning rate for all layers)
78 | optimizer = getattr(tf.train, config.optimizer)(**config.optimizer_params)
79 |
80 | # Calculate gradients
81 | gradients = tf.gradients(loss, trainables)
82 | # Clip all gradients
83 | if clip_gradient:
84 | gradients = [tf.clip_by_value(grad, -clip_gradient, clip_gradient) for grad in gradients]
85 | # Set and return weight update
86 | return optimizer.apply_gradients(zip(gradients, trainables))
87 |
88 |
89 | def evaluate_on_validation_set(validationset, step: int, session, model, summary_writer, validation_summary,
90 | val_loss, workspace: Workspace):
91 | """Convenience function for evaluating network on a validation set
92 |
93 | Parameters
94 | -------
95 | validationset : dataset reader
96 | Dataset reader for the validation set
97 | step : int
98 | Current step in training
99 | session : tf.session
100 | Tensorflow session to use
101 | model : network model
102 | Network model
103 | val_loss : tensor
104 | Tensor representing the validation loss computation
105 |
106 | Returns
107 | -------
108 | : float
109 | Loss averaged over validation set
110 | """
111 | loss = 0
112 |
113 | _pbw = ['Evaluating on validation set: ', progressbar.ETA()]
114 | progress = progressbar.ProgressBar(widgets=_pbw, maxval=validationset.n_mbs - 1, redirect_stdout=True).start()
115 |
116 | mb_validation = validationset.batch_loader()
117 |
118 | with Timer(verbose=True, name="Evaluate on Validation Set"):
119 | for mb_i, mb in enumerate(mb_validation):
120 | # Abort if indicated by file
121 | check_kill_file(workspace)
122 |
123 | val_summary, cur_loss = session.run([validation_summary, val_loss],
124 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
125 |
126 | loss += cur_loss
127 | progress.update(mb_i)
128 |
129 | mb.clear()
130 | del mb
131 |
132 | progress.finish()
133 |
134 | avg_loss = loss / validationset.n_mbs
135 |
136 | summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="Validation Loss", simple_value=avg_loss)]),
137 | step)
138 |
139 | print("Validation scores:\n\tstep {} validation loss {}".format(step, avg_loss))
140 | sys.stdout.flush()
141 |
142 | return avg_loss
143 |
144 |
145 | def main(_):
146 | # ------------------------------------------------------------------------------------------------------------------
147 | # Setup training
148 | # ------------------------------------------------------------------------------------------------------------------
149 |
150 | # Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
151 | config = Config()
152 |
153 | #
154 | # Prepare input data
155 | #
156 |
157 | # Make sure datareader is reproducible
158 | random_seed = config.get_value('random_seed', 12345)
159 | np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
160 | rnd_gen = np.random.RandomState(seed=random_seed)
161 |
162 | # Set datareaders
163 | readers = initialize_datareaders(config, required=("train", "val"))
164 |
165 | # Set Preprocessing
166 | trainingset = Normalize(readers["train"], apply_to=['X'])
167 | validationset = Normalize(readers["val"], apply_to=['X'])
168 |
169 | # Set minibatch loaders
170 | trainingset = DataLoader(trainingset, batchsize=50, batchsize_method='zeropad', verbose=False)
171 | validationset = DataLoader(validationset, batchsize=50, batchsize_method='zeropad', verbose=False)
172 |
173 | #
174 | # Initialize TeLL session
175 | #
176 | tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": trainingset})
177 |
178 | # Get some members from the session for easier usage
179 | sess = tell.tf_session
180 | summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
181 | model = tell.model
182 | workspace, config = tell.workspace, tell.config
183 |
184 | #
185 | # Define loss functions and update steps
186 | #
187 | print("Initializing loss calculation...")
188 | y_shape = trainingset.get_input_shapes()["y"].shape
189 | pos_target_weight = np.prod(y_shape[2:]) - 1 # only 1 pixel per sample is of positive class -> up-weight!
190 | loss = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(targets=model.y_, logits=model.output,
191 | pos_weight=pos_target_weight))
192 | train_summary = tf.summary.scalar("Training Loss", loss) # create summary to add to tensorboard
193 |
194 | # Loss function for validationset
195 | val_loss = tf.reduce_mean(tf.nn.weighted_cross_entropy_with_logits(targets=model.y_, logits=model.output,
196 | pos_weight=pos_target_weight))
197 | val_loss_summary = tf.summary.scalar("Validation Loss", val_loss) # create summary to add to tensorboard
198 |
199 | # Regularization
200 | reg_penalty = regularize(layers=model.get_layers(), l1=config.l1, l2=config.l2,
201 | regularize_weights=True, regularize_biases=True)
202 | regpen_summary = tf.summary.scalar("Regularization Penalty", reg_penalty) # create summary to add to tensorboard
203 |
204 | # Update step for weights
205 | update = update_step(loss + reg_penalty, config)
206 |
207 | #
208 | # Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
209 | #
210 | global_step = tell.initialize_tf_variables().global_step
211 |
212 | #
213 | # Set up plotting
214 | # (store tensors we want to plot in a dictionary for easier tensor-evaluation)
215 | #
216 | # We want to plot input, output and target for the 1st sample, last frame, and 1st channel in subplot 1
217 | tensors_subplot1 = OrderedDict()
218 | tensors_subplot1['input'] = model.X[0, -1, :, :, 0]
219 | tensors_subplot1['target'] = model.y_[0, -1, :, :, 0]
220 | tensors_subplot1['network_output'] = model.output[0, -1, :, :, 0]
221 | # We also want to plot the cell states and hidden states for each frame (again of the 1st sample and 1st lstm unit)
222 | # in subplot 2 and 3
223 | tensors_subplot2 = OrderedDict()
224 | tensors_subplot3 = OrderedDict()
225 | for frame in range(len(model.lstm_layer.c)):
226 | tensors_subplot2['hiddenstate_{}'.format(frame)] = model.lstm_layer.h[frame][0, :, :, 0]
227 | tensors_subplot3['cellstate_{}'.format(frame)] = model.lstm_layer.c[frame][0, :, :, 0]
228 | # Create a list to store all symbolic tensors for plotting
229 | plotting_tensors = list(tensors_subplot1.values()) + list(tensors_subplot2.values()) + \
230 | list(tensors_subplot3.values())
231 |
232 | #
233 | # Finalize graph
234 | # This makes our tensorflow graph read-only and prevents further additions to the graph
235 | #
236 | sess.graph.finalize()
237 | if sess.graph.finalized:
238 | print("Graph is finalized!")
239 | else:
240 | raise ValueError("Could not finalize graph!")
241 |
242 | sys.stdout.flush()
243 |
244 | # ------------------------------------------------------------------------------------------------------------------
245 | # Start training
246 | # ------------------------------------------------------------------------------------------------------------------
247 |
248 | try:
249 | epoch = int(global_step / trainingset.n_mbs)
250 | epochs = range(epoch, config.n_epochs)
251 |
252 | # Loop through epochs
253 | print("Starting training")
254 |
255 | for ep in epochs:
256 | epoch = ep
257 | print("Starting training epoch: {}".format(ep))
258 | # Initialize variables for over-all loss per epoch
259 | train_loss = 0
260 |
261 | # Load one minibatch at a time and perform a training step
262 | t_mb = Timer(verbose=True, name="Load Minibatch")
263 | mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
264 |
265 | #
266 | # Loop through minibatches
267 | #
268 | for mb_i, mb in enumerate(mb_training):
269 | sys.stdout.flush()
270 | # Print minibatch load time
271 | t_mb.print()
272 |
273 | # Abort if indicated by file
274 | check_kill_file(workspace)
275 |
276 | #
277 | # Calculate scores on validation set
278 | #
279 | if global_step % config.score_at == 0:
280 | print("Starting scoring on validation set...")
281 | evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
282 | val_loss_summary, val_loss, workspace)
283 |
284 | #
285 | # Perform weight updates and do plotting
286 | #
287 | if (mb_i % config.plot_at) == 0 and os.path.isfile(workspace.get_plot_file()):
288 | # Perform weight update, return summary values and values for plotting
289 | with Timer(verbose=True, name="Weight Update"):
290 | plotting_values = []
291 | train_summ, regpen_summ, _, cur_loss, *plotting_values = sess.run(
292 | [train_summary, regpen_summary, update, loss, *plotting_tensors],
293 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
294 |
295 | # Add current summary values to tensorboard
296 | summary_writer_train.add_summary(train_summ, global_step=global_step)
297 | summary_writer_train.add_summary(regpen_summ, global_step=global_step)
298 |
299 | # Create and save subplot 1 (input, target, output)
300 | save_subplots(images=plotting_values[:len(tensors_subplot1)],
301 | subfigtitles=list(tensors_subplot1.keys()),
302 | subplotranges=[None, (0, 1), (0, 1)], colorbar=True, automatic_positioning=True,
303 | filename=os.path.join(workspace.get_result_dir(),
304 | "output_ep{}_mb{}.png".format(ep, mb_i)))
305 | del plotting_values[:len(tensors_subplot1)]
306 |
307 | # Create and save subplot 2 (hidden states, i.e. ConvLSTM outputs)
308 | save_subplots(images=plotting_values[:len(tensors_subplot2)],
309 | subfigtitles=list(tensors_subplot2.keys()),
310 | title='ConvLSTM hidden states (outputs)', colorbar=True, automatic_positioning=True,
311 | filename=os.path.join(workspace.get_result_dir(),
312 | "hidden_ep{}_mb{}.png".format(ep, mb_i)))
313 | del plotting_values[:len(tensors_subplot2)]
314 |
315 | # Create and save subplot 3 (cell states)
316 | save_subplots(images=plotting_values[:len(tensors_subplot3)],
317 | subfigtitles=list(tensors_subplot3.keys()),
318 | title='ConvLSTM cell states', colorbar=True, automatic_positioning=True,
319 | filename=os.path.join(workspace.get_result_dir(),
320 | "cell_ep{}_mb{}.png".format(ep, mb_i)))
321 | del plotting_values[:len(tensors_subplot3)]
322 |
323 | else:
324 | #
325 | # Perform weight update without plotting
326 | #
327 | with Timer(verbose=True, name="Weight Update"):
328 | train_summ, regpen_summ, _, cur_loss = sess.run([train_summary, regpen_summary, update, loss],
329 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
330 |
331 | # Add current summary values to tensorboard
332 | summary_writer_train.add_summary(train_summ, global_step=global_step)
333 | summary_writer_train.add_summary(regpen_summ, global_step=global_step)
334 |
335 | # Add current loss to running average loss
336 | train_loss += cur_loss
337 |
338 | # Print some status info
339 | print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
340 |
341 | # Reset timer
342 | t_mb = Timer(name="Load Minibatch")
343 |
344 | # Free the memory allocated for the minibatch data
345 | mb.clear()
346 | del mb
347 |
348 | global_step += 1
349 |
350 | #
351 | # Calculate scores on validation set
352 | #
353 |
354 | # Perform scoring on validation set
355 | print("Starting scoring on validation set...")
356 | evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
357 | val_loss_summary, val_loss, workspace)
358 |
359 | # Save the model
360 | tell.save_checkpoint(global_step=global_step)
361 |
362 | # Abort if indicated by file
363 | check_kill_file(workspace)
364 |
365 | except AbortRun:
366 | print("Detected kill file, aborting...")
367 |
368 | finally:
369 | #
370 | # If the program executed correctly or an error was raised, close the data readers and save the model and exit
371 | #
372 | trainingset.close()
373 | validationset.close()
374 | tell.close(save_checkpoint=True, global_step=global_step)
375 |
376 |
377 | if __name__ == "__main__":
378 | tf.app.run()
379 |
--------------------------------------------------------------------------------
/samples/main_convlstm_mnist.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Main file for simple convLSTM example
6 |
7 | Main file for simple convLSTM example to be used with convLSTM architecture in sample_architectures and config file
8 | config_convlstm.json; Plots input, output, weights, ConvLSTM output (of 2 ConvLSTM units), and cell states in
9 | working_dir;
10 |
11 | """
12 | # ----------------------------------------------------------------------------------------------------------------------
13 | # Imports
14 | # ----------------------------------------------------------------------------------------------------------------------
15 |
16 | #
17 | # Imports before spawning workers (do NOT import tensorflow or matplotlib here)
18 | #
19 | import os
20 | import sys
21 | from collections import OrderedDict
22 |
23 | import numpy as np
24 | import progressbar
25 |
26 | from TeLL.config import Config
27 | from TeLL.dataprocessing import Normalize
28 | from TeLL.datareaders import initialize_datareaders, DataLoader
29 | from TeLL.utility.misc import AbortRun, check_kill_file
30 | from TeLL.utility.plotting import launch_plotting_daemon, save_subplots
31 | from TeLL.utility.timer import Timer
32 | from TeLL.utility.workingdir import Workspace
33 |
34 | if __name__ == "__main__":
35 | #
36 | # Start subprocess for plotting workers
37 | # Due to a garbage-collector bug with matplotlib/GPU, launch_plotting_daemon needs so be called before tensorflow
38 | # import
39 | #
40 | launch_plotting_daemon(num_workers=3)
41 |
42 | from TeLL.session import TeLLSession
43 |
44 | import tensorflow as tf
45 | from TeLL.regularization import regularize
46 | from TeLL.loss import image_crossentropy
47 |
48 |
49 | # ----------------------------------------------------------------------------------------------------------------------
50 | # Functions
51 | # ----------------------------------------------------------------------------------------------------------------------
52 |
53 | def update_step(loss, config, clip_gradient=1., scope='optimizer'):
54 | """Computation of gradients and application of weight updates
55 |
56 | Optimizer can be supplied via config file, e.g. as
57 | "optimizer_params": {"learning_rate": 1e-3},
58 | "optimizer": "'AdamOptimizer'"
59 |
60 | Parameters
61 | -------
62 | loss : tensor
63 | Tensor representing the
64 | config : config file
65 | Configuration file
66 | clip_gradient : (positive) float or False
67 | Clip gradient at +/- clip_gradient or don't clip gradient if clip_gradient=False
68 |
69 | Returns
70 | -------
71 | : tensor
72 | Application of gradients via optimizer
73 | """
74 | # Define weight update
75 | with tf.variable_scope(scope):
76 | trainables = tf.trainable_variables()
77 | # Set optimizer (one learning rate for all layers)
78 | optimizer = getattr(tf.train, config.optimizer)(**config.optimizer_params)
79 |
80 | # Calculate gradients
81 | gradients = tf.gradients(loss, trainables)
82 | # Clip all gradients
83 | if clip_gradient:
84 | gradients = [tf.clip_by_value(grad, -clip_gradient, clip_gradient) for grad in gradients]
85 | # Set and return weight update
86 | return optimizer.apply_gradients(zip(gradients, trainables))
87 |
88 |
89 | def evaluate_on_validation_set(validationset, step: int, session, model, summary_writer, validation_summary,
90 | val_loss, workspace: Workspace):
91 | """Convenience function for evaluating network on a validation set
92 |
93 | Parameters
94 | -------
95 | validationset : dataset reader
96 | Dataset reader for the validation set
97 | step : int
98 | Current step in training
99 | session : tf.session
100 | Tensorflow session to use
101 | model : network model
102 | Network model
103 | val_loss : tensor
104 | Tensor representing the validation loss computation
105 |
106 | Returns
107 | -------
108 | : float
109 | Loss averaged over validation set
110 | """
111 | loss = 0
112 |
113 | _pbw = ['Evaluating on validation set: ', progressbar.ETA()]
114 | progress = progressbar.ProgressBar(widgets=_pbw, maxval=validationset.n_mbs - 1, redirect_stdout=True).start()
115 |
116 | mb_validation = validationset.batch_loader()
117 |
118 | with Timer(verbose=True, name="Evaluate on Validation Set"):
119 | for mb_i, mb in enumerate(mb_validation):
120 | # Abort if indicated by file
121 | check_kill_file(workspace)
122 |
123 | val_summary, cur_loss = session.run([validation_summary, val_loss],
124 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
125 |
126 | loss += cur_loss
127 | progress.update(mb_i)
128 |
129 | mb.clear()
130 | del mb
131 |
132 | progress.finish()
133 |
134 | avg_loss = loss / validationset.n_mbs
135 |
136 | summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="Validation Loss", simple_value=avg_loss)]),
137 | step)
138 |
139 | print("Validation scores:\n\tstep {} validation loss {}".format(step, avg_loss))
140 | sys.stdout.flush()
141 |
142 | return avg_loss
143 |
144 |
145 | def main(_):
146 | # ------------------------------------------------------------------------------------------------------------------
147 | # Setup training
148 | # ------------------------------------------------------------------------------------------------------------------
149 |
150 | # Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
151 | config = Config()
152 |
153 | #
154 | # Prepare input data
155 | #
156 |
157 | # Make sure datareader is reproducible
158 | random_seed = config.get_value('random_seed', 12345)
159 | np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
160 | rnd_gen = np.random.RandomState(seed=random_seed)
161 |
162 | # Set datareaders
163 | n_timesteps = config.get_value('mnist_n_timesteps', 20)
164 |
165 | # Load datasets for trainingset
166 | with Timer(name="Loading Data"):
167 | readers = initialize_datareaders(config, required=("train", "val"))
168 |
169 | # Set Preprocessing
170 | trainingset = Normalize(readers["train"], apply_to=['X', 'y'])
171 | validationset = Normalize(readers["val"], apply_to=['X', 'y'])
172 |
173 | # Set minibatch loaders
174 | trainingset = DataLoader(trainingset, batchsize=2, batchsize_method='zeropad', verbose=False)
175 | validationset = DataLoader(validationset, batchsize=2, batchsize_method='zeropad', verbose=False)
176 |
177 | #
178 | # Initialize TeLL session
179 | #
180 | tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": trainingset})
181 |
182 | # Get some members from the session for easier usage
183 | sess = tell.tf_session
184 | summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
185 | model = tell.model
186 | workspace, config = tell.workspace, tell.config
187 |
188 | #
189 | # Define loss functions and update steps
190 | #
191 | print("Initializing loss calculation...")
192 | loss, _ = image_crossentropy(target=model.y_[:, 10:, :, :], pred=model.output[:, 10:, :, :, :],
193 | pixel_weights=model.pixel_weights[:, 10:, :, :], reduce_by='mean')
194 | train_summary = tf.summary.scalar("Training Loss", loss) # create summary to add to tensorboard
195 |
196 | # Loss function for validationset
197 | val_loss = loss
198 | val_loss_summary = tf.summary.scalar("Validation Loss", val_loss) # create summary to add to tensorboard
199 |
200 | # Regularization
201 | reg_penalty = regularize(layers=model.get_layers(), l1=config.l1, l2=config.l2,
202 | regularize_weights=True, regularize_biases=True)
203 | regpen_summary = tf.summary.scalar("Regularization Penalty", reg_penalty) # create summary to add to tensorboard
204 |
205 | # Update step for weights
206 | update = update_step(loss + reg_penalty, config)
207 |
208 | #
209 | # Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
210 | #
211 | global_step = tell.initialize_tf_variables().global_step
212 |
213 | #
214 | # Set up plotting
215 | # (store tensors we want to plot in a dictionary for easier tensor-evaluation)
216 | #
217 | # We want to plot input, output and target for the 1st sample, 1st frame, and 1st channel in subplot 1
218 | tensors_subplot1 = OrderedDict()
219 | tensors_subplot2 = OrderedDict()
220 | tensors_subplot3 = OrderedDict()
221 | for frame in range(n_timesteps):
222 | tensors_subplot1['input_{}'.format(frame)] = model.X[0, frame, :, :]
223 | tensors_subplot2['target_{}'.format(frame)] = model.y_[0, frame, :, :] - 1
224 | tensors_subplot3['network_output_{}'.format(frame)] = tf.argmax(model.output[0, frame, :, :, :], axis=-1) - 1
225 | # We also want to plot the cell states and hidden states for each frame (again of the 1st sample and 1st lstm unit)
226 | # in subplot 2 and 3
227 | tensors_subplot4 = OrderedDict()
228 | tensors_subplot5 = OrderedDict()
229 | for frame in range(len(model.lstm_layer.c)):
230 | tensors_subplot4['hiddenstate_{}'.format(frame)] = model.lstm_layer.h[frame][0, :, :, 0]
231 | tensors_subplot5['cellstate_{}'.format(frame)] = model.lstm_layer.c[frame][0, :, :, 0]
232 | # Create a list to store all symbolic tensors for plotting
233 | plotting_tensors = list(tensors_subplot1.values()) + list(tensors_subplot2.values()) + \
234 | list(tensors_subplot3.values()) + list(tensors_subplot4.values()) + \
235 | list(tensors_subplot5.values())
236 |
237 | #
238 | # Finalize graph
239 | # This makes our tensorflow graph read-only and prevents further additions to the graph
240 | #
241 | sess.graph.finalize()
242 | if sess.graph.finalized:
243 | print("Graph is finalized!")
244 | else:
245 | raise ValueError("Could not finalize graph!")
246 |
247 | sys.stdout.flush()
248 |
249 | # ------------------------------------------------------------------------------------------------------------------
250 | # Start training
251 | # ------------------------------------------------------------------------------------------------------------------
252 |
253 | try:
254 | epoch = int(global_step / trainingset.n_mbs)
255 | epochs = range(epoch, config.n_epochs)
256 |
257 | # Loop through epochs
258 | print("Starting training")
259 |
260 | for ep in epochs:
261 | epoch = ep
262 | print("Starting training epoch: {}".format(ep))
263 | # Initialize variables for over-all loss per epoch
264 | train_loss = 0
265 |
266 | # Load one minibatch at a time and perform a training step
267 | t_mb = Timer(verbose=True, name="Load Minibatch")
268 | mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
269 |
270 | #
271 | # Loop through minibatches
272 | #
273 | for mb_i, mb in enumerate(mb_training):
274 | sys.stdout.flush()
275 | # Print minibatch load time
276 | t_mb.print()
277 |
278 | # Abort if indicated by file
279 | check_kill_file(workspace)
280 |
281 | #
282 | # Calculate scores on validation set
283 | #
284 | if global_step % config.score_at == 0:
285 | print("Starting scoring on validation set...")
286 | evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
287 | val_loss_summary, val_loss, workspace)
288 |
289 | #
290 | # Perform weight updates and do plotting
291 | #
292 | if (mb_i % config.plot_at) == 0 and os.path.isfile(workspace.get_plot_file()):
293 | # Perform weight update, return summary values and values for plotting
294 | with Timer(verbose=True, name="Weight Update"):
295 | plotting_values = []
296 | train_summ, regpen_summ, _, cur_loss, *plotting_values = sess.run(
297 | [train_summary, regpen_summary, update, loss, *plotting_tensors],
298 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
299 |
300 | # Add current summary values to tensorboard
301 | summary_writer_train.add_summary(train_summ, global_step=global_step)
302 | summary_writer_train.add_summary(regpen_summ, global_step=global_step)
303 |
304 | # Create and save subplot 1 (input)
305 | save_subplots(images=plotting_values[:len(tensors_subplot1)],
306 | subfigtitles=list(tensors_subplot1.keys()),
307 | subplotranges=[(0, 1)] * n_timesteps, colorbar=True, automatic_positioning=True,
308 | tight_layout=True,
309 | filename=os.path.join(workspace.get_result_dir(),
310 | "input_ep{}_mb{}.png".format(ep, mb_i)))
311 | del plotting_values[:len(tensors_subplot1)]
312 |
313 | # Create and save subplot 2 (target)
314 | save_subplots(images=plotting_values[:len(tensors_subplot2)],
315 | subfigtitles=list(tensors_subplot2.keys()),
316 | subplotranges=[(0, 10) * n_timesteps], colorbar=True, automatic_positioning=True,
317 | tight_layout=True,
318 | filename=os.path.join(workspace.get_result_dir(),
319 | "target_ep{}_mb{}.png".format(ep, mb_i)))
320 | del plotting_values[:len(tensors_subplot2)]
321 |
322 | # Create and save subplot 3 (output)
323 | save_subplots(images=plotting_values[:len(tensors_subplot3)],
324 | subfigtitles=list(tensors_subplot3.keys()),
325 | # subplotranges=[(0, 10)] * n_timesteps,
326 | colorbar=True, automatic_positioning=True,
327 | tight_layout=True,
328 | filename=os.path.join(workspace.get_result_dir(),
329 | "output_ep{}_mb{}.png".format(ep, mb_i)))
330 | del plotting_values[:len(tensors_subplot3)]
331 |
332 | # Create and save subplot 2 (hidden states, i.e. ConvLSTM outputs)
333 | save_subplots(images=plotting_values[:len(tensors_subplot4)],
334 | subfigtitles=list(tensors_subplot4.keys()),
335 | title='ConvLSTM hidden states (outputs)', colorbar=True, automatic_positioning=True,
336 | tight_layout=True,
337 | filename=os.path.join(workspace.get_result_dir(),
338 | "hidden_ep{}_mb{}.png".format(ep, mb_i)))
339 | del plotting_values[:len(tensors_subplot4)]
340 |
341 | # Create and save subplot 3 (cell states)
342 | save_subplots(images=plotting_values[:len(tensors_subplot5)],
343 | subfigtitles=list(tensors_subplot5.keys()),
344 | title='ConvLSTM cell states', colorbar=True, automatic_positioning=True,
345 | tight_layout=True,
346 | filename=os.path.join(workspace.get_result_dir(),
347 | "cell_ep{}_mb{}.png".format(ep, mb_i)))
348 | del plotting_values[:len(tensors_subplot5)]
349 |
350 | else:
351 | #
352 | # Perform weight update without plotting
353 | #
354 | with Timer(verbose=True, name="Weight Update"):
355 | train_summ, regpen_summ, _, cur_loss = sess.run([
356 | train_summary, regpen_summary, update, loss],
357 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
358 |
359 | # Add current summary values to tensorboard
360 | summary_writer_train.add_summary(train_summ, global_step=global_step)
361 | summary_writer_train.add_summary(regpen_summ, global_step=global_step)
362 |
363 | # Add current loss to running average loss
364 | train_loss += cur_loss
365 |
366 | # Print some status info
367 | print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
368 |
369 | # Reset timer
370 | t_mb = Timer(name="Load Minibatch")
371 |
372 | # Free the memory allocated for the minibatch data
373 | mb.clear()
374 | del mb
375 |
376 | global_step += 1
377 |
378 | #
379 | # Calculate scores on validation set
380 | #
381 |
382 | # Perform scoring on validation set
383 | print("Starting scoring on validation set...")
384 | evaluate_on_validation_set(validationset, global_step, sess, model, summary_writer_validation,
385 | val_loss_summary, val_loss, workspace)
386 |
387 | # Save the model
388 | tell.save_checkpoint(global_step=global_step)
389 |
390 | # Abort if indicated by file
391 | check_kill_file(workspace)
392 |
393 | except AbortRun:
394 | print("Detected kill file, aborting...")
395 |
396 | finally:
397 | #
398 | # If the program executed correctly or an error was raised, close the data readers and save the model and exit
399 | #
400 | trainingset.close()
401 | validationset.close()
402 | tell.close(save_checkpoint=True, global_step=global_step)
403 |
404 |
405 | if __name__ == "__main__":
406 | tf.app.run()
407 |
--------------------------------------------------------------------------------
/samples/main_lstm.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | """
3 | © Michael Widrich, Markus Hofmarcher, 2017
4 |
5 | Main file for LSTM and dense layer example
6 |
7 | Main file for LSTM example to be used with LSTM architecture in sample_architectures and config file config_lstm.json;
8 | Also to be used with other LSTM example architectures and a dense layer architecture (see sample_architectures.py for
9 | the different examples and descriptions);
10 |
11 | """
12 | # ----------------------------------------------------------------------------------------------------------------------
13 | # Imports
14 | # ----------------------------------------------------------------------------------------------------------------------
15 |
16 | #
17 | # Imports before spawning workers (do NOT import tensorflow or matplotlib here)
18 | #
19 | import sys
20 |
21 | import numpy as np
22 | import progressbar
23 |
24 | # Import TeLL
25 | from TeLL.config import Config
26 | from TeLL.datareaders import initialize_datareaders, DataLoader
27 | from TeLL.utility.misc import AbortRun, check_kill_file
28 | from TeLL.utility.timer import Timer
29 | from TeLL.utility.workingdir import Workspace
30 |
31 | if __name__ == "__main__":
32 | from TeLL.session import TeLLSession
33 |
34 | import tensorflow as tf
35 | from TeLL.regularization import regularize
36 |
37 |
38 | # ----------------------------------------------------------------------------------------------------------------------
39 | # Functions
40 | # ----------------------------------------------------------------------------------------------------------------------
41 |
42 | def update_step(loss, config, clip_gradient=1., scope='optimizer'):
43 | """Computation of gradients and application of weight updates
44 |
45 | Optimizer can be supplied via config file, e.g. as
46 | "optimizer_params": {"learning_rate": 1e-3},
47 | "optimizer": "'AdamOptimizer'"
48 |
49 | Parameters
50 | -------
51 | loss : tensor
52 | Tensor representing the
53 | config : config file
54 | Configuration file
55 | clip_gradient : (positive) float or False
56 | Clip gradient at +/- clip_gradient or don't clip gradient if clip_gradient=False
57 |
58 | Returns
59 | -------
60 | : tensor
61 | Application of gradients via optimizer
62 | """
63 | # Define weight update
64 | with tf.variable_scope(scope):
65 | trainables = tf.trainable_variables()
66 | # Set optimizer (one learning rate for all layers)
67 | optimizer = getattr(tf.train, config.optimizer)(**config.optimizer_params)
68 |
69 | # Calculate gradients
70 | gradients = tf.gradients(loss, trainables)
71 | # Clip all gradients
72 | if clip_gradient:
73 | gradients = [tf.clip_by_value(grad, -clip_gradient, clip_gradient) for grad in gradients]
74 | # Set and return weight update
75 | return optimizer.apply_gradients(zip(gradients, trainables))
76 |
77 |
78 | def evaluate_on_validation_set(validationset, step: int, session, model, summary_writer, validation_summary,
79 | val_loss, workspace: Workspace):
80 | """Convenience function for evaluating network on a validation set
81 |
82 | Parameters
83 | -------
84 | validationset : dataset reader
85 | Dataset reader for the validation set
86 | step : int
87 | Current step in training
88 | session : tf.session
89 | Tensorflow session to use
90 | model : network model
91 | Network model
92 | val_loss : tensor
93 | Tensor representing the validation loss computation
94 |
95 | Returns
96 | -------
97 | : float
98 | Loss averaged over validation set
99 | """
100 | loss = 0
101 |
102 | _pbw = ['Evaluating on validation set: ', progressbar.ETA()]
103 | progress = progressbar.ProgressBar(widgets=_pbw, maxval=validationset.n_mbs - 1, redirect_stdout=True).start()
104 |
105 | mb_validation = validationset.batch_loader()
106 |
107 | with Timer(verbose=True, name="Evaluate on Validation Set"):
108 | for mb_i, mb in enumerate(mb_validation):
109 | # Abort if indicated by file
110 | check_kill_file(workspace)
111 |
112 | val_summary, cur_loss = session.run([validation_summary, val_loss],
113 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
114 |
115 | loss += cur_loss
116 | progress.update(mb_i)
117 |
118 | mb.clear()
119 | del mb
120 |
121 | progress.finish()
122 |
123 | avg_loss = loss / validationset.n_mbs
124 |
125 | summary_writer.add_summary(tf.Summary(value=[tf.Summary.Value(tag="Validation Loss", simple_value=avg_loss)]),
126 | step)
127 |
128 | print("Validation scores:\n\tstep {} validation loss {}".format(step, avg_loss))
129 | sys.stdout.flush()
130 |
131 | return avg_loss
132 |
133 |
134 | def main(_):
135 | # ------------------------------------------------------------------------------------------------------------------
136 | # Setup training
137 | # ------------------------------------------------------------------------------------------------------------------
138 |
139 | # Initialize config, parses command line and reads specified file; also supports overriding of values from cmd
140 | config = Config()
141 |
142 | random_seed = config.get_value('random_seed', 12345)
143 | np.random.seed(random_seed) # not threadsafe, use rnd_gen object where possible
144 | rnd_gen = np.random.RandomState(seed=random_seed)
145 |
146 | # Load datasets for trainingset
147 | with Timer(name="Loading Data"):
148 | readers = initialize_datareaders(config, required=("train", "val"))
149 | trainingset = DataLoader(readers["train"], batchsize=config.batchsize)
150 | validationset = DataLoader(readers["val"], batchsize=config.batchsize)
151 |
152 | # Initialize TeLL session
153 | tell = TeLLSession(config=config, summaries=["train"], model_params={"dataset": trainingset})
154 |
155 | # Get some members from the session for easier usage
156 | session = tell.tf_session
157 | summary_writer = tell.tf_summaries["train"]
158 | model = tell.model
159 | workspace, config = tell.workspace, tell.config
160 |
161 | # Loss function for trainingset
162 | print("Initializing loss calculation...")
163 | loss = tf.reduce_mean(tf.square(model.y_ - model.output))
164 | train_summary = tf.summary.scalar("Training Loss", loss) # add loss to tensorboard
165 |
166 | # Loss function for validationset
167 | val_loss = tf.reduce_mean(tf.square(model.y_ - model.output))
168 | val_loss_summary = tf.summary.scalar("Validation Loss", val_loss) # add val_loss to tensorboard
169 |
170 | # Regularization
171 | reg_penalty = regularize(layers=model.get_layers(), l1=config.l1, l2=config.l2,
172 | regularize_weights=True, regularize_biases=True)
173 | regpen_summary = tf.summary.scalar("Regularization Penalty", reg_penalty) # add reg_penalty to tensorboard
174 |
175 | # Update step for weights
176 | update = update_step(loss + reg_penalty, config)
177 |
178 | # Initialize Tensorflow variables
179 | global_step = tell.initialize_tf_variables().global_step
180 |
181 | sys.stdout.flush()
182 |
183 | # ------------------------------------------------------------------------------------------------------------------
184 | # Start training
185 | # ------------------------------------------------------------------------------------------------------------------
186 |
187 | try:
188 | epoch = int(global_step / trainingset.n_mbs)
189 | epochs = range(epoch, config.n_epochs)
190 |
191 | #
192 | # Loop through epochs
193 | #
194 | print("Starting training")
195 |
196 | for ep in epochs:
197 | print("Starting training epoch: {}".format(ep))
198 | # Initialize variables for over-all loss per epoch
199 | train_loss = 0
200 |
201 | # Load one minibatch at a time and perform a training step
202 | t_mb = Timer(name="Load Minibatch")
203 | mb_training = trainingset.batch_loader(rnd_gen=rnd_gen)
204 |
205 | #
206 | # Loop through minibatches
207 | #
208 | for mb_i, mb in enumerate(mb_training):
209 | sys.stdout.flush()
210 | # Print minibatch load time
211 | t_mb.print()
212 |
213 | # Abort if indicated by file
214 | check_kill_file(workspace)
215 |
216 | #
217 | # Calculate scores on validation set
218 | #
219 | if global_step % config.score_at == 0:
220 | print("Starting scoring on validation set...")
221 | evaluate_on_validation_set(validationset, global_step, session, model, summary_writer,
222 | val_loss_summary, val_loss, workspace)
223 |
224 | #
225 | # Perform weight update
226 | #
227 | with Timer(name="Weight Update"):
228 | train_summ, regpen_summ, _, cur_loss = session.run(
229 | [train_summary, regpen_summary, update, loss],
230 | feed_dict={model.X: mb['X'], model.y_: mb['y']})
231 |
232 | # Add current summary values to tensorboard
233 | summary_writer.add_summary(train_summ, global_step=global_step)
234 | summary_writer.add_summary(regpen_summ, global_step=global_step)
235 |
236 | # Add current loss to running average loss
237 | train_loss += cur_loss
238 |
239 | # Print some status info
240 | print("ep {} mb {} loss {} (avg. loss {})".format(ep, mb_i, cur_loss, train_loss / (mb_i + 1)))
241 |
242 | # Reset timer
243 | t_mb = Timer(name="Load Minibatch")
244 |
245 | # Free the memory allocated for the minibatch data
246 | mb.clear()
247 | del mb
248 |
249 | global_step += 1
250 |
251 | #
252 | # Calculate scores on validation set after training is done
253 | #
254 |
255 | # Perform scoring on validation set
256 | print("Starting scoring on validation set...")
257 | evaluate_on_validation_set(validationset, global_step, session, model, summary_writer, val_loss_summary,
258 | val_loss, workspace)
259 |
260 | tell.save_checkpoint(global_step=global_step)
261 |
262 | # Abort if indicated by file
263 | check_kill_file(workspace)
264 |
265 | except AbortRun:
266 | print("Detected kill file, aborting...")
267 |
268 | finally:
269 | tell.close(save_checkpoint=True, global_step=global_step)
270 |
271 |
272 | if __name__ == "__main__":
273 | tf.app.run()
274 |
--------------------------------------------------------------------------------
/samples/mnist/architectures.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 |
4 | """
5 | import tensorflow as tf
6 | from TeLL.layers import DenseLayer, DropoutLayer, ConvLayer, MaxPoolingLayer
7 | from TeLL.initializations import weight_truncated_normal, weight_gauss_conv2d
8 | from TeLL.config import Config
9 |
10 |
11 | class DenseNet(object):
12 | def __init__(self, config: Config):
13 | # Network Parameters
14 | n_input = 784 # MNIST data input (img shape: 28*28)
15 | n_classes = 10 # MNIST total classes (0-9 digits)
16 | n_hidden_1 = config.n_hidden_1
17 | n_hidden_2 = config.n_hidden_2
18 |
19 | # tf Graph input
20 | X = tf.placeholder(tf.float32, [None, n_input], name="Features")
21 | y_ = tf.placeholder(tf.float32, [None, n_classes], name="Labels")
22 | d = tf.placeholder(tf.float32)
23 |
24 | #X_2d = tf.reshape(X, [-1, 28, 28, 1])
25 |
26 | #hidden1 = ConvLayer(X_2d, W=weight_gauss_conv2d([3, 3, 1, n_hidden_1]), name="ConvLayer1")
27 | #maxpool1 = MaxPoolingLayer(hidden1, name="MaxPoolingLayer1")
28 | #hidden2 = ConvLayer(maxpool1, W=weight_gauss_conv2d([3, 3, n_hidden_1, n_hidden_2]), name="ConvLayer2")
29 | #maxpool2 = MaxPoolingLayer(hidden2, name="MaxPoolingLayer2")
30 | #flat = tf.contrib.layers.flatten(maxpool2.get_output())
31 |
32 | # Hidden 1
33 | hidden1 = DenseLayer(X, n_hidden_1, name="Hidden_Layer_1",
34 | a=tf.nn.sigmoid, W=weight_truncated_normal, b=tf.zeros)
35 | # Hidden 2
36 | hidden2 = DenseLayer(hidden1, n_hidden_2, name="Hidden_Layer_2",
37 | a=tf.nn.sigmoid, W=weight_truncated_normal, b=tf.zeros)
38 |
39 | # Output
40 | out = DenseLayer(hidden2, n_classes, name="Output_Layer",
41 | a=tf.identity, W=weight_truncated_normal, b=tf.zeros)
42 |
43 | self.X = X
44 | self.y_ = y_
45 | self.dropout = d
46 | self.hidden1 = hidden1
47 | self.hidden2 = hidden2
48 | self.output = out.get_output()
49 |
--------------------------------------------------------------------------------
/samples/mnist/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "mnist_dense_w0.125",
3 | "entropy_w": 0.125,
4 | "cuda_gpu": "0",
5 | "architecture": "architectures.DenseNet",
6 | "optimizer": "AdamOptimizer",
7 | "optimizer_params": {
8 | "learning_rate": 1e-3
9 | },
10 | "iterations": 10000,
11 | "batchsize": 256,
12 | "display_step": 100,
13 | "dropout": 0.0,
14 | "n_hidden_1": 800,
15 | "n_hidden_2": 800,
16 | "score_at": 1000,
17 | "plot_at": 20,
18 | "tf_allow_growth": true,
19 | "working_dir": "working_dir"
20 | }
21 |
--------------------------------------------------------------------------------
/samples/mnist/main.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 |
4 | Example for mnist predictions via dense network
5 |
6 | Command-line usage:
7 | >>> python3 samples/mnist/main_convlstm.py --config=samples/mnist/config.json
8 | """
9 | # Import MNIST data
10 | from tensorflow.examples.tutorials.mnist import input_data
11 |
12 | # Import TeLL
13 | from TeLL.config import Config
14 | from TeLL.session import TeLLSession
15 | from TeLL.utility.timer import Timer
16 | from TeLL.utility.misc import AbortRun, check_kill_file
17 | from TeLL.regularization import decor_penalty
18 |
19 | # Import Tensorflow
20 | if __name__ == "__main__":
21 | import tensorflow as tf
22 |
23 |
24 | # ----------------------------------------------------------------------------------------------------------------------
25 | # Functions
26 | # ----------------------------------------------------------------------------------------------------------------------
27 |
28 | def main(_):
29 | config = Config()
30 | # Create new TeLL session with two summary writers
31 | tell = TeLLSession(config=config, summaries=["train", "validation"])
32 |
33 | # Get some members from the session for easier usage
34 | session = tell.tf_session
35 | summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
36 | model = tell.model
37 | workspace, config = tell.workspace, tell.config
38 |
39 | # Parameters
40 | learning_rate = config.get_value("learning_rate", 1e-3)
41 | iterations = config.get_value("iterations", 1000)
42 | batchsize = config.get_value("batchsize", 250)
43 | display_step = config.get_value("display_step", 10)
44 | dropout = config.get_value("dropout_prob", 0.25)
45 |
46 | #
47 | # Load Data
48 | #
49 | with Timer(name="Load data"):
50 | mnist = input_data.read_data_sets("../MNIST_data", one_hot=True)
51 |
52 | # Define loss and optimizer
53 | with tf.name_scope("Cost"):
54 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model.output, labels=model.y_))
55 |
56 | ##entropy = tf.reduce_mean(tf.contrib.bayesflow.entropy.entropy_shannon(
57 | ## tf.contrib.distributions.Categorical(p=tf.nn.softmax(logits=model.output))))
58 |
59 | probs = tf.nn.softmax(logits=model.output)
60 | entropy = tf.reduce_mean(-tf.reduce_sum(tf.log(tf.maximum(probs, 1e-15)) * probs, 1))
61 |
62 | # test decor regularization
63 | #decor_penalty(model.hidden1, model.y_, 10, [1], 0.)
64 | #decor_penalty(model.hidden2, model.y_, 10, [1], 0.)
65 |
66 | optimizer = tell.tf_optimizer.minimize(cost - config.get_value("entropy_w", 0.) * entropy)
67 |
68 | tf.summary.scalar("Loss", cost)
69 | #tf.summary.scalar("Decor", decor1 + decor2)
70 | #tf.summary.scalar("Entropy", entropy)
71 | tf.summary.scalar("O-Prob", tf.reduce_mean(tf.reduce_sum(tf.nn.softmax(logits=model.output) * model.y_, 1)))
72 |
73 | # Evaluate model
74 | with tf.name_scope("Accuracy"):
75 | correct_pred = tf.equal(tf.argmax(model.output, 1), tf.argmax(model.y_, 1))
76 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
77 | tf.summary.scalar("Accuracy", accuracy)
78 |
79 | merged_summaries = tf.summary.merge_all()
80 |
81 | # Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
82 | step = tell.initialize_tf_variables(reset_optimizer_on_restore=True).global_step
83 |
84 | # -------------------------------------------------------------------------
85 | # Start training
86 | # -------------------------------------------------------------------------
87 | acc_train = 0.
88 | val_acc_best = 0.
89 | try:
90 | while step < iterations:
91 | check_kill_file(workspace=workspace)
92 | batch_x, batch_y = mnist.train.next_batch(batchsize)
93 |
94 | i = step * batchsize
95 | if step % display_step == 0:
96 | summary, acc = session.run([merged_summaries, accuracy],
97 | feed_dict={model.X: mnist.validation.images[:2048],
98 | model.y_: mnist.validation.labels[:2048],
99 | model.dropout: 0})
100 | summary_writer_validation.add_summary(summary, i)
101 | print('step {}: train acc {}, valid acc {}'.format(i, acc_train, acc))
102 |
103 | if acc > val_acc_best:
104 | val_acc_best = acc
105 | else:
106 | summary, acc_train, _ = session.run([merged_summaries, accuracy, optimizer],
107 | feed_dict={model.X: batch_x, model.y_: batch_y,
108 | model.dropout: dropout})
109 | summary_writer_train.add_summary(summary, i)
110 |
111 | step += 1
112 |
113 | print("Training Finished! best valid acc {}".format(val_acc_best))
114 |
115 | # Final Eval
116 | print("Test Accuracy:",
117 | session.run(accuracy, feed_dict={model.X: mnist.test.images[:2048],
118 | model.y_: mnist.test.labels[:2048],
119 | model.dropout: 0}))
120 | except AbortRun:
121 | print("Aborting...")
122 | finally:
123 | tell.close(global_step=step)
124 |
125 |
126 | if __name__ == "__main__":
127 | tf.app.run()
128 |
--------------------------------------------------------------------------------
/samples/mnist/main_datapreprocessing.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 |
4 | """
5 |
6 | # Import TeLL
7 | from TeLL.config import Config
8 | from TeLL.session import TeLLSession
9 | from TeLL.datareaders import MNISTReader, DataLoader
10 | from TeLL.dataprocessing import DataProcessing, Normalize, Zoom
11 | from TeLL.utility.misc import AbortRun, check_kill_file
12 | from TeLL.regularization import decor_penalty
13 |
14 | # Import Tensorflow
15 | if __name__ == "__main__":
16 | import tensorflow as tf
17 |
18 |
19 | # ----------------------------------------------------------------------------------------------------------------------
20 | # Functions
21 | # ----------------------------------------------------------------------------------------------------------------------
22 |
23 | def main(_):
24 | config = Config()
25 | # Create new TeLL session with two summary writers
26 | tell = TeLLSession(config=config, summaries=["train", "validation"])
27 |
28 | # Get some members from the session for easier usage
29 | session = tell.tf_session
30 | summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
31 | model = tell.model
32 | workspace, config = tell.workspace, tell.config
33 |
34 | # Parameters
35 | learning_rate = config.get_value("learning_rate", 1e-3)
36 | iterations = config.get_value("iterations", 1000)
37 | batchsize = config.get_value("batchsize", 250)
38 | display_step = config.get_value("display_step", 10)
39 | dropout = config.get_value("dropout_prob", 0.25)
40 |
41 | #
42 | # Prepare input data
43 | #
44 |
45 | # Set datareaders
46 | training_reader = MNISTReader(dset='train')
47 | validation_reader = MNISTReader(dset='validation')
48 | test_reader = MNISTReader(dset='test')
49 |
50 | # Set Preprocessing
51 | training_data_preprocessed = DataProcessing(training_reader, apply_to='X')
52 | training_data_preprocessed = Normalize(training_data_preprocessed, apply_to='X')
53 | training_data_preprocessed = Normalize(training_data_preprocessed, apply_to=['X', 'Y'])
54 |
55 | # Set minibatch loaders
56 | training_loader = DataLoader(training_data_preprocessed, batchsize=50, batchsize_method='zeropad')
57 | validation_loader = DataLoader(validation_reader, batchsize=50, batchsize_method='zeropad')
58 | test_loader = DataLoader(test_reader, batchsize=50, batchsize_method='zeropad')
59 |
60 | #
61 | # Define loss and optimizer
62 | #
63 | with tf.name_scope("Cost"):
64 | cost = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=model.output, labels=model.y_))
65 | decor1 = decor_penalty(model.hidden1, model.y_, 10, [1], 0.)
66 | decor2 = decor_penalty(model.hidden2, model.y_, 10, [1], 6e-5)
67 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cost + decor1 + decor2)
68 | tf.summary.scalar("Loss", cost)
69 | tf.summary.scalar("Decor", decor1 + decor2)
70 |
71 | # Evaluate model
72 | with tf.name_scope("Accuracy"):
73 | correct_pred = tf.equal(tf.argmax(model.output, 1), tf.argmax(model.y_, 1))
74 | accuracy = tf.reduce_mean(tf.cast(correct_pred, tf.float32))
75 | tf.summary.scalar("Accuracy", accuracy)
76 |
77 | merged_summaries = tf.summary.merge_all()
78 |
79 | # Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
80 | step = tell.initialize_tf_variables().global_step
81 |
82 | # -------------------------------------------------------------------------
83 | # Start training
84 | # -------------------------------------------------------------------------
85 | acc_train = 0.
86 | try:
87 | while step < iterations:
88 | # Loop through training set
89 | for mb_i, mb in enumerate(training_loader.batch_loader(num_cached=5, num_threads=3)):
90 | check_kill_file(workspace=workspace)
91 |
92 | # Perform weight update
93 | summary, acc_train, _ = session.run([merged_summaries, accuracy, optimizer],
94 | feed_dict={model.X: mb['X'], model.y_: mb['y'],
95 | model.dropout: dropout})
96 | summary_writer_train.add_summary(summary, mb_i + step * batchsize)
97 |
98 | if step % display_step == 0:
99 | # Loop through validation set
100 | cos_sum, acc_sum, cor_sum = (0, 0, 0)
101 | for vmb_i, vmb in enumerate(validation_loader.batch_loader(num_cached=5, num_threads=3)):
102 | cos, acc, cor = session.run([cost, accuracy, correct_pred],
103 | feed_dict={model.X: vmb['X'], model.y_: vmb['y'],
104 | model.dropout: 0})
105 | cos_sum += cos
106 | acc_sum += acc
107 | cor_sum += cor
108 | print('step {}: train acc {}, valid acc {}'.format(mb_i + step * batchsize, cos_sum/vmb_i,
109 | acc_sum/vmb_i, cor_sum/vmb_i))
110 |
111 | step += 1
112 | if step >= iterations:
113 | break
114 |
115 | print("Training Finished!")
116 |
117 | # Final Eval
118 | for tmb_i, tmb in enumerate(test_loader.batch_loader(num_cached=len(test_reader.get_sample_keys()),
119 | num_threads=1)):
120 | print("Test Accuracy:",
121 | session.run(accuracy, feed_dict={model.X: tmb['X'],
122 | model.y_: tmb['y'],
123 | model.dropout: 0}))
124 | except AbortRun:
125 | print("Aborting...")
126 | finally:
127 | tell.close(global_step=step)
128 |
129 |
130 | if __name__ == "__main__":
131 | tf.app.run()
132 |
--------------------------------------------------------------------------------
/samples/moving_mnist/architectures.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 |
4 | """
5 | import tensorflow as tf
6 | import TeLL
7 | from TeLL.layers import DenseLayer, DropoutLayer, ConvLayer, RNNInputLayer, MaxPoolingLayer, DeConvLayer, ConvLSTMLayer, ScalingLayer, ConcatLayer
8 | from TeLL.initializations import weight_truncated_normal, constant
9 | from TeLL.config import Config
10 | from TeLL.utility.misc import get_rec_attr
11 |
12 | from collections import OrderedDict
13 |
14 |
15 | class ConvLSTMSemsegEfor(object):
16 | def __init__(self, config: Config, dataset):
17 | """Architecture for semantic segmentation as described in presentation using standard for loop."""
18 | depth = config.get_value("enc_dec_depth", 2)
19 | basenr_convs = config.get_value("enc_dec_conv_maps_base", 16)
20 | include_org_label = config.get_value("include_org_label", False)
21 | init_name = config.get_value("conv_W_initializer", "weight_xavier_conv2d")
22 | conv_W_initializer = getattr(TeLL.initializations, init_name)
23 |
24 | #
25 | # Layer list
26 | #
27 | layers = list()
28 |
29 | #
30 | # Create placeholders for feeding an input frame and a label at the first timestep
31 | #
32 | n_seq_pos = dataset.X_shape[1] # dataset.X_shape is [sample, seq_pos, x, y, features)
33 | X = tf.placeholder(tf.float32, shape=dataset.X_shape)
34 | y_ = tf.placeholder(tf.int32, shape=dataset.y_shape)
35 |
36 | if include_org_label:
37 | y_org = tf.placeholder(tf.int32, shape=dataset.y_org_shape)
38 |
39 | # ----------------------------------------------------------------------------------------------------------
40 | # Define network architecture
41 | # ----------------------------------------------------------------------------------------------------------
42 | # initializer for weight values of kernels
43 | # conv_W_initializer = weight_xavier_conv2d
44 |
45 | #
46 | # Initialize input to network of shape [sample, 1, x, y, features] with zero tensor of size of a frame
47 | #
48 | input_shape = dataset.X_shape[:1] + (1,) + dataset.X_shape[2:]
49 | layers.append(RNNInputLayer(tf.zeros(input_shape, dtype=tf.float32)))
50 | rnn_input_layer = layers[-1]
51 |
52 | #
53 | # Encoder and maxpooling layers
54 | #
55 | encoders = list()
56 | for d in range(1, depth + 1):
57 | print("\tConvLayerEncoder{}...".format(d))
58 | layers.append(ConvLayer(incoming=layers[-1],
59 | W=conv_W_initializer([config.kernel_conv, config.kernel_conv,
60 | layers[-1].get_output_shape()[-1], basenr_convs * (2 ** d)]),
61 | padding='SAME', name='ConvLayerEncoder{}'.format(d), a=tf.nn.elu))
62 | encoders.append(layers[-1])
63 | print("\tMaxpoolingLayer{}...".format(d))
64 | layers.append(MaxPoolingLayer(incoming=layers[-1], ksize=(1, 3, 3, 1), strides=(1, 2, 2, 1), padding='SAME',
65 | name='MaxpoolingLayer{}'.format(d)))
66 |
67 | #
68 | # ConvLSTM Layer
69 | #
70 | if config.n_lstm:
71 | n_lstm = config.n_lstm
72 | lstm_x_fwd = config.kernel_lstm_fwd
73 | lstm_y_fwd = config.kernel_lstm_fwd
74 | lstm_x_bwd = config.kernel_lstm_bwd
75 | lstm_y_bwd = config.kernel_lstm_bwd
76 |
77 | lstm_input_channels_fwd = layers[-1].get_output_shape()[-1]
78 | if config.reduced_rec_lstm:
79 | lstm_input_channels_bwd = config.reduced_rec_lstm
80 | else:
81 | lstm_input_channels_bwd = n_lstm
82 |
83 | lstm_init = dict(W_ci=[conv_W_initializer([lstm_x_fwd, lstm_y_fwd, lstm_input_channels_fwd, n_lstm]),
84 | conv_W_initializer([lstm_x_bwd, lstm_y_bwd, lstm_input_channels_bwd, n_lstm])],
85 | W_ig=[conv_W_initializer([lstm_x_fwd, lstm_y_fwd, lstm_input_channels_fwd, n_lstm]),
86 | conv_W_initializer([lstm_x_bwd, lstm_y_bwd, lstm_input_channels_bwd, n_lstm])],
87 | W_og=[conv_W_initializer([lstm_x_fwd, lstm_y_fwd, lstm_input_channels_fwd, n_lstm]),
88 | conv_W_initializer([lstm_x_bwd, lstm_y_bwd, lstm_input_channels_bwd, n_lstm])],
89 | W_fg=[conv_W_initializer([lstm_x_fwd, lstm_y_fwd, lstm_input_channels_fwd, n_lstm]),
90 | conv_W_initializer([lstm_x_bwd, lstm_y_bwd, lstm_input_channels_bwd, n_lstm])],
91 | b_ci=constant([n_lstm]),
92 | b_ig=constant([n_lstm]),
93 | b_og=constant([n_lstm]),
94 | b_fg=constant([n_lstm], 1))
95 |
96 | print("\tConvLSTM...")
97 | layers.append(ConvLSTMLayer(incoming=layers[-1], n_units=n_lstm, **lstm_init,
98 | a_out=get_rec_attr(tf, config.lstm_act), forgetgate=config.forgetgate,
99 | comb=config.lstm_comb, store_states=config.store_states,
100 | tickerstep_biases=tf.zeros, output_dropout=config.lstm_output_dropout,
101 | precomp_fwds=False))
102 | lstm_layer = layers[-1]
103 |
104 | #
105 | # Optional maxpooling and upscaling of rec LSTM connections combined with/or optional feature squashing
106 | #
107 | ext_lstm_recurrence = None
108 | if config.lstm_rec_maxpooling:
109 | print("\tMaxpoolingDeconv...")
110 | layers.append(
111 | MaxPoolingLayer(incoming=layers[-1], ksize=(1, 3, 3, 1), strides=(1, 2, 2, 1), padding='SAME',
112 | name='MaxPoolingLayer'))
113 | layers.append(DeConvLayer(incoming=layers[-1], a=tf.nn.elu,
114 | W=conv_W_initializer([3, 3, layers[-1].get_output_shape()[-1],
115 | layers[-1].get_output_shape()[-1]]),
116 | strides=(1, 2, 2, 1),
117 | padding='SAME', name='DeConvLayer'))
118 | print("\tConvLSTMRecurrence...")
119 | ext_lstm_recurrence = layers[-1]
120 |
121 | if config.reduced_rec_lstm:
122 | print("\tFeatureSquashing...")
123 | layers.append(ConvLayer(incoming=layers[-1],
124 | W=conv_W_initializer([config.kernel_conv_out, config.kernel_conv_out,
125 | layers[-1].get_output_shape()[-1],
126 | config.reduced_rec_lstm]),
127 | padding='SAME', name='ConvLayerFeatureSquashing', a=tf.nn.elu))
128 | print("\tConvLSTMRecurrence...")
129 | ext_lstm_recurrence = layers[-1]
130 |
131 | if ext_lstm_recurrence is not None:
132 | lstm_layer.add_external_recurrence(ext_lstm_recurrence)
133 | else:
134 | print("\tSubstituteConvLayer...")
135 | n_lstm = basenr_convs * (2 ** depth) * 4
136 | layers.append(ConvLayer(incoming=layers[-1],
137 | W=conv_W_initializer([config.kernel_conv, config.kernel_conv,
138 | layers[-1].get_output_shape()[-1],
139 | int(basenr_convs * (2 ** depth) * 4.5)]),
140 | padding='SAME', name='SubstituteConvLayer', a=tf.nn.elu))
141 | lstm_layer = layers[-1]
142 |
143 | #
144 | # Decoder and upscaling layers
145 | #
146 | for d in list(range(1, depth + 1))[::-1]:
147 | print("\tUpscalingLayer{}...".format(d))
148 | layers[-1] = ScalingLayer(incoming=layers[-1], size=encoders[d - 1].get_output_shape()[-3:-1],
149 | name='UpscalingLayergLayer{}'.format(d))
150 |
151 | print("\tConcatLayer{}...".format(d))
152 | layers.append(ConcatLayer([encoders[d - 1], layers[-1]], name='ConcatLayer{}'.format(d)))
153 |
154 | print("\tConvLayerDecoder{}...".format(d))
155 | layers.append(ConvLayer(incoming=layers[-1],
156 | W=conv_W_initializer([config.kernel_conv, config.kernel_conv,
157 | layers[-1].get_output_shape()[-1], basenr_convs * (2 ** d)]),
158 | padding='SAME', name='ConvLayerDecoder{}'.format(d), a=tf.nn.elu))
159 |
160 | #
161 | # ConvLayer for semantic segmentation
162 | #
163 | print("\tConvLayerSemanticSegmentation...")
164 | layers.append(ConvLayer(incoming=layers[-1],
165 | W=conv_W_initializer([config.kernel_conv_out, config.kernel_conv_out,
166 | layers[-1].get_output_shape()[-1], 11]),
167 | padding='SAME', name='ConvLayerSemanticSegmentation', a=tf.identity))
168 | sem_seg_layer = layers[-1]
169 |
170 | # ----------------------------------------------------------------------------------------------------------
171 | # Loop through sequence positions and create graph
172 | # ----------------------------------------------------------------------------------------------------------
173 |
174 | #
175 | # Loop through sequence positions
176 | #
177 | print("\tRNN Loop...")
178 | sem_seg_out = list()
179 | for seq_pos in range(n_seq_pos):
180 | with tf.name_scope("Sequence_pos_{}".format(seq_pos)):
181 | print("\t seq. pos. {}...".format(seq_pos))
182 | # Set input layer to X at frame (t) and outputs of upper layers at (t-1)
183 | layers[0].update(X[:, seq_pos:seq_pos + 1, :])
184 |
185 | # Calculate new network output at (t), including new hidden states
186 | _ = lstm_layer.get_output()
187 | sem_seg_out.append(sem_seg_layer.get_output(prev_layers=encoders + [lstm_layer]))
188 |
189 | #
190 | # Loop through tickersteps
191 | #
192 | # # Use empty frame as X during ticker steps (did not work so good)
193 | # tickerstep_input = tf.zeros(dataset.X_shape[:1] + (1,) + dataset.X_shape[2:], dtype=tf.float32,
194 | # name="tickerframe")
195 |
196 | # Use last frame as X during ticker steps
197 | #tickerstep_input = X[:, -1:, :]
198 | #layers[0].update(tickerstep_input)
199 |
200 | #for tickerstep in range(config.tickersteps):
201 | # with tf.name_scope("Tickerstep_{}".format(tickerstep)):
202 | # print("\t tickerstep {}...".format(tickerstep))
203 | #
204 | # # Calculate new network output at (t), including new hidden states
205 | # _ = lstm_layer.get_output(tickerstep_nodes=True)
206 |
207 |
208 | #sem_seg_out = sem_seg_layer.get_output(prev_layers=encoders + [lstm_layer])
209 |
210 | print("\tDone!")
211 |
212 | #
213 | # Publish
214 | #
215 | self.X = X
216 | self.y_feed = y_
217 | self.y_ = y_[:, 10:]
218 | self.output = tf.concat(sem_seg_out[10:], 1)
219 | self.__layers = layers
220 | self.__n_lstm = n_lstm
221 | self.__lstm_layer = lstm_layer
222 | self.lstm_layer = lstm_layer
223 | self.__plot_dict, self.__plot_range_dict, self.__plotsink = self.__setup_plotting(config)
224 | if include_org_label:
225 | self.y_org = y_org
226 |
227 | def get_layers(self):
228 | return self.__layers
229 |
230 | def get_plotsink(self):
231 | return self.__plotsink
232 |
233 | def get_plot_dict(self):
234 | return self.__plot_dict
235 |
236 | def get_plot_range_dict(self):
237 | return self.__plot_range_dict
238 |
239 | def __setup_plotting(self, config):
240 | # Prepare for plotting
241 | # Create a list of lists with keys to plot in a subplot
242 | plotsink = []
243 | plot_dict = OrderedDict()
244 | plot_range_dict = OrderedDict()
245 | plot_sample = 0
246 | # only plot first, middle, and end frame when plotting LSTM outputs
247 | plot_frames = slice(-1, None) # slice(0, sample_len, int(sample_len/2))
248 |
249 | #
250 | # init plot sink
251 | #
252 | plot_dict['X'] = self.X[plot_sample, plot_frames, :, :, :]
253 | plot_range_dict['X'] = [0, 1]
254 | plotsink.append(['X'])
255 |
256 | plot_dict['y_'] = self.y_[plot_sample, None, :, :]
257 | plot_range_dict['y_'] = [0, 20]
258 | plotsink[0].append('y_')
259 |
260 | #
261 | # Plot LSTM layer
262 | #
263 |
264 | # Output for first sample in mb
265 | if config.n_lstm:
266 | try:
267 | plot_dict_new, plotsink_new, plot_range_dict_new = self.__lstm_layer.get_plots_out(sample=plot_sample)
268 |
269 | # add LSTM output plot to first subfigure
270 | plotsink[0] += plotsink_new[0]
271 | plot_dict.update(plot_dict_new)
272 | plot_range_dict.update(plot_range_dict_new)
273 |
274 | # Weights from first LSTM unit
275 | plot_dict_new, plotsink_new, _ = self.__lstm_layer.get_plots_w(max_num_inp=6, max_num_out=1)
276 |
277 | plotsink += plotsink_new
278 | plot_dict.update(plot_dict_new)
279 |
280 | # States, if possible
281 | plot_dict_new, plotsink_new, _ = self.__lstm_layer.get_plots_state()
282 | plotsink += plotsink_new
283 | plot_dict.update(plot_dict_new)
284 |
285 | #
286 | # Plot activations in outputs from first 4 units
287 | #
288 | for unit in range(4):
289 | plot_dict['ConvLSTM_f{}'.format(unit)] = self.__lstm_layer.out[plot_sample, plot_frames, :, :, unit]
290 | plotsink.append(['ConvLSTM_f{}'.format(unit)])
291 |
292 | except AttributeError:
293 | pass
294 | #
295 | # Plot outputs
296 | #
297 | plot_dict['out'] = tf.arg_max(self.output[plot_sample, plot_frames, :, :, :], 3)
298 | plot_range_dict['out'] = [0, 20]
299 | plotsink[0].append('out')
300 |
301 | #
302 | # Publish sink
303 | #
304 | return [plot_dict, plot_range_dict, plotsink]
305 |
306 |
307 | class Scaler(object):
308 | def __init__(self, config: Config, dataset):
309 | """
310 |
311 | """
312 |
313 | depth = config.get_value("enc_dec_depth", 2)
314 | basenr_convs = config.get_value("enc_dec_conv_maps_base", 32)
315 | include_org_label = config.get_value("include_org_label", False)
316 | init_name = config.get_value("conv_W_initializer", "weight_xavier_conv2d")
317 | conv_W_initializer = getattr(TeLL.initializations, init_name)
318 | shared = False
319 |
320 | #
321 | # Layer list
322 | #
323 | layers = list()
324 |
325 | #
326 | # Create placeholders for feeding an input frame and a label at the first timestep
327 | #
328 | n_seq_pos = dataset.X_shape[1] # dataset.X_shape is [sample, seq_pos, x, y, features)
329 | X = tf.placeholder(tf.float32, shape=dataset.X_shape)
330 | y_ = tf.placeholder(tf.int32, shape=dataset.y_shape)
331 |
332 | if include_org_label:
333 | y_org = tf.placeholder(tf.int32, shape=dataset.y_org_shape)
334 |
335 | #
336 | # Input Layer
337 | #
338 | input_shape = dataset.X_shape[:1] + (1,) + dataset.X_shape[2:]
339 | layers.append(RNNInputLayer(tf.zeros(input_shape, dtype=tf.float32)))
340 |
341 | #
342 | # Scaler Structure
343 | #
344 | conv_weights_shape = [config.kernel_conv,
345 | config.kernel_conv,
346 | layers[-1].get_output_shape()[-1],
347 | basenr_convs]
348 |
349 | if shared:
350 | shared_input_conv_weights = conv_W_initializer(conv_weights_shape)
351 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
352 | a=tf.nn.elu, dilation_rate=[11, 11]))
353 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
354 | a=tf.nn.elu, dilation_rate=[9, 9]))
355 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
356 | a=tf.nn.elu, dilation_rate=[7, 7]))
357 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
358 | a=tf.nn.elu, dilation_rate=[5, 5]))
359 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
360 | a=tf.nn.elu, dilation_rate=[3, 3]))
361 | layers.append(ConvLayer(incoming=layers[0], W=shared_input_conv_weights,
362 | a=tf.nn.elu, dilation_rate=[1, 1]))
363 | else:
364 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
365 | a=tf.nn.elu, dilation_rate=[11, 11]))
366 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
367 | a=tf.nn.elu, dilation_rate=[9, 9]))
368 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
369 | a=tf.nn.elu, dilation_rate=[7, 7]))
370 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
371 | a=tf.nn.elu, dilation_rate=[5, 5]))
372 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
373 | a=tf.nn.elu, dilation_rate=[3, 3]))
374 | layers.append(ConvLayer(incoming=layers[0], W=conv_W_initializer(conv_weights_shape),
375 | a=tf.nn.elu, dilation_rate=[1, 1]))
376 |
377 | # concat feature maps of all scale levels and reduce the number of features with a 1x1 conv
378 | layers.append(ConcatLayer(incomings=layers[1:]))
379 | conv_weights_shape = [1, 1, layers[-1].get_output_shape()[-1], basenr_convs]
380 | layers.append(ConvLayer(incoming=layers[-1], W=conv_W_initializer(conv_weights_shape)))
381 |
382 | # add 3 more conv layers to have some depth
383 | conv_weights_shape = [config.kernel_conv,
384 | config.kernel_conv,
385 | layers[-1].get_output_shape()[-1],
386 | basenr_convs]
387 | layers.append(ConvLayer(incoming=layers[-1], W=conv_W_initializer(conv_weights_shape)))
388 | layers.append(ConvLayer(incoming=layers[-1], W=conv_W_initializer(conv_weights_shape)))
389 | layers.append(ConvLayer(incoming=layers[-1], W=conv_W_initializer(conv_weights_shape)))
390 |
391 | #
392 | # Output Layer
393 | #
394 | layers.append(ConvLayer(incoming=layers[-1],
395 | W=conv_W_initializer([config.kernel_conv_out, config.kernel_conv_out,
396 | layers[-1].get_output_shape()[-1], 11]),
397 | padding='SAME', name='ConvLayerSemanticSegmentation', a=tf.identity))
398 | sem_seg_layer = layers[-1]
399 |
400 | self.X = X
401 | self.y_ = y_
402 | self.output = sem_seg_layer.get_output()
403 |
--------------------------------------------------------------------------------
/samples/moving_mnist/config.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "baseline",
3 | "cuda_gpu": "0",
4 | "architecture": "architectures.ConvLSTMSemsegEfor",
5 | "iterations": 160000,
6 | "display_step": 10,
7 | "learning_rate": 1e-3,
8 | "enc_dec_depth": 2,
9 | "n_lstm": 0,
10 | "score_at": 1000,
11 | "plot_at": 20,
12 | "tf_allow_growth": true,
13 | "working_dir": "working_dir",
14 | "mnist_train_images": "../MNIST_data/train-images-idx3-ubyte.gz",
15 | "mnist_train_labels": "../MNIST_data/train-labels-idx1-ubyte.gz",
16 | "mnist_test_images": "../MNIST_data/t10k-images-idx3-ubyte.gz",
17 | "mnist_test_labels": "../MNIST_data/t10k-labels-idx1-ubyte.gz",
18 | "higher_num_on_top": true,
19 | "num_frames": 20,
20 | "batch_size": 16,
21 | "image_size": 64,
22 | "num_digits": 2,
23 | "step_length": 0.1,
24 | "label_threshold": 127,
25 | "kernel_lstm_fwd": 3,
26 | "kernel_lstm_bwd": 3,
27 | "kernel_conv": 3,
28 | "kernel_conv_out": 1,
29 | "reduced_rec_lstm": false,
30 | "lstm_rec_maxpooling": false,
31 | "lstm_comb": "add",
32 | "forgetgate": true,
33 | "store_states": false,
34 | "lstm_act": "nn.elu",
35 | "lstm_output_dropout": false,
36 | "tickersteps": 0
37 | }
38 |
--------------------------------------------------------------------------------
/samples/moving_mnist/config_gauss.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "gauss",
3 | "cuda_gpu": "0",
4 | "architecture": "architectures.ConvLSTMSemsegEfor",
5 | "iterations": 160000,
6 | "display_step": 10,
7 | "learning_rate": 1e-3,
8 | "blur_filter_size": 11,
9 | "n_lstm": 0,
10 | "score_at": 1000,
11 | "plot_at": 20,
12 | "tf_allow_growth": true,
13 | "working_dir": "working_dir",
14 | "mnist_train_images": "../MNIST_data/train-images-idx3-ubyte.gz",
15 | "mnist_train_labels": "../MNIST_data/train-labels-idx1-ubyte.gz",
16 | "mnist_test_images": "../MNIST_data/t10k-images-idx3-ubyte.gz",
17 | "mnist_test_labels": "../MNIST_data/t10k-labels-idx1-ubyte.gz",
18 | "higher_num_on_top": true,
19 | "num_frames": 20,
20 | "batch_size": 16,
21 | "image_size": 64,
22 | "num_digits": 2,
23 | "step_length": 0.1,
24 | "label_threshold": 127,
25 | "kernel_lstm_fwd": 3,
26 | "kernel_lstm_bwd": 3,
27 | "kernel_conv": 3,
28 | "kernel_conv_out": 1,
29 | "reduced_rec_lstm": false,
30 | "lstm_rec_maxpooling": false,
31 | "lstm_comb": "add",
32 | "forgetgate": true,
33 | "store_states": false,
34 | "lstm_act": "nn.elu",
35 | "lstm_output_dropout": false,
36 | "tickersteps": 0
37 | }
38 |
--------------------------------------------------------------------------------
/samples/moving_mnist/config_lstm.json:
--------------------------------------------------------------------------------
1 | {
2 | "specs": "lstm",
3 | "cuda_gpu": "0",
4 | "architecture": "architectures.ConvLSTMSemsegEfor",
5 | "iterations": 160000,
6 | "display_step": 10,
7 | "learning_rate": 1e-3,
8 | "enc_dec_depth": 2,
9 | "n_lstm": 64,
10 | "score_at": 1000,
11 | "plot_at": 20,
12 | "tf_allow_growth": true,
13 | "working_dir": "working_dir",
14 | "mnist_train_images": "../MNIST_data/train-images-idx3-ubyte.gz",
15 | "mnist_train_labels": "../MNIST_data/train-labels-idx1-ubyte.gz",
16 | "mnist_test_images": "../MNIST_data/t10k-images-idx3-ubyte.gz",
17 | "mnist_test_labels": "../MNIST_data/t10k-labels-idx1-ubyte.gz",
18 | "higher_num_on_top": true,
19 | "num_frames": 20,
20 | "batch_size": 16,
21 | "image_size": 64,
22 | "num_digits": 2,
23 | "step_length": 0.1,
24 | "label_threshold": 127,
25 | "kernel_lstm_fwd": 3,
26 | "kernel_lstm_bwd": 3,
27 | "kernel_conv": 3,
28 | "kernel_conv_out": 1,
29 | "reduced_rec_lstm": false,
30 | "lstm_rec_maxpooling": false,
31 | "lstm_comb": "add",
32 | "forgetgate": true,
33 | "store_states": true,
34 | "lstm_act": "nn.elu",
35 | "lstm_output_dropout": false,
36 | "tickersteps": 0
37 | }
38 |
--------------------------------------------------------------------------------
/samples/moving_mnist/dataset.py:
--------------------------------------------------------------------------------
1 | """
2 | Essence downloaded from http://www.cs.toronto.edu/~nitish/unsupervised_video/
3 | """
4 |
5 | import numpy as np
6 | import gzip
7 | import sys
8 |
9 | from TeLL.config import Config
10 |
11 |
12 | def load_mnist(file, labels=False):
13 | print('open ' + file)
14 | with gzip.open(file, 'rb') as f:
15 | # skip header
16 | f.read(8 if labels else 16)
17 | a = np.array([float(i) for i in f.read()])
18 |
19 | if not labels:
20 | a = np.reshape(a, [-1, 28, 28])
21 |
22 | return a
23 |
24 |
25 | def load_mnist_images(file):
26 | return load_mnist(file, labels=False)
27 |
28 |
29 | def load_mnist_labels(file):
30 | return load_mnist(file, labels=True)
31 |
32 |
33 | class BouncingMNISTDataHandler(object):
34 | def __init__(self, config: Config, images_file, labels_file, rng):
35 | self.rng = rng
36 | self.seq_length_ = config.num_frames
37 | self.batch_size_ = config.batch_size
38 | self.image_size_ = config.image_size
39 | self.num_digits_ = config.num_digits
40 | self.step_length_ = config.step_length
41 | self.label_threshold_ = config.label_threshold
42 | self.higher_num_on_top_ = config.higher_num_on_top
43 | self.dataset_size_ = 10000 # The dataset is really infinite. This is just for validation.
44 | self.digit_size_ = 28
45 | self.frame_size_ = self.image_size_ ** 2
46 | self.data_ = load_mnist_images(images_file)
47 | self.labels_ = load_mnist_labels(labels_file)
48 | self.indices_ = np.arange(self.data_.shape[0])
49 | self.row_ = 0
50 | self.rng.shuffle(self.indices_)
51 |
52 | def GetBatchSize(self):
53 | return self.batch_size_
54 |
55 | def GetDims(self):
56 | return self.frame_size_
57 |
58 | def GetDatasetSize(self):
59 | return self.dataset_size_
60 |
61 | def GetSeqLength(self):
62 | return self.seq_length_
63 |
64 | def Reset(self):
65 | self.row_ = 0
66 |
67 | def GetRandomTrajectory(self, batch_size):
68 | length = self.seq_length_
69 | canvas_size = self.image_size_ - self.digit_size_
70 |
71 | # Initial position uniform random inside the box.
72 | y = self.rng.rand(batch_size)
73 | x = self.rng.rand(batch_size)
74 |
75 | # Choose a random velocity.
76 | theta = self.rng.rand(batch_size) * 2 * np.pi
77 | v_y = np.sin(theta)
78 | v_x = np.cos(theta)
79 |
80 | start_y = np.zeros((length, batch_size))
81 | start_x = np.zeros((length, batch_size))
82 | for i in range(length):
83 | # Take a step along velocity.
84 | y += v_y * self.step_length_
85 | x += v_x * self.step_length_
86 |
87 | # Bounce off edges.
88 | for j in range(batch_size):
89 | if x[j] <= 0:
90 | x[j] = 0
91 | v_x[j] = -v_x[j]
92 | if x[j] >= 1.0:
93 | x[j] = 1.0
94 | v_x[j] = -v_x[j]
95 | if y[j] <= 0:
96 | y[j] = 0
97 | v_y[j] = -v_y[j]
98 | if y[j] >= 1.0:
99 | y[j] = 1.0
100 | v_y[j] = -v_y[j]
101 | start_y[i, :] = y
102 | start_x[i, :] = x
103 |
104 | # Scale to the size of the canvas.
105 | start_y = (canvas_size * start_y).astype(np.int32)
106 | start_x = (canvas_size * start_x).astype(np.int32)
107 | return start_y, start_x
108 |
109 | def Overlap(self, a, b):
110 | """ Put b on top of a."""
111 | return np.maximum(a, b)
112 | # return b
113 |
114 | def GetBatch(self, batch_size=None):
115 | if batch_size is None:
116 | batch_size = self.batch_size_
117 |
118 | start_y, start_x = self.GetRandomTrajectory(batch_size * self.num_digits_)
119 | data = np.zeros((self.seq_length_, batch_size, self.image_size_, self.image_size_), dtype=np.float32)
120 | labels = np.zeros((self.seq_length_, batch_size, self.image_size_, self.image_size_, self.num_digits_),
121 | dtype=np.float32)
122 | for j in range(batch_size):
123 | for n in range(self.num_digits_):
124 | ind = self.indices_[self.row_]
125 | self.row_ += 1
126 | if self.row_ == self.data_.shape[0]:
127 | self.row_ = 0
128 | self.rng.shuffle(self.indices_)
129 |
130 | digit_image = self.data_[ind, :, :]
131 | digit_label = self.labels_[ind]
132 |
133 | for i in range(self.seq_length_):
134 | # draw digit into data
135 | top = start_y[i, j * self.num_digits_ + n]
136 | left = start_x[i, j * self.num_digits_ + n]
137 | bottom = top + self.digit_size_
138 | right = left + self.digit_size_
139 | data[i, j, top:bottom, left:right] = self.Overlap(
140 | data[i, j, top:bottom, left:right], digit_image)
141 |
142 | # draw digit into labels
143 | labels[i, j, top:bottom, left:right, n] = \
144 | np.asarray(digit_image > self.label_threshold_, np.int) * (digit_label + 1)
145 |
146 | if self.higher_num_on_top_:
147 | labels = np.amax(labels, 4)
148 |
149 | return np.swapaxes(np.reshape(data, list(data.shape) + [1]), 0, 1), \
150 | np.swapaxes(labels, 0, 1)
151 |
152 |
153 | #def main():
154 | # data_pb = ReadDataProto(sys.argv[1])
155 | #
156 | # print(data_pb.data_file)
157 | # print(data_pb.labels_file)
158 | #
159 | # dh = BouncingMNISTDataHandler(data_pb)
160 | # data, labels = dh.GetBatch()
161 | # np.save(data_pb.dataset_name + '.npy', data)
162 | # np.save(data_pb.dataset_name + '_labels.npy', labels)
163 |
164 |
165 | #if __name__ == '__main__':
166 | # main()
167 |
--------------------------------------------------------------------------------
/samples/moving_mnist/main.py:
--------------------------------------------------------------------------------
1 | """
2 | © Michael Widrich, Markus Hofmarcher, 2017
3 |
4 | """
5 | # Import MNIST data
6 | from tensorflow.examples.tutorials.mnist import input_data
7 |
8 | from dataset import BouncingMNISTDataHandler
9 | import numpy as np
10 |
11 | # Import TeLL
12 | from TeLL.config import Config
13 | from TeLL.session import TeLLSession
14 | from TeLL.utility.timer import Timer
15 | from TeLL.utility.misc import AbortRun, check_kill_file
16 | from TeLL.loss import image_crossentropy, iou_loss, blurred_cross_entropy
17 | from TeLL.evaluation import Evaluation
18 | from TeLL.utility.plotting import Plotter, save_subplots
19 | from TeLL.utility.plotting_daemons import start_plotting_daemon, stop_plotting_daemon
20 | from collections import OrderedDict
21 | import os
22 |
23 | # Import Tensorflow
24 | if __name__ == "__main__":
25 | plotter = Plotter(num_workers=5, plot_function=save_subplots)
26 |
27 | import tensorflow as tf
28 |
29 | from scipy.misc import imsave
30 |
31 | # ----------------------------------------------------------------------------------------------------------------------
32 | # Functions
33 | # ----------------------------------------------------------------------------------------------------------------------
34 |
35 | class DataSet(object):
36 | def __init__(self, x_shape, y_shape):
37 | self.X_shape = x_shape
38 | self.y_shape = y_shape
39 |
40 | def to_color(labels):
41 | image = np.zeros(labels.shape + (3,))
42 | print(labels.shape)
43 |
44 | for i in range(labels.shape[0]):
45 | for j in range(labels.shape[1]):
46 | for k in range(labels.shape[2]):
47 | for l in range(labels.shape[3]):
48 | image[i, j, k, l] = {
49 | 0: [ 0, 0, 0],
50 | 1: [255, 0, 0],
51 | 2: [ 0, 255, 0],
52 | 3: [ 0, 0, 255],
53 | 4: [255, 255, 0],
54 | 5: [ 0, 255, 255],
55 | 6: [255, 0, 255],
56 | 7: [255, 255, 255],
57 | 8: [128, 255, 0],
58 | 9: [ 0, 128, 255],
59 | 10: [255, 0, 128],
60 | 11: [255, 128, 0]
61 | }[labels[i, j, k, l]]
62 |
63 | return image
64 |
65 |
66 | def to_image(pred, true):
67 | # in shape is (20, batch_size, 64, 64, 3)
68 | # out shape is (batch_size, 64 * 10, 64 * 4, 3)
69 |
70 | assert(pred.shape == true.shape)
71 | shape = pred.shape
72 | out = np.zeros((shape[1], 256, 640, 3))
73 |
74 | for i in range(shape[0]):
75 | for t in range(shape[1]):
76 | h_from = (t % 10) * 64
77 | h_to = h_from + 64
78 | v_from = (0 if t < 10 else 2) * 64
79 | v_to = v_from + 64
80 | out[i, v_from:v_to, h_from:h_to] = true[i, t]
81 | out[i, v_from+64:v_to+64, h_from:h_to] = pred[i, t]
82 |
83 | return out
84 |
85 |
86 | def main(_):
87 | np.random.seed(0)
88 | rng = np.random.RandomState(seed=0)
89 |
90 | config = Config()
91 |
92 | #
93 | # Load Data
94 | #
95 | with Timer(name="Load data"):
96 | training_data = BouncingMNISTDataHandler(
97 | config, config.mnist_train_images, config.mnist_train_labels, rng)
98 | test_data = BouncingMNISTDataHandler(
99 | config, config.mnist_test_images, config.mnist_test_labels, rng)
100 |
101 | dataset = DataSet((config.batch_size, config.num_frames, config.image_size, config.image_size, 1),
102 | (config.batch_size, config.num_frames, config.image_size, config.image_size))
103 |
104 | # Create new TeLL session with two summary writers
105 | tell = TeLLSession(config=config, summaries=["train", "validation"], model_params={"dataset": dataset})
106 |
107 | # Get some members from the session for easier usage
108 | session = tell.tf_session
109 | summary_writer_train, summary_writer_validation = tell.tf_summaries["train"], tell.tf_summaries["validation"]
110 | model = tell.model
111 | workspace, config = tell.workspace, tell.config
112 |
113 | # Parameters
114 | learning_rate = config.get_value("learning_rate", 1e-3)
115 | iterations = config.get_value("iterations", 1000)
116 | batch_size = config.get_value("batch_size", 256)
117 | display_step = config.get_value("display_step", 10)
118 | calc_statistics = config.get_value("calc_statistics", False)
119 | blur_filter_size = config.get_value("blur_filter_size", None)
120 |
121 | training_summary_tensors = OrderedDict()
122 |
123 | # Define loss and optimizer
124 | #with tf.name_scope("Cost"):
125 | # sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
126 | # calc_statistics=calc_statistics, reduce_by="sum")
127 | # optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
128 | # tf.summary.scalar("Loss", sem_seg_loss)
129 |
130 | # Evaluate model
131 | validation_summary_tensors = OrderedDict()
132 |
133 | # validationset always uses class weights for loss calculation
134 | with tf.name_scope('Cost'):
135 | blur_sampling_range = tf.placeholder(tf.float32)
136 |
137 | if blur_filter_size is not None:
138 | sem_seg_loss = blurred_cross_entropy(output=model.output, target=model.y_,
139 | filter_size=blur_filter_size,
140 | sampling_range=blur_sampling_range)
141 | else:
142 | sem_seg_loss, _ = image_crossentropy(pred=model.output, target=model.y_,
143 | reduce_by="mean", calc_statistics=calc_statistics)
144 | optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(sem_seg_loss)
145 | iou, iou_op = tf.contrib.metrics.streaming_mean_iou(
146 | predictions=tf.squeeze(tf.arg_max(model.output, 4)),
147 | labels=tf.squeeze(model.y_),
148 | num_classes=model.output.get_shape()[-1])
149 | loss_prot = tf.summary.scalar("Loss", sem_seg_loss)
150 | iou_prot = tf.summary.scalar("IoU", iou)
151 |
152 | train_summaries = tf.summary.merge([loss_prot])
153 | valid_summaries = tf.summary.merge([loss_prot, iou_prot])
154 |
155 | # Initialize tensorflow variables (either initializes them from scratch or restores from checkpoint)
156 | step = tell.initialize_tf_variables().global_step
157 |
158 | # -------------------------------------------------------------------------
159 | # Start training
160 | # -------------------------------------------------------------------------
161 |
162 | plot_elements_sym = list(model.get_plot_dict().values())
163 | plot_elements = list()
164 | plot_ranges = model.get_plot_range_dict()
165 |
166 | try:
167 | while step < iterations:
168 | check_kill_file(workspace=workspace)
169 | batch_x, batch_y = training_data.GetBatch()
170 |
171 | i = step * batch_size
172 | if step % display_step == 0:
173 | mean_loss = 0.
174 | for j in range(10):
175 | test_x, test_y = test_data.GetBatch()
176 |
177 | summary, loss, _, *plot_elements = session.run([valid_summaries, sem_seg_loss, iou_op, *plot_elements_sym],
178 | feed_dict={model.X: test_x,
179 | model.y_feed: test_y,
180 | blur_sampling_range: 3.5})
181 |
182 | summary_writer_validation.add_summary(summary, i)
183 | mean_loss += loss
184 |
185 | # Re-associate returned tensorflow values to plotting keys
186 | plot_dict = OrderedDict(zip(list(model.get_plot_dict().keys()), plot_elements))
187 |
188 | # Plot outputs and cell states over frames if specified
189 | if config.store_states and 'ConvLSTMLayer_h' in plot_dict and step % config.plot_at == 0:
190 | convh = plot_dict['ConvLSTMLayer_h']
191 | convrh = [c[0, :, :, 0] for c in convh]
192 | convrh = [convrh[:6], convrh[6:12], convrh[12:18], convrh[18:24], convrh[24:]]
193 | plot_args = dict(images=convrh,
194 | filename=os.path.join(workspace.get_result_dir(),
195 | "step{}_h.png".format(step)))
196 | plotter.set_plot_kwargs(plot_args)
197 | plotter.plot()
198 |
199 | if config.store_states and 'ConvLSTMLayer_c' in plot_dict and step % config.plot_at == 0:
200 | convc = plot_dict['ConvLSTMLayer_c']
201 | convrc = [c[0, :, :, 0] for c in convc]
202 | convrc = [convrc[:6], convrc[6:12], convrc[12:18], convrc[18:24], convrc[24:]]
203 | plot_args = dict(images=convrc,
204 | filename=os.path.join(workspace.get_result_dir(),
205 | "step{}_c.png".format(step)))
206 | plotter.set_plot_kwargs(plot_args)
207 | plotter.plot()
208 | print('Validation Loss at step {}: {}'.format(i, mean_loss / 10))
209 |
210 | summary, loss, _ = session.run([train_summaries, sem_seg_loss, optimizer],
211 | feed_dict={model.X: batch_x,
212 | model.y_feed: batch_y,
213 | blur_sampling_range: 3.5})
214 | summary_writer_train.add_summary(summary, i)
215 |
216 | step += 1
217 |
218 | print("Training Finished!")
219 |
220 | # Final Eval
221 | mean_loss = 0.
222 |
223 | for j in range(100):
224 | test_x, test_y = test_data.GetBatch()
225 | summary, loss, _ = session.run([valid_summaries, sem_seg_loss, iou_op],
226 | feed_dict={model.X: test_x,
227 | model.y_feed: test_y,
228 | blur_sampling_range: 3.5})
229 | mean_loss += loss
230 |
231 | test_x, test_y = test_data.GetBatch()
232 | pred = session.run(tf.argmax(model.output, 4), feed_dict={model.X: test_x})
233 |
234 | pred = to_color(pred)
235 | true = to_color(test_y)
236 | out = to_image(pred, true)
237 |
238 | for i in range(pred.shape[0]):
239 | imsave(tell.workspace.get_result_dir() + '/sample_{:02d}.png'.format(i), out[i,])
240 |
241 | print("Validation Loss {}".format(mean_loss / 100))
242 | except AbortRun:
243 | print("Aborting...")
244 | finally:
245 | tell.close(global_step=step)
246 | plotter.close()
247 |
248 |
249 | if __name__ == "__main__":
250 | tf.app.run()
251 |
--------------------------------------------------------------------------------
/setup.cfg:
--------------------------------------------------------------------------------
1 | [bdist_wheel]
2 | # This flag says that the code is written to work on both Python 2 and Python
3 | # 3. If at all possible, it is good practice to do this. If you cannot, you
4 | # will need to generate wheels for each Python version that you support.
5 | universal=0
6 |
--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
1 | """Tensorflow Layer Library (TeLL)
2 | Provides a variety of tensorflow-based network layers, flexible (recurrent) network designs, convenience routines for saving and resuming networks, and more!
3 | Copyright (c) 2016-2017 Michael Widrich and Markus Hofmarcher, Institute of Bioinformatics, Johannes Kepler University Linz, Austria
4 | """
5 |
6 | # Always prefer setuptools over distutils
7 | from setuptools import setup, find_packages
8 | # To use a consistent encoding
9 | from codecs import open
10 | from os import path
11 |
12 | here = path.abspath(path.dirname(__file__))
13 |
14 | # Get the long description from the README file
15 | with open(path.join(here, 'README.rst'), encoding='utf-8') as f:
16 | long_description = f.read()
17 |
18 | setup(
19 | name='TeLL',
20 |
21 | # Versions should comply with PEP440. For a discussion on single-sourcing
22 | # the version across setup.py and the project code, see
23 | # https://packaging.python.org/en/latest/single_source_version.html
24 | version='1.0',
25 |
26 | description='Provides a variety of tensorflow-based network layers, flexible (recurrent) network designs, convenience routines for saving and resuming networks, and more!',
27 | long_description=long_description,
28 |
29 | # The project's main homepage.
30 | url='https://github.com/ml-jku/tensorflow-layer-library',
31 |
32 | # Author details
33 | author='Michael Widrich, Markus Hofmarcher',
34 | author_email='widrich@bioinf.jku.at, hofmarcher@bioinf.jku.at',
35 |
36 | # Choose your license
37 | license='MIT License',
38 |
39 | # See https://pypi.python.org/pypi?%3Aaction=list_classifiers
40 | classifiers=[
41 | # How mature is this project? Common values are
42 | # 3 - Alpha
43 | # 4 - Beta
44 | # 5 - Production/Stable
45 | 'Development Status :: 4 - Beta',
46 |
47 | # Indicate who your project is intended for
48 | 'Intended Audience :: Developers',
49 | 'Topic :: Software Development :: Deep Learning',
50 |
51 | # Pick your license as you wish (should match "license" above)
52 | 'License :: OSI Approved :: MIT License',
53 |
54 | # Specify the Python versions you support here. In particular, ensure
55 | # that you indicate whether you support Python 2, Python 3 or both.
56 | 'Programming Language :: Python :: 3.5',
57 | 'Programming Language :: Python :: 3.6',
58 | ],
59 |
60 | # What does your project relate to?
61 | keywords='Library on top of tensorflow for convenient design of deep (recurrent) networks.',
62 |
63 | # You can just specify the packages manually here if your project is
64 | # simple. Or you can use find_packages().
65 | packages=find_packages(),
66 |
67 | # List run-time dependencies here. These will be installed by pip when
68 | # your project is installed. For an analysis of "install_requires" vs pip's
69 | # requirements files see:
70 | # https://packaging.python.org/en/latest/requirements.html
71 | install_requires=['matplotlib>=2',
72 | 'numpy>=1.12.0',
73 | 'pandas>=0.19.0',
74 | 'Pillow>=4.0.0',
75 | 'natsort>=5.0.2',
76 | 'progressbar2>=3.10.0',
77 | 'multiprocess>=0.70.5'
78 | ],
79 |
80 | extras_require={
81 | 'tensorflow-gpu': ["tensorflow-gpu>=1.0.0"],
82 | 'tensorflow': ["tensorflow>=1.0.0"]
83 | },
84 |
85 | # To provide executable scripts, use entry points in preference to the
86 | # "scripts" keyword. Entry points provide cross-platform support and allow
87 | # pip to create the appropriate form of executable for the target platform.
88 | entry_points={
89 | 'console_scripts': [
90 | 'tell-resume=TeLL.scripts.resume:main',
91 | 'tell-ising-dropout=TeLL.scripts.dropoutmask:main',
92 | ],
93 | }
94 | )
95 |
--------------------------------------------------------------------------------