├── .gitignore
├── LICENSE
├── README.md
├── ladder
    ├── decoder.py
    ├── encoder.py
    └── ladder.py
├── logs
    ├── ladder.extra_noise_0.05.log
    ├── ladder_supervised_only.log
    └── ladder_supervised_unsupervised.log
├── stacked_denoising_autoencoder
    ├── autoencoder.py
    └── sda.py
└── utils
    └── mnist_data.py


/.gitignore:
--------------------------------------------------------------------------------
 1 | # data
 2 | data/
 3 | 
 4 | # Pycharm files
 5 | .idea/
 6 | 
 7 | # Byte-compiled / optimized / DLL files
 8 | __pycache__/
 9 | *.py[cod]
10 | *$py.class
11 | 
12 | # C extensions
13 | *.so
14 | 
15 | # Distribution / packaging
16 | .Python
17 | env/
18 | build/
19 | develop-eggs/
20 | dist/
21 | downloads/
22 | eggs/
23 | .eggs/
24 | lib/
25 | lib64/
26 | parts/
27 | sdist/
28 | var/
29 | *.egg-info/
30 | .installed.cfg
31 | *.egg
32 | 
33 | # PyInstaller
34 | #  Usually these files are written by a python script from a template
35 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
36 | *.manifest
37 | *.spec
38 | 
39 | # Installer logs
40 | pip-log.txt
41 | pip-delete-this-directory.txt
42 | 
43 | # Unit test / coverage reports
44 | htmlcov/
45 | .tox/
46 | .coverage
47 | .coverage.*
48 | .cache
49 | nosetests.xml
50 | coverage.xml
51 | *,cover
52 | .hypothesis/
53 | 
54 | # Translations
55 | *.mo
56 | *.pot
57 | 
58 | # Django stuff:
59 | # *.log
60 | local_settings.py
61 | 
62 | # Flask stuff:
63 | instance/
64 | .webassets-cache
65 | 
66 | # Scrapy stuff:
67 | .scrapy
68 | 
69 | # Sphinx documentation
70 | docs/_build/
71 | 
72 | # PyBuilder
73 | target/
74 | 
75 | # IPython Notebook
76 | .ipynb_checkpoints
77 | 
78 | # pyenv
79 | .python-version
80 | 
81 | # celery beat schedule file
82 | celerybeat-schedule
83 | 
84 | # dotenv
85 | .env
86 | 
87 | # virtualenv
88 | venv/
89 | ENV/
90 | 
91 | # Spyder project settings
92 | .spyderproject
93 | 
94 | # Rope project settings
95 | .ropeproject
96 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                     GNU GENERAL PUBLIC LICENSE
  2 |                        Version 3, 29 June 2007
  3 | 
  4 |  Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
  5 |  Everyone is permitted to copy and distribute verbatim copies
  6 |  of this license document, but changing it is not allowed.
  7 | 
  8 |                             Preamble
  9 | 
 10 |   The GNU General Public License is a free, copyleft license for
 11 | software and other kinds of works.
 12 | 
 13 |   The licenses for most software and other practical works are designed
 14 | to take away your freedom to share and change the works.  By contrast,
 15 | the GNU General Public License is intended to guarantee your freedom to
 16 | share and change all versions of a program--to make sure it remains free
 17 | software for all its users.  We, the Free Software Foundation, use the
 18 | GNU General Public License for most of our software; it applies also to
 19 | any other work released this way by its authors.  You can apply it to
 20 | your programs, too.
 21 | 
 22 |   When we speak of free software, we are referring to freedom, not
 23 | price.  Our General Public Licenses are designed to make sure that you
 24 | have the freedom to distribute copies of free software (and charge for
 25 | them if you wish), that you receive source code or can get it if you
 26 | want it, that you can change the software or use pieces of it in new
 27 | free programs, and that you know you can do these things.
 28 | 
 29 |   To protect your rights, we need to prevent others from denying you
 30 | these rights or asking you to surrender the rights.  Therefore, you have
 31 | certain responsibilities if you distribute copies of the software, or if
 32 | you modify it: responsibilities to respect the freedom of others.
 33 | 
 34 |   For example, if you distribute copies of such a program, whether
 35 | gratis or for a fee, you must pass on to the recipients the same
 36 | freedoms that you received.  You must make sure that they, too, receive
 37 | or can get the source code.  And you must show them these terms so they
 38 | know their rights.
 39 | 
 40 |   Developers that use the GNU GPL protect your rights with two steps:
 41 | (1) assert copyright on the software, and (2) offer you this License
 42 | giving you legal permission to copy, distribute and/or modify it.
 43 | 
 44 |   For the developers' and authors' protection, the GPL clearly explains
 45 | that there is no warranty for this free software.  For both users' and
 46 | authors' sake, the GPL requires that modified versions be marked as
 47 | changed, so that their problems will not be attributed erroneously to
 48 | authors of previous versions.
 49 | 
 50 |   Some devices are designed to deny users access to install or run
 51 | modified versions of the software inside them, although the manufacturer
 52 | can do so.  This is fundamentally incompatible with the aim of
 53 | protecting users' freedom to change the software.  The systematic
 54 | pattern of such abuse occurs in the area of products for individuals to
 55 | use, which is precisely where it is most unacceptable.  Therefore, we
 56 | have designed this version of the GPL to prohibit the practice for those
 57 | products.  If such problems arise substantially in other domains, we
 58 | stand ready to extend this provision to those domains in future versions
 59 | of the GPL, as needed to protect the freedom of users.
 60 | 
 61 |   Finally, every program is threatened constantly by software patents.
 62 | States should not allow patents to restrict development and use of
 63 | software on general-purpose computers, but in those that do, we wish to
 64 | avoid the special danger that patents applied to a free program could
 65 | make it effectively proprietary.  To prevent this, the GPL assures that
 66 | patents cannot be used to render the program non-free.
 67 | 
 68 |   The precise terms and conditions for copying, distribution and
 69 | modification follow.
 70 | 
 71 |                        TERMS AND CONDITIONS
 72 | 
 73 |   0. Definitions.
 74 | 
 75 |   "This License" refers to version 3 of the GNU General Public License.
 76 | 
 77 |   "Copyright" also means copyright-like laws that apply to other kinds of
 78 | works, such as semiconductor masks.
 79 | 
 80 |   "The Program" refers to any copyrightable work licensed under this
 81 | License.  Each licensee is addressed as "you".  "Licensees" and
 82 | "recipients" may be individuals or organizations.
 83 | 
 84 |   To "modify" a work means to copy from or adapt all or part of the work
 85 | in a fashion requiring copyright permission, other than the making of an
 86 | exact copy.  The resulting work is called a "modified version" of the
 87 | earlier work or a work "based on" the earlier work.
 88 | 
 89 |   A "covered work" means either the unmodified Program or a work based
 90 | on the Program.
 91 | 
 92 |   To "propagate" a work means to do anything with it that, without
 93 | permission, would make you directly or secondarily liable for
 94 | infringement under applicable copyright law, except executing it on a
 95 | computer or modifying a private copy.  Propagation includes copying,
 96 | distribution (with or without modification), making available to the
 97 | public, and in some countries other activities as well.
 98 | 
 99 |   To "convey" a work means any kind of propagation that enables other
100 | parties to make or receive copies.  Mere interaction with a user through
101 | a computer network, with no transfer of a copy, is not conveying.
102 | 
103 |   An interactive user interface displays "Appropriate Legal Notices"
104 | to the extent that it includes a convenient and prominently visible
105 | feature that (1) displays an appropriate copyright notice, and (2)
106 | tells the user that there is no warranty for the work (except to the
107 | extent that warranties are provided), that licensees may convey the
108 | work under this License, and how to view a copy of this License.  If
109 | the interface presents a list of user commands or options, such as a
110 | menu, a prominent item in the list meets this criterion.
111 | 
112 |   1. Source Code.
113 | 
114 |   The "source code" for a work means the preferred form of the work
115 | for making modifications to it.  "Object code" means any non-source
116 | form of a work.
117 | 
118 |   A "Standard Interface" means an interface that either is an official
119 | standard defined by a recognized standards body, or, in the case of
120 | interfaces specified for a particular programming language, one that
121 | is widely used among developers working in that language.
122 | 
123 |   The "System Libraries" of an executable work include anything, other
124 | than the work as a whole, that (a) is included in the normal form of
125 | packaging a Major Component, but which is not part of that Major
126 | Component, and (b) serves only to enable use of the work with that
127 | Major Component, or to implement a Standard Interface for which an
128 | implementation is available to the public in source code form.  A
129 | "Major Component", in this context, means a major essential component
130 | (kernel, window system, and so on) of the specific operating system
131 | (if any) on which the executable work runs, or a compiler used to
132 | produce the work, or an object code interpreter used to run it.
133 | 
134 |   The "Corresponding Source" for a work in object code form means all
135 | the source code needed to generate, install, and (for an executable
136 | work) run the object code and to modify the work, including scripts to
137 | control those activities.  However, it does not include the work's
138 | System Libraries, or general-purpose tools or generally available free
139 | programs which are used unmodified in performing those activities but
140 | which are not part of the work.  For example, Corresponding Source
141 | includes interface definition files associated with source files for
142 | the work, and the source code for shared libraries and dynamically
143 | linked subprograms that the work is specifically designed to require,
144 | such as by intimate data communication or control flow between those
145 | subprograms and other parts of the work.
146 | 
147 |   The Corresponding Source need not include anything that users
148 | can regenerate automatically from other parts of the Corresponding
149 | Source.
150 | 
151 |   The Corresponding Source for a work in source code form is that
152 | same work.
153 | 
154 |   2. Basic Permissions.
155 | 
156 |   All rights granted under this License are granted for the term of
157 | copyright on the Program, and are irrevocable provided the stated
158 | conditions are met.  This License explicitly affirms your unlimited
159 | permission to run the unmodified Program.  The output from running a
160 | covered work is covered by this License only if the output, given its
161 | content, constitutes a covered work.  This License acknowledges your
162 | rights of fair use or other equivalent, as provided by copyright law.
163 | 
164 |   You may make, run and propagate covered works that you do not
165 | convey, without conditions so long as your license otherwise remains
166 | in force.  You may convey covered works to others for the sole purpose
167 | of having them make modifications exclusively for you, or provide you
168 | with facilities for running those works, provided that you comply with
169 | the terms of this License in conveying all material for which you do
170 | not control copyright.  Those thus making or running the covered works
171 | for you must do so exclusively on your behalf, under your direction
172 | and control, on terms that prohibit them from making any copies of
173 | your copyrighted material outside their relationship with you.
174 | 
175 |   Conveying under any other circumstances is permitted solely under
176 | the conditions stated below.  Sublicensing is not allowed; section 10
177 | makes it unnecessary.
178 | 
179 |   3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180 | 
181 |   No covered work shall be deemed part of an effective technological
182 | measure under any applicable law fulfilling obligations under article
183 | 11 of the WIPO copyright treaty adopted on 20 December 1996, or
184 | similar laws prohibiting or restricting circumvention of such
185 | measures.
186 | 
187 |   When you convey a covered work, you waive any legal power to forbid
188 | circumvention of technological measures to the extent such circumvention
189 | is effected by exercising rights under this License with respect to
190 | the covered work, and you disclaim any intention to limit operation or
191 | modification of the work as a means of enforcing, against the work's
192 | users, your or third parties' legal rights to forbid circumvention of
193 | technological measures.
194 | 
195 |   4. Conveying Verbatim Copies.
196 | 
197 |   You may convey verbatim copies of the Program's source code as you
198 | receive it, in any medium, provided that you conspicuously and
199 | appropriately publish on each copy an appropriate copyright notice;
200 | keep intact all notices stating that this License and any
201 | non-permissive terms added in accord with section 7 apply to the code;
202 | keep intact all notices of the absence of any warranty; and give all
203 | recipients a copy of this License along with the Program.
204 | 
205 |   You may charge any price or no price for each copy that you convey,
206 | and you may offer support or warranty protection for a fee.
207 | 
208 |   5. Conveying Modified Source Versions.
209 | 
210 |   You may convey a work based on the Program, or the modifications to
211 | produce it from the Program, in the form of source code under the
212 | terms of section 4, provided that you also meet all of these conditions:
213 | 
214 |     a) The work must carry prominent notices stating that you modified
215 |     it, and giving a relevant date.
216 | 
217 |     b) The work must carry prominent notices stating that it is
218 |     released under this License and any conditions added under section
219 |     7.  This requirement modifies the requirement in section 4 to
220 |     "keep intact all notices".
221 | 
222 |     c) You must license the entire work, as a whole, under this
223 |     License to anyone who comes into possession of a copy.  This
224 |     License will therefore apply, along with any applicable section 7
225 |     additional terms, to the whole of the work, and all its parts,
226 |     regardless of how they are packaged.  This License gives no
227 |     permission to license the work in any other way, but it does not
228 |     invalidate such permission if you have separately received it.
229 | 
230 |     d) If the work has interactive user interfaces, each must display
231 |     Appropriate Legal Notices; however, if the Program has interactive
232 |     interfaces that do not display Appropriate Legal Notices, your
233 |     work need not make them do so.
234 | 
235 |   A compilation of a covered work with other separate and independent
236 | works, which are not by their nature extensions of the covered work,
237 | and which are not combined with it such as to form a larger program,
238 | in or on a volume of a storage or distribution medium, is called an
239 | "aggregate" if the compilation and its resulting copyright are not
240 | used to limit the access or legal rights of the compilation's users
241 | beyond what the individual works permit.  Inclusion of a covered work
242 | in an aggregate does not cause this License to apply to the other
243 | parts of the aggregate.
244 | 
245 |   6. Conveying Non-Source Forms.
246 | 
247 |   You may convey a covered work in object code form under the terms
248 | of sections 4 and 5, provided that you also convey the
249 | machine-readable Corresponding Source under the terms of this License,
250 | in one of these ways:
251 | 
252 |     a) Convey the object code in, or embodied in, a physical product
253 |     (including a physical distribution medium), accompanied by the
254 |     Corresponding Source fixed on a durable physical medium
255 |     customarily used for software interchange.
256 | 
257 |     b) Convey the object code in, or embodied in, a physical product
258 |     (including a physical distribution medium), accompanied by a
259 |     written offer, valid for at least three years and valid for as
260 |     long as you offer spare parts or customer support for that product
261 |     model, to give anyone who possesses the object code either (1) a
262 |     copy of the Corresponding Source for all the software in the
263 |     product that is covered by this License, on a durable physical
264 |     medium customarily used for software interchange, for a price no
265 |     more than your reasonable cost of physically performing this
266 |     conveying of source, or (2) access to copy the
267 |     Corresponding Source from a network server at no charge.
268 | 
269 |     c) Convey individual copies of the object code with a copy of the
270 |     written offer to provide the Corresponding Source.  This
271 |     alternative is allowed only occasionally and noncommercially, and
272 |     only if you received the object code with such an offer, in accord
273 |     with subsection 6b.
274 | 
275 |     d) Convey the object code by offering access from a designated
276 |     place (gratis or for a charge), and offer equivalent access to the
277 |     Corresponding Source in the same way through the same place at no
278 |     further charge.  You need not require recipients to copy the
279 |     Corresponding Source along with the object code.  If the place to
280 |     copy the object code is a network server, the Corresponding Source
281 |     may be on a different server (operated by you or a third party)
282 |     that supports equivalent copying facilities, provided you maintain
283 |     clear directions next to the object code saying where to find the
284 |     Corresponding Source.  Regardless of what server hosts the
285 |     Corresponding Source, you remain obligated to ensure that it is
286 |     available for as long as needed to satisfy these requirements.
287 | 
288 |     e) Convey the object code using peer-to-peer transmission, provided
289 |     you inform other peers where the object code and Corresponding
290 |     Source of the work are being offered to the general public at no
291 |     charge under subsection 6d.
292 | 
293 |   A separable portion of the object code, whose source code is excluded
294 | from the Corresponding Source as a System Library, need not be
295 | included in conveying the object code work.
296 | 
297 |   A "User Product" is either (1) a "consumer product", which means any
298 | tangible personal property which is normally used for personal, family,
299 | or household purposes, or (2) anything designed or sold for incorporation
300 | into a dwelling.  In determining whether a product is a consumer product,
301 | doubtful cases shall be resolved in favor of coverage.  For a particular
302 | product received by a particular user, "normally used" refers to a
303 | typical or common use of that class of product, regardless of the status
304 | of the particular user or of the way in which the particular user
305 | actually uses, or expects or is expected to use, the product.  A product
306 | is a consumer product regardless of whether the product has substantial
307 | commercial, industrial or non-consumer uses, unless such uses represent
308 | the only significant mode of use of the product.
309 | 
310 |   "Installation Information" for a User Product means any methods,
311 | procedures, authorization keys, or other information required to install
312 | and execute modified versions of a covered work in that User Product from
313 | a modified version of its Corresponding Source.  The information must
314 | suffice to ensure that the continued functioning of the modified object
315 | code is in no case prevented or interfered with solely because
316 | modification has been made.
317 | 
318 |   If you convey an object code work under this section in, or with, or
319 | specifically for use in, a User Product, and the conveying occurs as
320 | part of a transaction in which the right of possession and use of the
321 | User Product is transferred to the recipient in perpetuity or for a
322 | fixed term (regardless of how the transaction is characterized), the
323 | Corresponding Source conveyed under this section must be accompanied
324 | by the Installation Information.  But this requirement does not apply
325 | if neither you nor any third party retains the ability to install
326 | modified object code on the User Product (for example, the work has
327 | been installed in ROM).
328 | 
329 |   The requirement to provide Installation Information does not include a
330 | requirement to continue to provide support service, warranty, or updates
331 | for a work that has been modified or installed by the recipient, or for
332 | the User Product in which it has been modified or installed.  Access to a
333 | network may be denied when the modification itself materially and
334 | adversely affects the operation of the network or violates the rules and
335 | protocols for communication across the network.
336 | 
337 |   Corresponding Source conveyed, and Installation Information provided,
338 | in accord with this section must be in a format that is publicly
339 | documented (and with an implementation available to the public in
340 | source code form), and must require no special password or key for
341 | unpacking, reading or copying.
342 | 
343 |   7. Additional Terms.
344 | 
345 |   "Additional permissions" are terms that supplement the terms of this
346 | License by making exceptions from one or more of its conditions.
347 | Additional permissions that are applicable to the entire Program shall
348 | be treated as though they were included in this License, to the extent
349 | that they are valid under applicable law.  If additional permissions
350 | apply only to part of the Program, that part may be used separately
351 | under those permissions, but the entire Program remains governed by
352 | this License without regard to the additional permissions.
353 | 
354 |   When you convey a copy of a covered work, you may at your option
355 | remove any additional permissions from that copy, or from any part of
356 | it.  (Additional permissions may be written to require their own
357 | removal in certain cases when you modify the work.)  You may place
358 | additional permissions on material, added by you to a covered work,
359 | for which you have or can give appropriate copyright permission.
360 | 
361 |   Notwithstanding any other provision of this License, for material you
362 | add to a covered work, you may (if authorized by the copyright holders of
363 | that material) supplement the terms of this License with terms:
364 | 
365 |     a) Disclaiming warranty or limiting liability differently from the
366 |     terms of sections 15 and 16 of this License; or
367 | 
368 |     b) Requiring preservation of specified reasonable legal notices or
369 |     author attributions in that material or in the Appropriate Legal
370 |     Notices displayed by works containing it; or
371 | 
372 |     c) Prohibiting misrepresentation of the origin of that material, or
373 |     requiring that modified versions of such material be marked in
374 |     reasonable ways as different from the original version; or
375 | 
376 |     d) Limiting the use for publicity purposes of names of licensors or
377 |     authors of the material; or
378 | 
379 |     e) Declining to grant rights under trademark law for use of some
380 |     trade names, trademarks, or service marks; or
381 | 
382 |     f) Requiring indemnification of licensors and authors of that
383 |     material by anyone who conveys the material (or modified versions of
384 |     it) with contractual assumptions of liability to the recipient, for
385 |     any liability that these contractual assumptions directly impose on
386 |     those licensors and authors.
387 | 
388 |   All other non-permissive additional terms are considered "further
389 | restrictions" within the meaning of section 10.  If the Program as you
390 | received it, or any part of it, contains a notice stating that it is
391 | governed by this License along with a term that is a further
392 | restriction, you may remove that term.  If a license document contains
393 | a further restriction but permits relicensing or conveying under this
394 | License, you may add to a covered work material governed by the terms
395 | of that license document, provided that the further restriction does
396 | not survive such relicensing or conveying.
397 | 
398 |   If you add terms to a covered work in accord with this section, you
399 | must place, in the relevant source files, a statement of the
400 | additional terms that apply to those files, or a notice indicating
401 | where to find the applicable terms.
402 | 
403 |   Additional terms, permissive or non-permissive, may be stated in the
404 | form of a separately written license, or stated as exceptions;
405 | the above requirements apply either way.
406 | 
407 |   8. Termination.
408 | 
409 |   You may not propagate or modify a covered work except as expressly
410 | provided under this License.  Any attempt otherwise to propagate or
411 | modify it is void, and will automatically terminate your rights under
412 | this License (including any patent licenses granted under the third
413 | paragraph of section 11).
414 | 
415 |   However, if you cease all violation of this License, then your
416 | license from a particular copyright holder is reinstated (a)
417 | provisionally, unless and until the copyright holder explicitly and
418 | finally terminates your license, and (b) permanently, if the copyright
419 | holder fails to notify you of the violation by some reasonable means
420 | prior to 60 days after the cessation.
421 | 
422 |   Moreover, your license from a particular copyright holder is
423 | reinstated permanently if the copyright holder notifies you of the
424 | violation by some reasonable means, this is the first time you have
425 | received notice of violation of this License (for any work) from that
426 | copyright holder, and you cure the violation prior to 30 days after
427 | your receipt of the notice.
428 | 
429 |   Termination of your rights under this section does not terminate the
430 | licenses of parties who have received copies or rights from you under
431 | this License.  If your rights have been terminated and not permanently
432 | reinstated, you do not qualify to receive new licenses for the same
433 | material under section 10.
434 | 
435 |   9. Acceptance Not Required for Having Copies.
436 | 
437 |   You are not required to accept this License in order to receive or
438 | run a copy of the Program.  Ancillary propagation of a covered work
439 | occurring solely as a consequence of using peer-to-peer transmission
440 | to receive a copy likewise does not require acceptance.  However,
441 | nothing other than this License grants you permission to propagate or
442 | modify any covered work.  These actions infringe copyright if you do
443 | not accept this License.  Therefore, by modifying or propagating a
444 | covered work, you indicate your acceptance of this License to do so.
445 | 
446 |   10. Automatic Licensing of Downstream Recipients.
447 | 
448 |   Each time you convey a covered work, the recipient automatically
449 | receives a license from the original licensors, to run, modify and
450 | propagate that work, subject to this License.  You are not responsible
451 | for enforcing compliance by third parties with this License.
452 | 
453 |   An "entity transaction" is a transaction transferring control of an
454 | organization, or substantially all assets of one, or subdividing an
455 | organization, or merging organizations.  If propagation of a covered
456 | work results from an entity transaction, each party to that
457 | transaction who receives a copy of the work also receives whatever
458 | licenses to the work the party's predecessor in interest had or could
459 | give under the previous paragraph, plus a right to possession of the
460 | Corresponding Source of the work from the predecessor in interest, if
461 | the predecessor has it or can get it with reasonable efforts.
462 | 
463 |   You may not impose any further restrictions on the exercise of the
464 | rights granted or affirmed under this License.  For example, you may
465 | not impose a license fee, royalty, or other charge for exercise of
466 | rights granted under this License, and you may not initiate litigation
467 | (including a cross-claim or counterclaim in a lawsuit) alleging that
468 | any patent claim is infringed by making, using, selling, offering for
469 | sale, or importing the Program or any portion of it.
470 | 
471 |   11. Patents.
472 | 
473 |   A "contributor" is a copyright holder who authorizes use under this
474 | License of the Program or a work on which the Program is based.  The
475 | work thus licensed is called the contributor's "contributor version".
476 | 
477 |   A contributor's "essential patent claims" are all patent claims
478 | owned or controlled by the contributor, whether already acquired or
479 | hereafter acquired, that would be infringed by some manner, permitted
480 | by this License, of making, using, or selling its contributor version,
481 | but do not include claims that would be infringed only as a
482 | consequence of further modification of the contributor version.  For
483 | purposes of this definition, "control" includes the right to grant
484 | patent sublicenses in a manner consistent with the requirements of
485 | this License.
486 | 
487 |   Each contributor grants you a non-exclusive, worldwide, royalty-free
488 | patent license under the contributor's essential patent claims, to
489 | make, use, sell, offer for sale, import and otherwise run, modify and
490 | propagate the contents of its contributor version.
491 | 
492 |   In the following three paragraphs, a "patent license" is any express
493 | agreement or commitment, however denominated, not to enforce a patent
494 | (such as an express permission to practice a patent or covenant not to
495 | sue for patent infringement).  To "grant" such a patent license to a
496 | party means to make such an agreement or commitment not to enforce a
497 | patent against the party.
498 | 
499 |   If you convey a covered work, knowingly relying on a patent license,
500 | and the Corresponding Source of the work is not available for anyone
501 | to copy, free of charge and under the terms of this License, through a
502 | publicly available network server or other readily accessible means,
503 | then you must either (1) cause the Corresponding Source to be so
504 | available, or (2) arrange to deprive yourself of the benefit of the
505 | patent license for this particular work, or (3) arrange, in a manner
506 | consistent with the requirements of this License, to extend the patent
507 | license to downstream recipients.  "Knowingly relying" means you have
508 | actual knowledge that, but for the patent license, your conveying the
509 | covered work in a country, or your recipient's use of the covered work
510 | in a country, would infringe one or more identifiable patents in that
511 | country that you have reason to believe are valid.
512 | 
513 |   If, pursuant to or in connection with a single transaction or
514 | arrangement, you convey, or propagate by procuring conveyance of, a
515 | covered work, and grant a patent license to some of the parties
516 | receiving the covered work authorizing them to use, propagate, modify
517 | or convey a specific copy of the covered work, then the patent license
518 | you grant is automatically extended to all recipients of the covered
519 | work and works based on it.
520 | 
521 |   A patent license is "discriminatory" if it does not include within
522 | the scope of its coverage, prohibits the exercise of, or is
523 | conditioned on the non-exercise of one or more of the rights that are
524 | specifically granted under this License.  You may not convey a covered
525 | work if you are a party to an arrangement with a third party that is
526 | in the business of distributing software, under which you make payment
527 | to the third party based on the extent of your activity of conveying
528 | the work, and under which the third party grants, to any of the
529 | parties who would receive the covered work from you, a discriminatory
530 | patent license (a) in connection with copies of the covered work
531 | conveyed by you (or copies made from those copies), or (b) primarily
532 | for and in connection with specific products or compilations that
533 | contain the covered work, unless you entered into that arrangement,
534 | or that patent license was granted, prior to 28 March 2007.
535 | 
536 |   Nothing in this License shall be construed as excluding or limiting
537 | any implied license or other defenses to infringement that may
538 | otherwise be available to you under applicable patent law.
539 | 
540 |   12. No Surrender of Others' Freedom.
541 | 
542 |   If conditions are imposed on you (whether by court order, agreement or
543 | otherwise) that contradict the conditions of this License, they do not
544 | excuse you from the conditions of this License.  If you cannot convey a
545 | covered work so as to satisfy simultaneously your obligations under this
546 | License and any other pertinent obligations, then as a consequence you may
547 | not convey it at all.  For example, if you agree to terms that obligate you
548 | to collect a royalty for further conveying from those to whom you convey
549 | the Program, the only way you could satisfy both those terms and this
550 | License would be to refrain entirely from conveying the Program.
551 | 
552 |   13. Use with the GNU Affero General Public License.
553 | 
554 |   Notwithstanding any other provision of this License, you have
555 | permission to link or combine any covered work with a work licensed
556 | under version 3 of the GNU Affero General Public License into a single
557 | combined work, and to convey the resulting work.  The terms of this
558 | License will continue to apply to the part which is the covered work,
559 | but the special requirements of the GNU Affero General Public License,
560 | section 13, concerning interaction through a network will apply to the
561 | combination as such.
562 | 
563 |   14. Revised Versions of this License.
564 | 
565 |   The Free Software Foundation may publish revised and/or new versions of
566 | the GNU General Public License from time to time.  Such new versions will
567 | be similar in spirit to the present version, but may differ in detail to
568 | address new problems or concerns.
569 | 
570 |   Each version is given a distinguishing version number.  If the
571 | Program specifies that a certain numbered version of the GNU General
572 | Public License "or any later version" applies to it, you have the
573 | option of following the terms and conditions either of that numbered
574 | version or of any later version published by the Free Software
575 | Foundation.  If the Program does not specify a version number of the
576 | GNU General Public License, you may choose any version ever published
577 | by the Free Software Foundation.
578 | 
579 |   If the Program specifies that a proxy can decide which future
580 | versions of the GNU General Public License can be used, that proxy's
581 | public statement of acceptance of a version permanently authorizes you
582 | to choose that version for the Program.
583 | 
584 |   Later license versions may give you additional or different
585 | permissions.  However, no additional obligations are imposed on any
586 | author or copyright holder as a result of your choosing to follow a
587 | later version.
588 | 
589 |   15. Disclaimer of Warranty.
590 | 
591 |   THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592 | APPLICABLE LAW.  EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593 | HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594 | OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596 | PURPOSE.  THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597 | IS WITH YOU.  SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598 | ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599 | 
600 |   16. Limitation of Liability.
601 | 
602 |   IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604 | THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605 | GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606 | USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607 | DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608 | PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609 | EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610 | SUCH DAMAGES.
611 | 
612 |   17. Interpretation of Sections 15 and 16.
613 | 
614 |   If the disclaimer of warranty and limitation of liability provided
615 | above cannot be given local legal effect according to their terms,
616 | reviewing courts shall apply local law that most closely approximates
617 | an absolute waiver of all civil liability in connection with the
618 | Program, unless a warranty or assumption of liability accompanies a
619 | copy of the Program in return for a fee.
620 | 
621 |                      END OF TERMS AND CONDITIONS
622 | 
623 |             How to Apply These Terms to Your New Programs
624 | 
625 |   If you develop a new program, and you want it to be of the greatest
626 | possible use to the public, the best way to achieve this is to make it
627 | free software which everyone can redistribute and change under these terms.
628 | 
629 |   To do so, attach the following notices to the program.  It is safest
630 | to attach them to the start of each source file to most effectively
631 | state the exclusion of warranty; and each file should have at least
632 | the "copyright" line and a pointer to where the full notice is found.
633 | 
634 |     {one line to give the program's name and a brief idea of what it does.}
635 |     Copyright (C) {year}  {name of author}
636 | 
637 |     This program is free software: you can redistribute it and/or modify
638 |     it under the terms of the GNU General Public License as published by
639 |     the Free Software Foundation, either version 3 of the License, or
640 |     (at your option) any later version.
641 | 
642 |     This program is distributed in the hope that it will be useful,
643 |     but WITHOUT ANY WARRANTY; without even the implied warranty of
644 |     MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
645 |     GNU General Public License for more details.
646 | 
647 |     You should have received a copy of the GNU General Public License
648 |     along with this program.  If not, see <http://www.gnu.org/licenses/>.
649 | 
650 | Also add information on how to contact you by electronic and paper mail.
651 | 
652 |   If the program does terminal interaction, make it output a short
653 | notice like this when it starts in an interactive mode:
654 | 
655 |     {project}  Copyright (C) {year}  {fullname}
656 |     This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657 |     This is free software, and you are welcome to redistribute it
658 |     under certain conditions; type `show c' for details.
659 | 
660 | The hypothetical commands `show w' and `show c' should show the appropriate
661 | parts of the General Public License.  Of course, your program's commands
662 | might be different; for a GUI interface, you would use an "about box".
663 | 
664 |   You should also get your employer (if you work as a programmer) or school,
665 | if any, to sign a "copyright disclaimer" for the program, if necessary.
666 | For more information on this, and how to apply and follow the GNU GPL, see
667 | <http://www.gnu.org/licenses/>.
668 | 
669 |   The GNU General Public License does not permit incorporating your program
670 | into proprietary programs.  If your program is a subroutine library, you
671 | may consider it more useful to permit linking proprietary applications with
672 | the library.  If this is what you want to do, use the GNU Lesser General
673 | Public License instead of this License.  But first, please read
674 | <http://www.gnu.org/philosophy/why-not-lgpl.html>.
675 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # ladder
 2 | 
 3 | Implementation of [Ladder Network](https://arxiv.org/abs/1507.02672) and Stacked Denoising Autoencoder in [PyTorch](http://pytorch.org/).
 4 |  
 5 | ### Requirements
 6 | 
 7 | - [PyTorch](http://pytorch.org/)
 8 | 
 9 | ### Training ladder
10 | 
11 | 1. Run ```python utils/mnist_data.py``` to create the [MNIST](http://yann.lecun.com/exdb/mnist/) dataset.
12 | 
13 | 2. Run the following command to train the *ladder* network:
14 |   - ```python ladder/ladder.py --batch 100 --epochs 20 --noise_std 0.2 --data_dir data```
15 | 
16 | **Status**: The unsupervised loss starts at a high value because of which the network overfits the unsupervised loss and the supervised performance is bad. Current best accuracy on MNIST validation set using 3000 labelled and 47000 unlabelled examples: 98.33%.
17 | 


--------------------------------------------------------------------------------
/ladder/decoder.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import torch
  3 | from torch.nn.parameter import Parameter
  4 | from torch.autograd import Variable
  5 | 
  6 | 
  7 | class Decoder(torch.nn.Module):
  8 |     def __init__(self, d_in, d_out, use_cuda):
  9 |         super(Decoder, self).__init__()
 10 | 
 11 |         self.d_in = d_in
 12 |         self.d_out = d_out
 13 |         self.use_cuda = use_cuda
 14 | 
 15 |         if self.use_cuda:
 16 |             self.a1 = Parameter(0. * torch.ones(1, d_in).cuda())
 17 |             self.a2 = Parameter(1. * torch.ones(1, d_in).cuda())
 18 |             self.a3 = Parameter(0. * torch.ones(1, d_in).cuda())
 19 |             self.a4 = Parameter(0. * torch.ones(1, d_in).cuda())
 20 |             self.a5 = Parameter(0. * torch.ones(1, d_in).cuda())
 21 | 
 22 |             self.a6 = Parameter(0. * torch.ones(1, d_in).cuda())
 23 |             self.a7 = Parameter(1. * torch.ones(1, d_in).cuda())
 24 |             self.a8 = Parameter(0. * torch.ones(1, d_in).cuda())
 25 |             self.a9 = Parameter(0. * torch.ones(1, d_in).cuda())
 26 |             self.a10 = Parameter(0. * torch.ones(1, d_in).cuda())
 27 |         else:
 28 |             self.a1 = Parameter(0. * torch.ones(1, d_in))
 29 |             self.a2 = Parameter(1. * torch.ones(1, d_in))
 30 |             self.a3 = Parameter(0. * torch.ones(1, d_in))
 31 |             self.a4 = Parameter(0. * torch.ones(1, d_in))
 32 |             self.a5 = Parameter(0. * torch.ones(1, d_in))
 33 | 
 34 |             self.a6 = Parameter(0. * torch.ones(1, d_in))
 35 |             self.a7 = Parameter(1. * torch.ones(1, d_in))
 36 |             self.a8 = Parameter(0. * torch.ones(1, d_in))
 37 |             self.a9 = Parameter(0. * torch.ones(1, d_in))
 38 |             self.a10 = Parameter(0. * torch.ones(1, d_in))
 39 | 
 40 | 
 41 |         if self.d_out is not None:
 42 |             self.V = torch.nn.Linear(d_in, d_out, bias=False)
 43 |             self.V.weight.data = torch.randn(self.V.weight.data.size()) / np.sqrt(d_in)
 44 |             # batch-normalization for u
 45 |             self.bn_normalize = torch.nn.BatchNorm1d(d_out, affine=False)
 46 | 
 47 |         # buffer for hat_z_l to be used for cost calculation
 48 |         self.buffer_hat_z_l = None
 49 | 
 50 |     def g(self, tilde_z_l, u_l):
 51 |         if self.use_cuda:
 52 |             ones = Parameter(torch.ones(tilde_z_l.size()[0], 1).cuda())
 53 |         else:
 54 |             ones = Parameter(torch.ones(tilde_z_l.size()[0], 1))
 55 | 
 56 |         b_a1 = ones.mm(self.a1)
 57 |         b_a2 = ones.mm(self.a2)
 58 |         b_a3 = ones.mm(self.a3)
 59 |         b_a4 = ones.mm(self.a4)
 60 |         b_a5 = ones.mm(self.a5)
 61 | 
 62 |         b_a6 = ones.mm(self.a6)
 63 |         b_a7 = ones.mm(self.a7)
 64 |         b_a8 = ones.mm(self.a8)
 65 |         b_a9 = ones.mm(self.a9)
 66 |         b_a10 = ones.mm(self.a10)
 67 | 
 68 |         mu_l = torch.mul(b_a1, torch.sigmoid(torch.mul(b_a2, u_l) + b_a3)) + \
 69 |                torch.mul(b_a4, u_l) + \
 70 |                b_a5
 71 | 
 72 |         v_l = torch.mul(b_a6, torch.sigmoid(torch.mul(b_a7, u_l) + b_a8)) + \
 73 |               torch.mul(b_a9, u_l) + \
 74 |               b_a10
 75 | 
 76 |         hat_z_l = torch.mul(tilde_z_l - mu_l, v_l) + mu_l
 77 | 
 78 |         return hat_z_l
 79 | 
 80 |     def forward(self, tilde_z_l, u_l):
 81 |         # hat_z_l will be used for calculating decoder costs
 82 |         hat_z_l = self.g(tilde_z_l, u_l)
 83 |         # store hat_z_l in buffer for cost calculation
 84 |         self.buffer_hat_z_l = hat_z_l
 85 | 
 86 |         if self.d_out is not None:
 87 |             t = self.V.forward(hat_z_l)
 88 |             u_l_below = self.bn_normalize(t)
 89 |             return u_l_below
 90 |         else:
 91 |             return None
 92 | 
 93 | 
 94 | class StackedDecoders(torch.nn.Module):
 95 |     def __init__(self, d_in, d_decoders, image_size, use_cuda):
 96 |         super(StackedDecoders, self).__init__()
 97 |         self.bn_u_top = torch.nn.BatchNorm1d(d_in, affine=False)
 98 |         self.decoders_ref = []
 99 |         self.decoders = torch.nn.Sequential()
100 |         self.use_cuda = use_cuda
101 |         n_decoders = len(d_decoders)
102 |         for i in range(n_decoders):
103 |             if i == 0:
104 |                 d_input = d_in
105 |             else:
106 |                 d_input = d_decoders[i - 1]
107 |             d_output = d_decoders[i]
108 |             decoder_ref = "decoder_" + str(i)
109 |             decoder = Decoder(d_input, d_output, use_cuda)
110 |             self.decoders_ref.append(decoder_ref)
111 |             self.decoders.add_module(decoder_ref, decoder)
112 | 
113 |         self.bottom_decoder = Decoder(image_size, None, use_cuda)
114 | 
115 |     def forward(self, tilde_z_layers, u_top, tilde_z_bottom):
116 |         # Note that tilde_z_layers should be in reversed order of encoders
117 |         hat_z = []
118 |         u = self.bn_u_top(u_top)
119 |         for i in range(len(self.decoders_ref)):
120 |             d_ref = self.decoders_ref[i]
121 |             decoder = getattr(self.decoders, d_ref)
122 |             tilde_z = tilde_z_layers[i]
123 |             u = decoder.forward(tilde_z, u)
124 |             hat_z.append(decoder.buffer_hat_z_l)
125 |         self.bottom_decoder.forward(tilde_z_bottom, u)
126 |         hat_z_bottom = self.bottom_decoder.buffer_hat_z_l.clone()
127 |         hat_z.append(hat_z_bottom)
128 |         return hat_z
129 | 
130 |     def bn_hat_z_layers(self, hat_z_layers, z_pre_layers):
131 |         # TODO: Calculate batchnorm using GPU Tensors.
132 |         assert len(hat_z_layers) == len(z_pre_layers)
133 |         hat_z_layers_normalized = []
134 |         for i, (hat_z, z_pre) in enumerate(zip(hat_z_layers, z_pre_layers)):
135 |             if self.use_cuda:
136 |                 ones = Variable(torch.ones(z_pre.size()[0], 1).cuda())
137 |             else:
138 |                 ones = Variable(torch.ones(z_pre.size()[0], 1))
139 |             mean = torch.mean(z_pre, 0)
140 |             noise_var = np.random.normal(loc=0.0, scale=1 - 1e-10, size=z_pre.size())
141 |             if self.use_cuda:
142 |                 var = np.var(z_pre.data.cpu().numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
143 |             else:
144 |                 var = np.var(z_pre.data.numpy() + noise_var, axis=0).reshape(1, z_pre.size()[1])
145 |             var = Variable(torch.FloatTensor(var))
146 |             if self.use_cuda:
147 |                 hat_z = hat_z.cpu()
148 |                 ones = ones.cpu()
149 |                 mean = mean.cpu()
150 |             hat_z_normalized = torch.div(hat_z - ones.mm(mean), ones.mm(torch.sqrt(var + 1e-10)))
151 |             if self.use_cuda:
152 |                 hat_z_normalized = hat_z_normalized.cuda()
153 |             hat_z_layers_normalized.append(hat_z_normalized)
154 |         return hat_z_layers_normalized
155 | 


--------------------------------------------------------------------------------
/ladder/encoder.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import torch
  3 | from torch.nn.parameter import Parameter
  4 | from torch.autograd import Variable
  5 | 
  6 | 
  7 | class Encoder(torch.nn.Module):
  8 |     def __init__(self, d_in, d_out, activation_type,
  9 |                  train_bn_scaling, noise_level, use_cuda):
 10 |         super(Encoder, self).__init__()
 11 |         self.d_in = d_in
 12 |         self.d_out = d_out
 13 |         self.activation_type = activation_type
 14 |         self.train_bn_scaling = train_bn_scaling
 15 |         self.noise_level = noise_level
 16 |         self.use_cuda = use_cuda
 17 | 
 18 |         # Encoder
 19 |         # Encoder only uses W matrix, no bias
 20 |         self.linear = torch.nn.Linear(d_in, d_out, bias=False)
 21 |         self.linear.weight.data = torch.randn(self.linear.weight.data.size()) / np.sqrt(d_in)
 22 | 
 23 |         # Batch Normalization
 24 |         # For Relu Beta of batch-norm is redundant, hence only Gamma is trained
 25 |         # For Softmax Beta, Gamma are trained
 26 |         # batch-normalization bias
 27 |         self.bn_normalize_clean = torch.nn.BatchNorm1d(d_out, affine=False)
 28 |         self.bn_normalize = torch.nn.BatchNorm1d(d_out, affine=False)
 29 |         if self.use_cuda:
 30 |             self.bn_beta = Parameter(torch.cuda.FloatTensor(1, d_out))
 31 |         else:
 32 |             self.bn_beta = Parameter(torch.FloatTensor(1, d_out))
 33 |         self.bn_beta.data.zero_()
 34 |         if self.train_bn_scaling:
 35 |             # batch-normalization scaling
 36 |             if self.use_cuda:
 37 |                 self.bn_gamma = Parameter(torch.cuda.FloatTensor(1, d_out))
 38 |                 self.bn_gamma.data = torch.ones(self.bn_gamma.size()).cuda()
 39 |             else:
 40 |                 self.bn_gamma = Parameter(torch.FloatTensor(1, d_out))
 41 |                 self.bn_gamma.data = torch.ones(self.bn_gamma.size())
 42 | 
 43 |         # Activation
 44 |         if activation_type == 'relu':
 45 |             self.activation = torch.nn.ReLU()
 46 |         elif activation_type == 'softmax':
 47 |             self.activation = torch.nn.Softmax()
 48 |         else:
 49 |             raise ValueError("invalid Acitvation type")
 50 | 
 51 |         # buffer for z_pre, z which will be used in decoder cost
 52 |         self.buffer_z_pre = None
 53 |         self.buffer_z = None
 54 |         # buffer for tilde_z which will be used by decoder for reconstruction
 55 |         self.buffer_tilde_z = None
 56 | 
 57 |     def bn_gamma_beta(self, x):
 58 |         if self.use_cuda:
 59 |             ones = Parameter(torch.ones(x.size()[0], 1).cuda())
 60 |         else:
 61 |             ones = Parameter(torch.ones(x.size()[0], 1))
 62 |         t = x + ones.mm(self.bn_beta)
 63 |         if self.train_bn_scaling:
 64 |             t = torch.mul(t, ones.mm(self.bn_gamma))
 65 |         return t
 66 | 
 67 |     def forward_clean(self, h):
 68 |         z_pre = self.linear(h)
 69 |         # Store z_pre, z to be used in calculation of reconstruction cost
 70 |         self.buffer_z_pre = z_pre.detach().clone()
 71 |         z = self.bn_normalize_clean(z_pre)
 72 |         self.buffer_z = z.detach().clone()
 73 |         z_gb = self.bn_gamma_beta(z)
 74 |         h = self.activation(z_gb)
 75 |         return h
 76 | 
 77 |     def forward_noise(self, tilde_h):
 78 |         # z_pre will be used in the decoder cost
 79 |         z_pre = self.linear(tilde_h)
 80 |         z_pre_norm = self.bn_normalize(z_pre)
 81 |         # Add noise
 82 |         noise = np.random.normal(loc=0.0, scale=self.noise_level, size=z_pre_norm.size())
 83 |         if self.use_cuda:
 84 |             noise = Variable(torch.cuda.FloatTensor(noise))
 85 |         else:
 86 |             noise = Variable(torch.FloatTensor(noise))
 87 |         # tilde_z will be used by decoder for reconstruction
 88 |         tilde_z = z_pre_norm + noise
 89 |         # store tilde_z in buffer
 90 |         self.buffer_tilde_z = tilde_z
 91 |         z = self.bn_gamma_beta(tilde_z)
 92 |         h = self.activation(z)
 93 |         return h
 94 | 
 95 | 
 96 | class StackedEncoders(torch.nn.Module):
 97 |     def __init__(self, d_in, d_encoders, activation_types,
 98 |                  train_batch_norms, noise_std, use_cuda):
 99 |         super(StackedEncoders, self).__init__()
100 |         self.buffer_tilde_z_bottom = None
101 |         self.encoders_ref = []
102 |         self.encoders = torch.nn.Sequential()
103 |         self.noise_level = noise_std
104 |         self.use_cuda = use_cuda
105 |         n_encoders = len(d_encoders)
106 |         for i in range(n_encoders):
107 |             if i == 0:
108 |                 d_input = d_in
109 |             else:
110 |                 d_input = d_encoders[i - 1]
111 |             d_output = d_encoders[i]
112 |             activation = activation_types[i]
113 |             train_batch_norm = train_batch_norms[i]
114 |             encoder_ref = "encoder_" + str(i)
115 |             encoder = Encoder(d_input, d_output, activation, train_batch_norm, noise_std, use_cuda)
116 |             self.encoders_ref.append(encoder_ref)
117 |             self.encoders.add_module(encoder_ref, encoder)
118 | 
119 |     def forward_clean(self, x):
120 |         h = x
121 |         for e_ref in self.encoders_ref:
122 |             encoder = getattr(self.encoders, e_ref)
123 |             h = encoder.forward_clean(h)
124 |         return h
125 | 
126 |     def forward_noise(self, x):
127 |         noise = np.random.normal(loc=0.0, scale=self.noise_level, size=x.size())
128 |         if self.use_cuda:
129 |             noise = Variable(torch.cuda.FloatTensor(noise))
130 |         else:
131 |             noise = Variable(torch.FloatTensor(noise))
132 |         h = x + noise
133 |         self.buffer_tilde_z_bottom = h.clone()
134 |         # pass through encoders
135 |         for e_ref in self.encoders_ref:
136 |             encoder = getattr(self.encoders, e_ref)
137 |             h = encoder.forward_noise(h)
138 |         return h
139 | 
140 |     def get_encoders_tilde_z(self, reverse=True):
141 |         tilde_z_layers = []
142 |         for e_ref in self.encoders_ref:
143 |             encoder = getattr(self.encoders, e_ref)
144 |             tilde_z = encoder.buffer_tilde_z.clone()
145 |             tilde_z_layers.append(tilde_z)
146 |         if reverse:
147 |             tilde_z_layers.reverse()
148 |         return tilde_z_layers
149 | 
150 |     def get_encoders_z_pre(self, reverse=True):
151 |         z_pre_layers = []
152 |         for e_ref in self.encoders_ref:
153 |             encoder = getattr(self.encoders, e_ref)
154 |             z_pre = encoder.buffer_z_pre.clone()
155 |             z_pre_layers.append(z_pre)
156 |         if reverse:
157 |             z_pre_layers.reverse()
158 |         return z_pre_layers
159 | 
160 |     def get_encoders_z(self, reverse=True):
161 |         z_layers = []
162 |         for e_ref in self.encoders_ref:
163 |             encoder = getattr(self.encoders, e_ref)
164 |             z = encoder.buffer_z.clone()
165 |             z_layers.append(z)
166 |         if reverse:
167 |             z_layers.reverse()
168 |         return z_layers


--------------------------------------------------------------------------------
/ladder/ladder.py:
--------------------------------------------------------------------------------
  1 | from __future__ import print_function
  2 | 
  3 | import numpy as np
  4 | import argparse
  5 | import pickle
  6 | import os
  7 | 
  8 | import torch
  9 | from torch.autograd import Variable
 10 | from torch.optim import Adam
 11 | from torch.utils.data import TensorDataset, DataLoader
 12 | from encoder import StackedEncoders
 13 | from decoder import StackedDecoders
 14 | 
 15 | 
 16 | class Ladder(torch.nn.Module):
 17 |     def __init__(self, encoder_sizes, decoder_sizes, encoder_activations,
 18 |                  encoder_train_bn_scaling, noise_std, use_cuda):
 19 |         super(Ladder, self).__init__()
 20 |         self.use_cuda = use_cuda
 21 |         decoder_in = encoder_sizes[-1]
 22 |         encoder_in = decoder_sizes[-1]
 23 |         self.se = StackedEncoders(encoder_in, encoder_sizes, encoder_activations,
 24 |                                   encoder_train_bn_scaling, noise_std, use_cuda)
 25 |         self.de = StackedDecoders(decoder_in, decoder_sizes, encoder_in, use_cuda)
 26 |         self.bn_image = torch.nn.BatchNorm1d(encoder_in, affine=False)
 27 | 
 28 |     def forward_encoders_clean(self, data):
 29 |         return self.se.forward_clean(data)
 30 | 
 31 |     def forward_encoders_noise(self, data):
 32 |         return self.se.forward_noise(data)
 33 | 
 34 |     def forward_decoders(self, tilde_z_layers, encoder_output, tilde_z_bottom):
 35 |         return self.de.forward(tilde_z_layers, encoder_output, tilde_z_bottom)
 36 | 
 37 |     def get_encoders_tilde_z(self, reverse=True):
 38 |         return self.se.get_encoders_tilde_z(reverse)
 39 | 
 40 |     def get_encoders_z_pre(self, reverse=True):
 41 |         return self.se.get_encoders_z_pre(reverse)
 42 | 
 43 |     def get_encoder_tilde_z_bottom(self):
 44 |         return self.se.buffer_tilde_z_bottom.clone()
 45 | 
 46 |     def get_encoders_z(self, reverse=True):
 47 |         return self.se.get_encoders_z(reverse)
 48 | 
 49 |     def decoder_bn_hat_z_layers(self, hat_z_layers, z_pre_layers):
 50 |         return self.de.bn_hat_z_layers(hat_z_layers, z_pre_layers)
 51 | 
 52 | 
 53 | def evaluate_performance(ladder, valid_loader, e, agg_cost_scaled, agg_supervised_cost_scaled,
 54 |                          agg_unsupervised_cost_scaled, args):
 55 |     correct = 0.
 56 |     total = 0.
 57 |     for batch_idx, (data, target) in enumerate(valid_loader):
 58 |         if args.cuda:
 59 |             data = data.cuda()
 60 |         data, target = Variable(data), Variable(target)
 61 |         output = ladder.forward_encoders_clean(data)
 62 |         # TODO: Do away with the below hack for GPU tensors.
 63 |         if args.cuda:
 64 |             output = output.cpu()
 65 |             target = target.cpu()
 66 |         output = output.data.numpy()
 67 |         preds = np.argmax(output, axis=1)
 68 |         target = target.data.numpy()
 69 |         correct += np.sum(target == preds)
 70 |         total += target.shape[0]
 71 | 
 72 |     print("Epoch:", e + 1, "\t",
 73 |           "Total Cost:", "{:.4f}".format(agg_cost_scaled), "\t",
 74 |           "Supervised Cost:", "{:.4f}".format(agg_supervised_cost_scaled), "\t",
 75 |           "Unsupervised Cost:", "{:.4f}".format(agg_unsupervised_cost_scaled), "\t",
 76 |           "Validation Accuracy:", correct / total)
 77 | 
 78 | 
 79 | def main():
 80 |     # command line arguments
 81 |     parser = argparse.ArgumentParser(description="Parser for Ladder network")
 82 |     parser.add_argument("--batch", type=int, default=100)
 83 |     parser.add_argument("--epochs", type=int, default=10)
 84 |     parser.add_argument("--noise_std", type=float, default=0.2)
 85 |     parser.add_argument("--data_dir", type=str, default="data")
 86 |     parser.add_argument("--seed", type=int, default=42)
 87 |     parser.add_argument("--u_costs", type=str, default="0.1, 0.1, 0.1, 0.1, 0.1, 10., 1000.")
 88 |     parser.add_argument("--cuda", type=bool, default=False)
 89 |     parser.add_argument("--decay_epoch", type=int, default=15)
 90 |     args = parser.parse_args()
 91 | 
 92 |     batch_size = args.batch
 93 |     epochs = args.epochs
 94 |     noise_std = args.noise_std
 95 |     seed = args.seed
 96 |     decay_epoch = args.decay_epoch
 97 |     if args.cuda and not torch.cuda.is_available():
 98 |         print("WARNING: torch.cuda not available, using CPU.\n")
 99 |         args.cuda = False
100 | 
101 |     print("=====================")
102 |     print("BATCH SIZE:", batch_size)
103 |     print("EPOCHS:", epochs)
104 |     print("RANDOM SEED:", args.seed)
105 |     print("NOISE STD:", noise_std)
106 |     print("LR DECAY EPOCH:", decay_epoch)
107 |     print("CUDA:", args.cuda)
108 |     print("=====================\n")
109 | 
110 |     np.random.seed(seed)
111 |     torch.manual_seed(seed)
112 |     if args.cuda:
113 |         torch.cuda.manual_seed(seed)
114 | 
115 |     kwargs = {'num_workers': 1, 'pin_memory': True} if args.cuda else {}
116 | 
117 |     train_labelled_images_filename = os.path.join(args.data_dir, "train_labelled_images.p")
118 |     train_labelled_labels_filename = os.path.join(args.data_dir, "train_labelled_labels.p")
119 |     train_unlabelled_images_filename = os.path.join(args.data_dir, "train_unlabelled_images.p")
120 |     train_unlabelled_labels_filename = os.path.join(args.data_dir, "train_unlabelled_labels.p")
121 |     validation_images_filename = os.path.join(args.data_dir, "validation_images.p")
122 |     validation_labels_filename = os.path.join(args.data_dir, "validation_labels.p")
123 | 
124 |     print("Loading Data")
125 |     with open(train_labelled_images_filename) as f:
126 |         train_labelled_images = pickle.load(f)
127 |     train_labelled_images = train_labelled_images.reshape(train_labelled_images.shape[0], 784)
128 |     with open(train_labelled_labels_filename) as f:
129 |         train_labelled_labels = pickle.load(f).astype(int)
130 |     with open(train_unlabelled_images_filename) as f:
131 |         train_unlabelled_images = pickle.load(f)
132 |     train_unlabelled_images = train_unlabelled_images.reshape(train_unlabelled_images.shape[0], 784)
133 |     with open(train_unlabelled_labels_filename) as f:
134 |         train_unlabelled_labels = pickle.load(f).astype(int)
135 |     with open(validation_images_filename) as f:
136 |         validation_images = pickle.load(f)
137 |     validation_images = validation_images.reshape(validation_images.shape[0], 784)
138 |     with open(validation_labels_filename) as f:
139 |         validation_labels = pickle.load(f).astype(int)
140 | 
141 |     # Create DataLoaders
142 |     unlabelled_dataset = TensorDataset(torch.FloatTensor(train_unlabelled_images), torch.LongTensor(train_unlabelled_labels))
143 |     unlabelled_loader = DataLoader(unlabelled_dataset, batch_size=batch_size, shuffle=True, **kwargs)
144 |     validation_dataset = TensorDataset(torch.FloatTensor(validation_images), torch.LongTensor(validation_labels))
145 |     validation_loader = DataLoader(validation_dataset, batch_size=batch_size, shuffle=True, **kwargs)
146 | 
147 |     # Configure the Ladder
148 |     starter_lr = 0.02
149 |     encoder_sizes = [1000, 500, 250, 250, 250, 10]
150 |     decoder_sizes = [250, 250, 250, 500, 1000, 784]
151 |     unsupervised_costs_lambda = [float(x) for x in args.u_costs.split(",")]
152 |     encoder_activations = ["relu", "relu", "relu", "relu", "relu", "softmax"]
153 |     encoder_train_bn_scaling = [False, False, False, False, False, True]
154 |     ladder = Ladder(encoder_sizes, decoder_sizes, encoder_activations,
155 |                     encoder_train_bn_scaling, noise_std, args.cuda)
156 |     optimizer = Adam(ladder.parameters(), lr=starter_lr)
157 |     loss_supervised = torch.nn.CrossEntropyLoss()
158 |     loss_unsupervised = torch.nn.MSELoss()
159 | 
160 |     if args.cuda:
161 |         ladder.cuda()
162 | 
163 |     assert len(unsupervised_costs_lambda) == len(decoder_sizes) + 1
164 |     assert len(encoder_sizes) == len(decoder_sizes)
165 | 
166 |     print("")
167 |     print("========NETWORK=======")
168 |     print(ladder)
169 |     print("======================")
170 | 
171 |     print("")
172 |     print("==UNSUPERVISED-COSTS==")
173 |     print(unsupervised_costs_lambda)
174 | 
175 |     print("")
176 |     print("=====================")
177 |     print("TRAINING\n")
178 | 
179 |     # TODO: Add annealing of learning rate after 100 epochs
180 | 
181 |     for e in range(epochs):
182 |         agg_cost = 0.
183 |         agg_supervised_cost = 0.
184 |         agg_unsupervised_cost = 0.
185 |         num_batches = 0
186 |         ladder.train()
187 |         # TODO: Add volatile for the input parameters in training and validation
188 |         ind_labelled = 0
189 |         ind_limit = np.ceil(float(train_labelled_images.shape[0]) / batch_size)
190 | 
191 |         if e > args.decay_epoch:
192 |             ratio = float(epochs - e) / (epochs - decay_epoch)
193 |             current_lr = starter_lr * ratio
194 |             optimizer = Adam(ladder.parameters(), lr=current_lr)
195 | 
196 | 
197 |         for batch_idx, (unlabelled_images, unlabelled_labels) in enumerate(unlabelled_loader):
198 |             if ind_labelled == ind_limit:
199 |                 randomize = np.arange(train_labelled_images.shape[0])
200 |                 np.random.shuffle(randomize)
201 |                 train_labelled_images = train_labelled_images[randomize]
202 |                 train_labelled_labels = train_labelled_labels[randomize]
203 |                 ind_labelled = 0
204 | 
205 |             # TODO: Verify whether labelled examples are used for calculating unsupervised loss.
206 | 
207 |             labelled_start = batch_size * ind_labelled
208 |             labelled_end = batch_size * (ind_labelled + 1)
209 |             ind_labelled += 1
210 |             batch_train_labelled_images = torch.FloatTensor(train_labelled_images[labelled_start:labelled_end])
211 |             batch_train_labelled_labels = torch.LongTensor(train_labelled_labels[labelled_start:labelled_end])
212 | 
213 |             if args.cuda:
214 |                 batch_train_labelled_images = batch_train_labelled_images.cuda()
215 |                 batch_train_labelled_labels = batch_train_labelled_labels.cuda()
216 |                 unlabelled_images = unlabelled_images.cuda()
217 | 
218 |             labelled_data = Variable(batch_train_labelled_images, requires_grad=False)
219 |             labelled_target = Variable(batch_train_labelled_labels, requires_grad=False)
220 |             unlabelled_data = Variable(unlabelled_images)
221 | 
222 |             optimizer.zero_grad()
223 | 
224 |             # do a noisy pass for labelled data
225 |             output_noise_labelled = ladder.forward_encoders_noise(labelled_data)
226 | 
227 |             # do a noisy pass for unlabelled_data
228 |             output_noise_unlabelled = ladder.forward_encoders_noise(unlabelled_data)
229 |             tilde_z_layers_unlabelled = ladder.get_encoders_tilde_z(reverse=True)
230 | 
231 |             # do a clean pass for unlabelled data
232 |             output_clean_unlabelled = ladder.forward_encoders_clean(unlabelled_data)
233 |             z_pre_layers_unlabelled = ladder.get_encoders_z_pre(reverse=True)
234 |             z_layers_unlabelled = ladder.get_encoders_z(reverse=True)
235 | 
236 |             tilde_z_bottom_unlabelled = ladder.get_encoder_tilde_z_bottom()
237 | 
238 |             # pass through decoders
239 |             hat_z_layers_unlabelled = ladder.forward_decoders(tilde_z_layers_unlabelled,
240 |                                                               output_noise_unlabelled,
241 |                                                               tilde_z_bottom_unlabelled)
242 | 
243 |             z_pre_layers_unlabelled.append(unlabelled_data)
244 |             z_layers_unlabelled.append(unlabelled_data)
245 | 
246 |             # TODO: Verify if you have to batch-normalize the bottom-most layer also
247 |             # batch normalize using mean, var of z_pre
248 |             bn_hat_z_layers_unlabelled = ladder.decoder_bn_hat_z_layers(hat_z_layers_unlabelled, z_pre_layers_unlabelled)
249 | 
250 |             # calculate costs
251 |             cost_supervised = loss_supervised.forward(output_noise_labelled, labelled_target)
252 |             cost_unsupervised = 0.
253 |             assert len(z_layers_unlabelled) == len(bn_hat_z_layers_unlabelled)
254 |             for cost_lambda, z, bn_hat_z in zip(unsupervised_costs_lambda, z_layers_unlabelled, bn_hat_z_layers_unlabelled):
255 |                 c = cost_lambda * loss_unsupervised.forward(bn_hat_z, z)
256 |                 cost_unsupervised += c
257 | 
258 |             # backprop
259 |             cost = cost_supervised + cost_unsupervised
260 |             cost.backward()
261 |             optimizer.step()
262 | 
263 |             agg_cost += cost.data[0]
264 |             agg_supervised_cost += cost_supervised.data[0]
265 |             agg_unsupervised_cost += cost_unsupervised.data[0]
266 |             num_batches += 1
267 | 
268 |             if ind_labelled == ind_limit:
269 |                 # Evaluation
270 |                 ladder.eval()
271 |                 evaluate_performance(ladder, validation_loader, e,
272 |                                      agg_cost / num_batches,
273 |                                      agg_supervised_cost / num_batches,
274 |                                      agg_unsupervised_cost / num_batches,
275 |                                      args)
276 |                 ladder.train()
277 |     print("=====================\n")
278 |     print("Done :)")
279 | 
280 | 
281 | if __name__ == "__main__":
282 |     main()
283 | 


--------------------------------------------------------------------------------
/logs/ladder_supervised_only.log:
--------------------------------------------------------------------------------
  1 | =====================
  2 | BATCH SIZE: 100
  3 | EPOCHS: 20
  4 | NOISE STD: 0.2
  5 | CUDA: True
  6 | =====================
  7 | 
  8 | Loading Data
  9 | 
 10 | ========NETWORK=======
 11 | Ladder (
 12 |   (se): StackedEncoders (
 13 |     (encoders): Sequential (
 14 |       (encoder_0): Encoder (
 15 |         (linear): Linear (784 -> 1000)
 16 |         (bn_normalize_clean): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 17 |         (bn_normalize): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 18 |         (activation): ReLU ()
 19 |       )
 20 |       (encoder_1): Encoder (
 21 |         (linear): Linear (1000 -> 500)
 22 |         (bn_normalize_clean): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 23 |         (bn_normalize): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 24 |         (activation): ReLU ()
 25 |       )
 26 |       (encoder_2): Encoder (
 27 |         (linear): Linear (500 -> 250)
 28 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 29 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 30 |         (activation): ReLU ()
 31 |       )
 32 |       (encoder_3): Encoder (
 33 |         (linear): Linear (250 -> 250)
 34 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 35 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 36 |         (activation): ReLU ()
 37 |       )
 38 |       (encoder_4): Encoder (
 39 |         (linear): Linear (250 -> 250)
 40 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 41 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 42 |         (activation): ReLU ()
 43 |       )
 44 |       (encoder_5): Encoder (
 45 |         (linear): Linear (250 -> 10)
 46 |         (bn_normalize_clean): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 47 |         (bn_normalize): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 48 |         (activation): Softmax ()
 49 |       )
 50 |     )
 51 |   )
 52 |   (de): StackedDecoders (
 53 |     (bn_u_top): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 54 |     (decoders): Sequential (
 55 |       (decoder_0): Decoder (
 56 |         (V): Linear (10 -> 250)
 57 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 58 |       )
 59 |       (decoder_1): Decoder (
 60 |         (V): Linear (250 -> 250)
 61 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 62 |       )
 63 |       (decoder_2): Decoder (
 64 |         (V): Linear (250 -> 250)
 65 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 66 |       )
 67 |       (decoder_3): Decoder (
 68 |         (V): Linear (250 -> 500)
 69 |         (bn_normalize): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 70 |       )
 71 |       (decoder_4): Decoder (
 72 |         (V): Linear (500 -> 1000)
 73 |         (bn_normalize): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 74 |       )
 75 |       (decoder_5): Decoder (
 76 |         (V): Linear (1000 -> 784)
 77 |         (bn_normalize): BatchNorm1d(784, eps=1e-05, momentum=0.1, affine=False)
 78 |       )
 79 |     )
 80 |     (bottom_decoder): Decoder (
 81 |     )
 82 |   )
 83 |   (bn_image): BatchNorm1d(784, eps=1e-05, momentum=0.1, affine=False)
 84 | )
 85 | ======================
 86 | 
 87 | ==UNSUPERVISED-COSTS==
 88 | [0.1, 0.1, 0.1, 0.1, 0.1, 10.0, 1000.0]
 89 | 
 90 | =====================
 91 | TRAINING
 92 | 
 93 | Epoch: 1 	 Total Cost: 1.9142 	 Supervised Cost: 1.9142 	 Unsupervised Cost: 618.5100 	 Validation Accuracy: 0.9019
 94 | Epoch: 1 	 Total Cost: 1.8333 	 Supervised Cost: 1.8333 	 Unsupervised Cost: 620.7994 	 Validation Accuracy: 0.9233
 95 | Epoch: 1 	 Total Cost: 1.7922 	 Supervised Cost: 1.7922 	 Unsupervised Cost: 619.6075 	 Validation Accuracy: 0.9318
 96 | Epoch: 1 	 Total Cost: 1.7620 	 Supervised Cost: 1.7620 	 Unsupervised Cost: 616.9266 	 Validation Accuracy: 0.9326
 97 | Epoch: 1 	 Total Cost: 1.7400 	 Supervised Cost: 1.7400 	 Unsupervised Cost: 616.5165 	 Validation Accuracy: 0.9369
 98 | Epoch: 1 	 Total Cost: 1.7210 	 Supervised Cost: 1.7210 	 Unsupervised Cost: 617.6431 	 Validation Accuracy: 0.9383
 99 | Epoch: 1 	 Total Cost: 1.7051 	 Supervised Cost: 1.7051 	 Unsupervised Cost: 617.2827 	 Validation Accuracy: 0.9394
100 | Epoch: 1 	 Total Cost: 1.6925 	 Supervised Cost: 1.6925 	 Unsupervised Cost: 617.5151 	 Validation Accuracy: 0.9402
101 | Epoch: 1 	 Total Cost: 1.6812 	 Supervised Cost: 1.6812 	 Unsupervised Cost: 617.9331 	 Validation Accuracy: 0.936
102 | Epoch: 1 	 Total Cost: 1.6709 	 Supervised Cost: 1.6709 	 Unsupervised Cost: 617.7159 	 Validation Accuracy: 0.9456
103 | Epoch: 1 	 Total Cost: 1.6614 	 Supervised Cost: 1.6614 	 Unsupervised Cost: 617.6630 	 Validation Accuracy: 0.9418
104 | Epoch: 1 	 Total Cost: 1.6532 	 Supervised Cost: 1.6532 	 Unsupervised Cost: 617.6635 	 Validation Accuracy: 0.9363
105 | Epoch: 1 	 Total Cost: 1.6455 	 Supervised Cost: 1.6455 	 Unsupervised Cost: 618.0994 	 Validation Accuracy: 0.9398
106 | Epoch: 1 	 Total Cost: 1.6387 	 Supervised Cost: 1.6387 	 Unsupervised Cost: 618.5156 	 Validation Accuracy: 0.9406
107 | Epoch: 1 	 Total Cost: 1.6326 	 Supervised Cost: 1.6326 	 Unsupervised Cost: 617.9165 	 Validation Accuracy: 0.9412
108 | Epoch: 2 	 Total Cost: 1.5294 	 Supervised Cost: 1.5294 	 Unsupervised Cost: 620.4811 	 Validation Accuracy: 0.9457
109 | Epoch: 2 	 Total Cost: 1.5279 	 Supervised Cost: 1.5279 	 Unsupervised Cost: 622.4965 	 Validation Accuracy: 0.9452
110 | Epoch: 2 	 Total Cost: 1.5297 	 Supervised Cost: 1.5297 	 Unsupervised Cost: 620.4405 	 Validation Accuracy: 0.9445
111 | Epoch: 2 	 Total Cost: 1.5279 	 Supervised Cost: 1.5279 	 Unsupervised Cost: 620.1837 	 Validation Accuracy: 0.9457
112 | Epoch: 2 	 Total Cost: 1.5270 	 Supervised Cost: 1.5270 	 Unsupervised Cost: 620.7291 	 Validation Accuracy: 0.9473
113 | Epoch: 2 	 Total Cost: 1.5255 	 Supervised Cost: 1.5255 	 Unsupervised Cost: 619.5989 	 Validation Accuracy: 0.9454
114 | Epoch: 2 	 Total Cost: 1.5241 	 Supervised Cost: 1.5241 	 Unsupervised Cost: 619.4321 	 Validation Accuracy: 0.9478
115 | Epoch: 2 	 Total Cost: 1.5226 	 Supervised Cost: 1.5226 	 Unsupervised Cost: 619.4156 	 Validation Accuracy: 0.9431
116 | Epoch: 2 	 Total Cost: 1.5211 	 Supervised Cost: 1.5211 	 Unsupervised Cost: 618.8360 	 Validation Accuracy: 0.9457
117 | Epoch: 2 	 Total Cost: 1.5199 	 Supervised Cost: 1.5199 	 Unsupervised Cost: 619.1564 	 Validation Accuracy: 0.9428
118 | Epoch: 2 	 Total Cost: 1.5185 	 Supervised Cost: 1.5185 	 Unsupervised Cost: 619.5857 	 Validation Accuracy: 0.9452
119 | Epoch: 2 	 Total Cost: 1.5177 	 Supervised Cost: 1.5177 	 Unsupervised Cost: 618.7934 	 Validation Accuracy: 0.9465
120 | Epoch: 2 	 Total Cost: 1.5169 	 Supervised Cost: 1.5169 	 Unsupervised Cost: 618.6807 	 Validation Accuracy: 0.9436
121 | Epoch: 2 	 Total Cost: 1.5158 	 Supervised Cost: 1.5158 	 Unsupervised Cost: 618.3623 	 Validation Accuracy: 0.9448
122 | Epoch: 2 	 Total Cost: 1.5147 	 Supervised Cost: 1.5147 	 Unsupervised Cost: 618.2650 	 Validation Accuracy: 0.9424
123 | Epoch: 3 	 Total Cost: 1.4962 	 Supervised Cost: 1.4962 	 Unsupervised Cost: 618.9858 	 Validation Accuracy: 0.9466
124 | Epoch: 3 	 Total Cost: 1.4974 	 Supervised Cost: 1.4974 	 Unsupervised Cost: 615.5651 	 Validation Accuracy: 0.9505
125 | Epoch: 3 	 Total Cost: 1.4968 	 Supervised Cost: 1.4968 	 Unsupervised Cost: 612.7291 	 Validation Accuracy: 0.9503
126 | Epoch: 3 	 Total Cost: 1.4968 	 Supervised Cost: 1.4968 	 Unsupervised Cost: 612.6727 	 Validation Accuracy: 0.9468
127 | Epoch: 3 	 Total Cost: 1.4965 	 Supervised Cost: 1.4965 	 Unsupervised Cost: 613.7999 	 Validation Accuracy: 0.9486
128 | Epoch: 3 	 Total Cost: 1.4963 	 Supervised Cost: 1.4963 	 Unsupervised Cost: 614.9219 	 Validation Accuracy: 0.9498
129 | Epoch: 3 	 Total Cost: 1.4964 	 Supervised Cost: 1.4964 	 Unsupervised Cost: 615.0452 	 Validation Accuracy: 0.9445
130 | Epoch: 3 	 Total Cost: 1.4963 	 Supervised Cost: 1.4963 	 Unsupervised Cost: 616.3901 	 Validation Accuracy: 0.9467
131 | Epoch: 3 	 Total Cost: 1.4959 	 Supervised Cost: 1.4959 	 Unsupervised Cost: 617.9411 	 Validation Accuracy: 0.9532
132 | Epoch: 3 	 Total Cost: 1.4954 	 Supervised Cost: 1.4954 	 Unsupervised Cost: 618.1083 	 Validation Accuracy: 0.9497
133 | Epoch: 3 	 Total Cost: 1.4953 	 Supervised Cost: 1.4953 	 Unsupervised Cost: 618.7433 	 Validation Accuracy: 0.9461
134 | Epoch: 3 	 Total Cost: 1.4946 	 Supervised Cost: 1.4946 	 Unsupervised Cost: 618.7519 	 Validation Accuracy: 0.9515
135 | Epoch: 3 	 Total Cost: 1.4938 	 Supervised Cost: 1.4938 	 Unsupervised Cost: 618.9872 	 Validation Accuracy: 0.95
136 | Epoch: 3 	 Total Cost: 1.4931 	 Supervised Cost: 1.4931 	 Unsupervised Cost: 619.0372 	 Validation Accuracy: 0.9512
137 | Epoch: 3 	 Total Cost: 1.4926 	 Supervised Cost: 1.4926 	 Unsupervised Cost: 618.1797 	 Validation Accuracy: 0.9495
138 | Epoch: 4 	 Total Cost: 1.4844 	 Supervised Cost: 1.4844 	 Unsupervised Cost: 612.3150 	 Validation Accuracy: 0.9506
139 | Epoch: 4 	 Total Cost: 1.4849 	 Supervised Cost: 1.4849 	 Unsupervised Cost: 617.2573 	 Validation Accuracy: 0.9521
140 | Epoch: 4 	 Total Cost: 1.4848 	 Supervised Cost: 1.4848 	 Unsupervised Cost: 614.0663 	 Validation Accuracy: 0.9488
141 | Epoch: 4 	 Total Cost: 1.4852 	 Supervised Cost: 1.4852 	 Unsupervised Cost: 616.2141 	 Validation Accuracy: 0.9463
142 | Epoch: 4 	 Total Cost: 1.4855 	 Supervised Cost: 1.4855 	 Unsupervised Cost: 615.9186 	 Validation Accuracy: 0.9417
143 | Epoch: 4 	 Total Cost: 1.4852 	 Supervised Cost: 1.4852 	 Unsupervised Cost: 617.0228 	 Validation Accuracy: 0.9469
144 | Epoch: 4 	 Total Cost: 1.4850 	 Supervised Cost: 1.4850 	 Unsupervised Cost: 615.7368 	 Validation Accuracy: 0.948
145 | Epoch: 4 	 Total Cost: 1.4848 	 Supervised Cost: 1.4848 	 Unsupervised Cost: 616.7120 	 Validation Accuracy: 0.9534
146 | Epoch: 4 	 Total Cost: 1.4844 	 Supervised Cost: 1.4844 	 Unsupervised Cost: 617.1201 	 Validation Accuracy: 0.9519
147 | Epoch: 4 	 Total Cost: 1.4840 	 Supervised Cost: 1.4840 	 Unsupervised Cost: 617.6033 	 Validation Accuracy: 0.9448
148 | Epoch: 4 	 Total Cost: 1.4839 	 Supervised Cost: 1.4839 	 Unsupervised Cost: 617.7571 	 Validation Accuracy: 0.9502
149 | Epoch: 4 	 Total Cost: 1.4838 	 Supervised Cost: 1.4838 	 Unsupervised Cost: 618.0640 	 Validation Accuracy: 0.95
150 | Epoch: 4 	 Total Cost: 1.4836 	 Supervised Cost: 1.4836 	 Unsupervised Cost: 618.4387 	 Validation Accuracy: 0.9524
151 | Epoch: 4 	 Total Cost: 1.4836 	 Supervised Cost: 1.4836 	 Unsupervised Cost: 618.5917 	 Validation Accuracy: 0.9504
152 | Epoch: 4 	 Total Cost: 1.4833 	 Supervised Cost: 1.4833 	 Unsupervised Cost: 618.7482 	 Validation Accuracy: 0.9516
153 | Epoch: 5 	 Total Cost: 1.4777 	 Supervised Cost: 1.4777 	 Unsupervised Cost: 615.8302 	 Validation Accuracy: 0.9519
154 | Epoch: 5 	 Total Cost: 1.4791 	 Supervised Cost: 1.4791 	 Unsupervised Cost: 617.1536 	 Validation Accuracy: 0.9513
155 | Epoch: 5 	 Total Cost: 1.4788 	 Supervised Cost: 1.4788 	 Unsupervised Cost: 617.2538 	 Validation Accuracy: 0.9493
156 | Epoch: 5 	 Total Cost: 1.4790 	 Supervised Cost: 1.4790 	 Unsupervised Cost: 616.9599 	 Validation Accuracy: 0.9561
157 | Epoch: 5 	 Total Cost: 1.4792 	 Supervised Cost: 1.4792 	 Unsupervised Cost: 617.3005 	 Validation Accuracy: 0.9531
158 | Epoch: 5 	 Total Cost: 1.4792 	 Supervised Cost: 1.4792 	 Unsupervised Cost: 618.7010 	 Validation Accuracy: 0.9515
159 | Epoch: 5 	 Total Cost: 1.4794 	 Supervised Cost: 1.4794 	 Unsupervised Cost: 619.6169 	 Validation Accuracy: 0.9457
160 | Epoch: 5 	 Total Cost: 1.4796 	 Supervised Cost: 1.4796 	 Unsupervised Cost: 618.6788 	 Validation Accuracy: 0.9513
161 | Epoch: 5 	 Total Cost: 1.4797 	 Supervised Cost: 1.4797 	 Unsupervised Cost: 618.7621 	 Validation Accuracy: 0.9519
162 | Epoch: 5 	 Total Cost: 1.4799 	 Supervised Cost: 1.4799 	 Unsupervised Cost: 618.9513 	 Validation Accuracy: 0.9498
163 | Epoch: 5 	 Total Cost: 1.4795 	 Supervised Cost: 1.4795 	 Unsupervised Cost: 618.6521 	 Validation Accuracy: 0.951
164 | Epoch: 5 	 Total Cost: 1.4796 	 Supervised Cost: 1.4796 	 Unsupervised Cost: 618.9246 	 Validation Accuracy: 0.9509
165 | Epoch: 5 	 Total Cost: 1.4795 	 Supervised Cost: 1.4795 	 Unsupervised Cost: 618.8821 	 Validation Accuracy: 0.9525
166 | Epoch: 5 	 Total Cost: 1.4794 	 Supervised Cost: 1.4794 	 Unsupervised Cost: 618.6477 	 Validation Accuracy: 0.9509
167 | Epoch: 5 	 Total Cost: 1.4794 	 Supervised Cost: 1.4794 	 Unsupervised Cost: 619.1569 	 Validation Accuracy: 0.9458
168 | Epoch: 6 	 Total Cost: 1.4757 	 Supervised Cost: 1.4757 	 Unsupervised Cost: 622.8494 	 Validation Accuracy: 0.9526
169 | Epoch: 6 	 Total Cost: 1.4756 	 Supervised Cost: 1.4756 	 Unsupervised Cost: 622.5154 	 Validation Accuracy: 0.9489
170 | Epoch: 6 	 Total Cost: 1.4754 	 Supervised Cost: 1.4754 	 Unsupervised Cost: 619.3404 	 Validation Accuracy: 0.9536
171 | Epoch: 6 	 Total Cost: 1.4756 	 Supervised Cost: 1.4756 	 Unsupervised Cost: 618.8445 	 Validation Accuracy: 0.9514
172 | Epoch: 6 	 Total Cost: 1.4755 	 Supervised Cost: 1.4755 	 Unsupervised Cost: 618.8122 	 Validation Accuracy: 0.9488
173 | Epoch: 6 	 Total Cost: 1.4759 	 Supervised Cost: 1.4759 	 Unsupervised Cost: 618.1279 	 Validation Accuracy: 0.9457
174 | Epoch: 6 	 Total Cost: 1.4756 	 Supervised Cost: 1.4756 	 Unsupervised Cost: 618.9873 	 Validation Accuracy: 0.9453
175 | Epoch: 6 	 Total Cost: 1.4762 	 Supervised Cost: 1.4762 	 Unsupervised Cost: 619.2980 	 Validation Accuracy: 0.9486
176 | Epoch: 6 	 Total Cost: 1.4761 	 Supervised Cost: 1.4761 	 Unsupervised Cost: 618.4123 	 Validation Accuracy: 0.9517
177 | Epoch: 6 	 Total Cost: 1.4758 	 Supervised Cost: 1.4758 	 Unsupervised Cost: 618.3842 	 Validation Accuracy: 0.9494
178 | Epoch: 6 	 Total Cost: 1.4759 	 Supervised Cost: 1.4759 	 Unsupervised Cost: 618.8385 	 Validation Accuracy: 0.9472
179 | Epoch: 6 	 Total Cost: 1.4759 	 Supervised Cost: 1.4759 	 Unsupervised Cost: 618.4535 	 Validation Accuracy: 0.9501
180 | Epoch: 6 	 Total Cost: 1.4757 	 Supervised Cost: 1.4757 	 Unsupervised Cost: 618.2563 	 Validation Accuracy: 0.9487
181 | Epoch: 6 	 Total Cost: 1.4757 	 Supervised Cost: 1.4757 	 Unsupervised Cost: 618.4445 	 Validation Accuracy: 0.9499
182 | Epoch: 6 	 Total Cost: 1.4756 	 Supervised Cost: 1.4756 	 Unsupervised Cost: 618.5836 	 Validation Accuracy: 0.9506
183 | Epoch: 7 	 Total Cost: 1.4727 	 Supervised Cost: 1.4727 	 Unsupervised Cost: 617.5818 	 Validation Accuracy: 0.9514
184 | Epoch: 7 	 Total Cost: 1.4717 	 Supervised Cost: 1.4717 	 Unsupervised Cost: 616.4717 	 Validation Accuracy: 0.9528
185 | Epoch: 7 	 Total Cost: 1.4720 	 Supervised Cost: 1.4720 	 Unsupervised Cost: 620.7560 	 Validation Accuracy: 0.9519
186 | Epoch: 7 	 Total Cost: 1.4719 	 Supervised Cost: 1.4719 	 Unsupervised Cost: 621.2713 	 Validation Accuracy: 0.9483
187 | Epoch: 7 	 Total Cost: 1.4721 	 Supervised Cost: 1.4721 	 Unsupervised Cost: 621.2478 	 Validation Accuracy: 0.9483
188 | Epoch: 7 	 Total Cost: 1.4723 	 Supervised Cost: 1.4723 	 Unsupervised Cost: 621.9237 	 Validation Accuracy: 0.9491
189 | Epoch: 7 	 Total Cost: 1.4723 	 Supervised Cost: 1.4723 	 Unsupervised Cost: 621.8816 	 Validation Accuracy: 0.9507
190 | Epoch: 7 	 Total Cost: 1.4725 	 Supervised Cost: 1.4725 	 Unsupervised Cost: 622.7776 	 Validation Accuracy: 0.9492
191 | Epoch: 7 	 Total Cost: 1.4726 	 Supervised Cost: 1.4726 	 Unsupervised Cost: 622.0819 	 Validation Accuracy: 0.9502
192 | Epoch: 7 	 Total Cost: 1.4726 	 Supervised Cost: 1.4726 	 Unsupervised Cost: 621.2672 	 Validation Accuracy: 0.9493
193 | Epoch: 7 	 Total Cost: 1.4725 	 Supervised Cost: 1.4725 	 Unsupervised Cost: 621.0383 	 Validation Accuracy: 0.9543
194 | Epoch: 7 	 Total Cost: 1.4723 	 Supervised Cost: 1.4723 	 Unsupervised Cost: 619.9118 	 Validation Accuracy: 0.9548
195 | Epoch: 7 	 Total Cost: 1.4722 	 Supervised Cost: 1.4722 	 Unsupervised Cost: 619.9572 	 Validation Accuracy: 0.9531
196 | Epoch: 7 	 Total Cost: 1.4723 	 Supervised Cost: 1.4723 	 Unsupervised Cost: 619.6496 	 Validation Accuracy: 0.9481
197 | Epoch: 7 	 Total Cost: 1.4725 	 Supervised Cost: 1.4725 	 Unsupervised Cost: 618.7641 	 Validation Accuracy: 0.9505
198 | Epoch: 8 	 Total Cost: 1.4705 	 Supervised Cost: 1.4705 	 Unsupervised Cost: 612.6473 	 Validation Accuracy: 0.9501
199 | Epoch: 8 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 616.5771 	 Validation Accuracy: 0.9512
200 | Epoch: 8 	 Total Cost: 1.4711 	 Supervised Cost: 1.4711 	 Unsupervised Cost: 615.5698 	 Validation Accuracy: 0.9534
201 | Epoch: 8 	 Total Cost: 1.4714 	 Supervised Cost: 1.4714 	 Unsupervised Cost: 617.1499 	 Validation Accuracy: 0.9538
202 | Epoch: 8 	 Total Cost: 1.4713 	 Supervised Cost: 1.4713 	 Unsupervised Cost: 617.1438 	 Validation Accuracy: 0.9534
203 | Epoch: 8 	 Total Cost: 1.4711 	 Supervised Cost: 1.4711 	 Unsupervised Cost: 618.2339 	 Validation Accuracy: 0.9525
204 | Epoch: 8 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 618.7961 	 Validation Accuracy: 0.951
205 | Epoch: 8 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 618.4404 	 Validation Accuracy: 0.9528
206 | Epoch: 8 	 Total Cost: 1.4705 	 Supervised Cost: 1.4705 	 Unsupervised Cost: 618.3040 	 Validation Accuracy: 0.9518
207 | Epoch: 8 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 617.7002 	 Validation Accuracy: 0.9532
208 | Epoch: 8 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 618.1112 	 Validation Accuracy: 0.9546
209 | Epoch: 8 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 618.7341 	 Validation Accuracy: 0.9537
210 | Epoch: 8 	 Total Cost: 1.4711 	 Supervised Cost: 1.4711 	 Unsupervised Cost: 618.7944 	 Validation Accuracy: 0.9546
211 | Epoch: 8 	 Total Cost: 1.4709 	 Supervised Cost: 1.4709 	 Unsupervised Cost: 618.5388 	 Validation Accuracy: 0.9535
212 | Epoch: 8 	 Total Cost: 1.4713 	 Supervised Cost: 1.4713 	 Unsupervised Cost: 618.6676 	 Validation Accuracy: 0.9515
213 | Epoch: 9 	 Total Cost: 1.4754 	 Supervised Cost: 1.4754 	 Unsupervised Cost: 615.2986 	 Validation Accuracy: 0.9535
214 | Epoch: 9 	 Total Cost: 1.4752 	 Supervised Cost: 1.4752 	 Unsupervised Cost: 613.1156 	 Validation Accuracy: 0.9484
215 | Epoch: 9 	 Total Cost: 1.4741 	 Supervised Cost: 1.4741 	 Unsupervised Cost: 615.2637 	 Validation Accuracy: 0.9525
216 | Epoch: 9 	 Total Cost: 1.4733 	 Supervised Cost: 1.4733 	 Unsupervised Cost: 617.5958 	 Validation Accuracy: 0.954
217 | Epoch: 9 	 Total Cost: 1.4728 	 Supervised Cost: 1.4728 	 Unsupervised Cost: 617.5449 	 Validation Accuracy: 0.953
218 | Epoch: 9 	 Total Cost: 1.4721 	 Supervised Cost: 1.4721 	 Unsupervised Cost: 617.4679 	 Validation Accuracy: 0.9526
219 | Epoch: 9 	 Total Cost: 1.4721 	 Supervised Cost: 1.4721 	 Unsupervised Cost: 618.1932 	 Validation Accuracy: 0.9493
220 | Epoch: 9 	 Total Cost: 1.4719 	 Supervised Cost: 1.4719 	 Unsupervised Cost: 617.2785 	 Validation Accuracy: 0.9513
221 | Epoch: 9 	 Total Cost: 1.4719 	 Supervised Cost: 1.4719 	 Unsupervised Cost: 617.4493 	 Validation Accuracy: 0.9483
222 | Epoch: 9 	 Total Cost: 1.4718 	 Supervised Cost: 1.4718 	 Unsupervised Cost: 617.7627 	 Validation Accuracy: 0.9512
223 | Epoch: 9 	 Total Cost: 1.4715 	 Supervised Cost: 1.4715 	 Unsupervised Cost: 617.7989 	 Validation Accuracy: 0.9524
224 | Epoch: 9 	 Total Cost: 1.4713 	 Supervised Cost: 1.4713 	 Unsupervised Cost: 618.0077 	 Validation Accuracy: 0.9523
225 | Epoch: 9 	 Total Cost: 1.4710 	 Supervised Cost: 1.4710 	 Unsupervised Cost: 617.4954 	 Validation Accuracy: 0.9523
226 | Epoch: 9 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 618.5108 	 Validation Accuracy: 0.9535
227 | Epoch: 9 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 618.3572 	 Validation Accuracy: 0.9533
228 | Epoch: 10 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 616.9483 	 Validation Accuracy: 0.9502
229 | Epoch: 10 	 Total Cost: 1.4711 	 Supervised Cost: 1.4711 	 Unsupervised Cost: 620.6355 	 Validation Accuracy: 0.9474
230 | Epoch: 10 	 Total Cost: 1.4706 	 Supervised Cost: 1.4706 	 Unsupervised Cost: 620.8645 	 Validation Accuracy: 0.9519
231 | Epoch: 10 	 Total Cost: 1.4706 	 Supervised Cost: 1.4706 	 Unsupervised Cost: 621.3131 	 Validation Accuracy: 0.9481
232 | Epoch: 10 	 Total Cost: 1.4713 	 Supervised Cost: 1.4713 	 Unsupervised Cost: 622.1932 	 Validation Accuracy: 0.9502
233 | Epoch: 10 	 Total Cost: 1.4711 	 Supervised Cost: 1.4711 	 Unsupervised Cost: 621.2913 	 Validation Accuracy: 0.9543
234 | Epoch: 10 	 Total Cost: 1.4712 	 Supervised Cost: 1.4712 	 Unsupervised Cost: 620.9110 	 Validation Accuracy: 0.9519
235 | Epoch: 10 	 Total Cost: 1.4709 	 Supervised Cost: 1.4709 	 Unsupervised Cost: 620.7592 	 Validation Accuracy: 0.952
236 | Epoch: 10 	 Total Cost: 1.4706 	 Supervised Cost: 1.4706 	 Unsupervised Cost: 620.9691 	 Validation Accuracy: 0.9547
237 | Epoch: 10 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 620.0175 	 Validation Accuracy: 0.9534
238 | Epoch: 10 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 620.0332 	 Validation Accuracy: 0.9515
239 | Epoch: 10 	 Total Cost: 1.4709 	 Supervised Cost: 1.4709 	 Unsupervised Cost: 619.2283 	 Validation Accuracy: 0.9491
240 | Epoch: 10 	 Total Cost: 1.4709 	 Supervised Cost: 1.4709 	 Unsupervised Cost: 618.9982 	 Validation Accuracy: 0.9513
241 | Epoch: 10 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 619.2131 	 Validation Accuracy: 0.9538
242 | Epoch: 10 	 Total Cost: 1.4709 	 Supervised Cost: 1.4709 	 Unsupervised Cost: 619.2032 	 Validation Accuracy: 0.9531
243 | Epoch: 11 	 Total Cost: 1.4705 	 Supervised Cost: 1.4705 	 Unsupervised Cost: 623.7873 	 Validation Accuracy: 0.9513
244 | Epoch: 11 	 Total Cost: 1.4705 	 Supervised Cost: 1.4705 	 Unsupervised Cost: 620.9264 	 Validation Accuracy: 0.9459
245 | Epoch: 11 	 Total Cost: 1.4724 	 Supervised Cost: 1.4724 	 Unsupervised Cost: 617.8487 	 Validation Accuracy: 0.9462
246 | Epoch: 11 	 Total Cost: 1.4732 	 Supervised Cost: 1.4732 	 Unsupervised Cost: 616.7703 	 Validation Accuracy: 0.9491
247 | Epoch: 11 	 Total Cost: 1.4722 	 Supervised Cost: 1.4722 	 Unsupervised Cost: 616.9033 	 Validation Accuracy: 0.953
248 | Epoch: 11 	 Total Cost: 1.4721 	 Supervised Cost: 1.4721 	 Unsupervised Cost: 618.4076 	 Validation Accuracy: 0.9513
249 | Epoch: 11 	 Total Cost: 1.4719 	 Supervised Cost: 1.4719 	 Unsupervised Cost: 617.3346 	 Validation Accuracy: 0.9534
250 | Epoch: 11 	 Total Cost: 1.4715 	 Supervised Cost: 1.4715 	 Unsupervised Cost: 617.5575 	 Validation Accuracy: 0.9537
251 | Epoch: 11 	 Total Cost: 1.4714 	 Supervised Cost: 1.4714 	 Unsupervised Cost: 617.7079 	 Validation Accuracy: 0.9522
252 | Epoch: 11 	 Total Cost: 1.4715 	 Supervised Cost: 1.4715 	 Unsupervised Cost: 618.6717 	 Validation Accuracy: 0.9486
253 | Epoch: 11 	 Total Cost: 1.4713 	 Supervised Cost: 1.4713 	 Unsupervised Cost: 619.4664 	 Validation Accuracy: 0.9532
254 | Epoch: 11 	 Total Cost: 1.4710 	 Supervised Cost: 1.4710 	 Unsupervised Cost: 619.2876 	 Validation Accuracy: 0.9495
255 | Epoch: 11 	 Total Cost: 1.4710 	 Supervised Cost: 1.4710 	 Unsupervised Cost: 619.4511 	 Validation Accuracy: 0.9536
256 | Epoch: 11 	 Total Cost: 1.4708 	 Supervised Cost: 1.4708 	 Unsupervised Cost: 619.0159 	 Validation Accuracy: 0.9546
257 | Epoch: 11 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 619.1477 	 Validation Accuracy: 0.9548
258 | Epoch: 12 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 617.0487 	 Validation Accuracy: 0.9517
259 | Epoch: 12 	 Total Cost: 1.4687 	 Supervised Cost: 1.4687 	 Unsupervised Cost: 616.0842 	 Validation Accuracy: 0.952
260 | Epoch: 12 	 Total Cost: 1.4683 	 Supervised Cost: 1.4683 	 Unsupervised Cost: 618.2027 	 Validation Accuracy: 0.95
261 | Epoch: 12 	 Total Cost: 1.4681 	 Supervised Cost: 1.4681 	 Unsupervised Cost: 618.9679 	 Validation Accuracy: 0.9532
262 | Epoch: 12 	 Total Cost: 1.4681 	 Supervised Cost: 1.4681 	 Unsupervised Cost: 617.7097 	 Validation Accuracy: 0.9485
263 | Epoch: 12 	 Total Cost: 1.4682 	 Supervised Cost: 1.4682 	 Unsupervised Cost: 619.8421 	 Validation Accuracy: 0.9513
264 | Epoch: 12 	 Total Cost: 1.4684 	 Supervised Cost: 1.4684 	 Unsupervised Cost: 620.0510 	 Validation Accuracy: 0.9507
265 | Epoch: 12 	 Total Cost: 1.4688 	 Supervised Cost: 1.4688 	 Unsupervised Cost: 619.9316 	 Validation Accuracy: 0.9522
266 | Epoch: 12 	 Total Cost: 1.4688 	 Supervised Cost: 1.4688 	 Unsupervised Cost: 620.4592 	 Validation Accuracy: 0.9522
267 | Epoch: 12 	 Total Cost: 1.4689 	 Supervised Cost: 1.4689 	 Unsupervised Cost: 620.0145 	 Validation Accuracy: 0.9516
268 | Epoch: 12 	 Total Cost: 1.4687 	 Supervised Cost: 1.4687 	 Unsupervised Cost: 620.0562 	 Validation Accuracy: 0.9536
269 | Epoch: 12 	 Total Cost: 1.4686 	 Supervised Cost: 1.4686 	 Unsupervised Cost: 619.0913 	 Validation Accuracy: 0.9526
270 | Epoch: 12 	 Total Cost: 1.4685 	 Supervised Cost: 1.4685 	 Unsupervised Cost: 618.5659 	 Validation Accuracy: 0.9511
271 | Epoch: 12 	 Total Cost: 1.4684 	 Supervised Cost: 1.4684 	 Unsupervised Cost: 618.8801 	 Validation Accuracy: 0.9505
272 | Epoch: 12 	 Total Cost: 1.4686 	 Supervised Cost: 1.4686 	 Unsupervised Cost: 619.5449 	 Validation Accuracy: 0.9467
273 | Epoch: 13 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 619.3216 	 Validation Accuracy: 0.9528
274 | Epoch: 13 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 618.3703 	 Validation Accuracy: 0.9498
275 | Epoch: 13 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 616.2033 	 Validation Accuracy: 0.9514
276 | Epoch: 13 	 Total Cost: 1.4695 	 Supervised Cost: 1.4695 	 Unsupervised Cost: 617.0727 	 Validation Accuracy: 0.9532
277 | Epoch: 13 	 Total Cost: 1.4695 	 Supervised Cost: 1.4695 	 Unsupervised Cost: 618.0974 	 Validation Accuracy: 0.9535
278 | Epoch: 13 	 Total Cost: 1.4695 	 Supervised Cost: 1.4695 	 Unsupervised Cost: 618.1368 	 Validation Accuracy: 0.9546
279 | Epoch: 13 	 Total Cost: 1.4694 	 Supervised Cost: 1.4694 	 Unsupervised Cost: 618.9428 	 Validation Accuracy: 0.9513
280 | Epoch: 13 	 Total Cost: 1.4692 	 Supervised Cost: 1.4692 	 Unsupervised Cost: 617.7905 	 Validation Accuracy: 0.9544
281 | Epoch: 13 	 Total Cost: 1.4691 	 Supervised Cost: 1.4691 	 Unsupervised Cost: 618.8152 	 Validation Accuracy: 0.9559
282 | Epoch: 13 	 Total Cost: 1.4690 	 Supervised Cost: 1.4690 	 Unsupervised Cost: 618.6389 	 Validation Accuracy: 0.9568
283 | Epoch: 13 	 Total Cost: 1.4687 	 Supervised Cost: 1.4687 	 Unsupervised Cost: 619.1425 	 Validation Accuracy: 0.9559
284 | Epoch: 13 	 Total Cost: 1.4686 	 Supervised Cost: 1.4686 	 Unsupervised Cost: 619.7048 	 Validation Accuracy: 0.9551
285 | Epoch: 13 	 Total Cost: 1.4684 	 Supervised Cost: 1.4684 	 Unsupervised Cost: 619.5794 	 Validation Accuracy: 0.9561
286 | Epoch: 13 	 Total Cost: 1.4685 	 Supervised Cost: 1.4685 	 Unsupervised Cost: 619.9695 	 Validation Accuracy: 0.9521
287 | Epoch: 13 	 Total Cost: 1.4685 	 Supervised Cost: 1.4685 	 Unsupervised Cost: 619.6965 	 Validation Accuracy: 0.9528
288 | Epoch: 14 	 Total Cost: 1.4696 	 Supervised Cost: 1.4696 	 Unsupervised Cost: 619.7625 	 Validation Accuracy: 0.9538
289 | Epoch: 14 	 Total Cost: 1.4696 	 Supervised Cost: 1.4696 	 Unsupervised Cost: 615.8887 	 Validation Accuracy: 0.9532
290 | Epoch: 14 	 Total Cost: 1.4693 	 Supervised Cost: 1.4693 	 Unsupervised Cost: 616.9642 	 Validation Accuracy: 0.9555
291 | Epoch: 14 	 Total Cost: 1.4689 	 Supervised Cost: 1.4689 	 Unsupervised Cost: 617.9998 	 Validation Accuracy: 0.9546
292 | Epoch: 14 	 Total Cost: 1.4686 	 Supervised Cost: 1.4686 	 Unsupervised Cost: 618.0280 	 Validation Accuracy: 0.9556
293 | Epoch: 14 	 Total Cost: 1.4683 	 Supervised Cost: 1.4683 	 Unsupervised Cost: 618.7677 	 Validation Accuracy: 0.9546
294 | Epoch: 14 	 Total Cost: 1.4684 	 Supervised Cost: 1.4684 	 Unsupervised Cost: 620.3161 	 Validation Accuracy: 0.953
295 | Epoch: 14 	 Total Cost: 1.4681 	 Supervised Cost: 1.4681 	 Unsupervised Cost: 621.0992 	 Validation Accuracy: 0.956
296 | Epoch: 14 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 620.5372 	 Validation Accuracy: 0.9574
297 | Epoch: 14 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 620.8937 	 Validation Accuracy: 0.9536
298 | Epoch: 14 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 620.3749 	 Validation Accuracy: 0.9538
299 | Epoch: 14 	 Total Cost: 1.4677 	 Supervised Cost: 1.4677 	 Unsupervised Cost: 620.0313 	 Validation Accuracy: 0.9536
300 | Epoch: 14 	 Total Cost: 1.4677 	 Supervised Cost: 1.4677 	 Unsupervised Cost: 620.2450 	 Validation Accuracy: 0.9583
301 | Epoch: 14 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 620.0166 	 Validation Accuracy: 0.9565
302 | Epoch: 14 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.2813 	 Validation Accuracy: 0.9567
303 | Epoch: 15 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 612.8694 	 Validation Accuracy: 0.9517
304 | Epoch: 15 	 Total Cost: 1.4688 	 Supervised Cost: 1.4688 	 Unsupervised Cost: 613.1500 	 Validation Accuracy: 0.9494
305 | Epoch: 15 	 Total Cost: 1.4688 	 Supervised Cost: 1.4688 	 Unsupervised Cost: 615.0682 	 Validation Accuracy: 0.9516
306 | Epoch: 15 	 Total Cost: 1.4689 	 Supervised Cost: 1.4689 	 Unsupervised Cost: 618.1895 	 Validation Accuracy: 0.9532
307 | Epoch: 15 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 617.7075 	 Validation Accuracy: 0.9471
308 | Epoch: 15 	 Total Cost: 1.4705 	 Supervised Cost: 1.4705 	 Unsupervised Cost: 617.1302 	 Validation Accuracy: 0.9547
309 | Epoch: 15 	 Total Cost: 1.4701 	 Supervised Cost: 1.4701 	 Unsupervised Cost: 618.2320 	 Validation Accuracy: 0.9562
310 | Epoch: 15 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 618.7428 	 Validation Accuracy: 0.9548
311 | Epoch: 15 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 619.3585 	 Validation Accuracy: 0.953
312 | Epoch: 15 	 Total Cost: 1.4698 	 Supervised Cost: 1.4698 	 Unsupervised Cost: 618.9759 	 Validation Accuracy: 0.9524
313 | Epoch: 15 	 Total Cost: 1.4697 	 Supervised Cost: 1.4697 	 Unsupervised Cost: 619.5906 	 Validation Accuracy: 0.9524
314 | Epoch: 15 	 Total Cost: 1.4695 	 Supervised Cost: 1.4695 	 Unsupervised Cost: 618.6243 	 Validation Accuracy: 0.9515
315 | Epoch: 15 	 Total Cost: 1.4694 	 Supervised Cost: 1.4694 	 Unsupervised Cost: 619.0312 	 Validation Accuracy: 0.9547
316 | Epoch: 15 	 Total Cost: 1.4692 	 Supervised Cost: 1.4692 	 Unsupervised Cost: 619.1289 	 Validation Accuracy: 0.956
317 | Epoch: 15 	 Total Cost: 1.4689 	 Supervised Cost: 1.4689 	 Unsupervised Cost: 619.3169 	 Validation Accuracy: 0.9568
318 | Epoch: 16 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 629.1044 	 Validation Accuracy: 0.9545
319 | Epoch: 16 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 625.3202 	 Validation Accuracy: 0.9519
320 | Epoch: 16 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 622.8749 	 Validation Accuracy: 0.9506
321 | Epoch: 16 	 Total Cost: 1.4680 	 Supervised Cost: 1.4680 	 Unsupervised Cost: 621.0934 	 Validation Accuracy: 0.9477
322 | Epoch: 16 	 Total Cost: 1.4680 	 Supervised Cost: 1.4680 	 Unsupervised Cost: 621.1300 	 Validation Accuracy: 0.954
323 | Epoch: 16 	 Total Cost: 1.4679 	 Supervised Cost: 1.4679 	 Unsupervised Cost: 621.3480 	 Validation Accuracy: 0.9544
324 | Epoch: 16 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 619.5699 	 Validation Accuracy: 0.9545
325 | Epoch: 16 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.2104 	 Validation Accuracy: 0.9546
326 | Epoch: 16 	 Total Cost: 1.4674 	 Supervised Cost: 1.4674 	 Unsupervised Cost: 619.7170 	 Validation Accuracy: 0.9575
327 | Epoch: 16 	 Total Cost: 1.4672 	 Supervised Cost: 1.4672 	 Unsupervised Cost: 619.5116 	 Validation Accuracy: 0.9549
328 | Epoch: 16 	 Total Cost: 1.4672 	 Supervised Cost: 1.4672 	 Unsupervised Cost: 619.1034 	 Validation Accuracy: 0.9542
329 | Epoch: 16 	 Total Cost: 1.4671 	 Supervised Cost: 1.4671 	 Unsupervised Cost: 618.5131 	 Validation Accuracy: 0.9529
330 | Epoch: 16 	 Total Cost: 1.4670 	 Supervised Cost: 1.4670 	 Unsupervised Cost: 619.3877 	 Validation Accuracy: 0.9533
331 | Epoch: 16 	 Total Cost: 1.4670 	 Supervised Cost: 1.4670 	 Unsupervised Cost: 619.0357 	 Validation Accuracy: 0.9566
332 | Epoch: 16 	 Total Cost: 1.4669 	 Supervised Cost: 1.4669 	 Unsupervised Cost: 619.3765 	 Validation Accuracy: 0.9547
333 | Epoch: 17 	 Total Cost: 1.4707 	 Supervised Cost: 1.4707 	 Unsupervised Cost: 617.6874 	 Validation Accuracy: 0.9473
334 | Epoch: 17 	 Total Cost: 1.4690 	 Supervised Cost: 1.4690 	 Unsupervised Cost: 620.5945 	 Validation Accuracy: 0.9553
335 | Epoch: 17 	 Total Cost: 1.4689 	 Supervised Cost: 1.4689 	 Unsupervised Cost: 620.2999 	 Validation Accuracy: 0.9528
336 | Epoch: 17 	 Total Cost: 1.4683 	 Supervised Cost: 1.4683 	 Unsupervised Cost: 619.0939 	 Validation Accuracy: 0.9529
337 | Epoch: 17 	 Total Cost: 1.4679 	 Supervised Cost: 1.4679 	 Unsupervised Cost: 618.4993 	 Validation Accuracy: 0.9509
338 | Epoch: 17 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 617.1129 	 Validation Accuracy: 0.9517
339 | Epoch: 17 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 616.0190 	 Validation Accuracy: 0.9537
340 | Epoch: 17 	 Total Cost: 1.4682 	 Supervised Cost: 1.4682 	 Unsupervised Cost: 618.3886 	 Validation Accuracy: 0.9506
341 | Epoch: 17 	 Total Cost: 1.4682 	 Supervised Cost: 1.4682 	 Unsupervised Cost: 619.1242 	 Validation Accuracy: 0.9531
342 | Epoch: 17 	 Total Cost: 1.4678 	 Supervised Cost: 1.4678 	 Unsupervised Cost: 619.0249 	 Validation Accuracy: 0.9535
343 | Epoch: 17 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.3063 	 Validation Accuracy: 0.9532
344 | Epoch: 17 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.6612 	 Validation Accuracy: 0.9527
345 | Epoch: 17 	 Total Cost: 1.4677 	 Supervised Cost: 1.4677 	 Unsupervised Cost: 619.3156 	 Validation Accuracy: 0.9537
346 | Epoch: 17 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.3701 	 Validation Accuracy: 0.9548
347 | Epoch: 17 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 619.3220 	 Validation Accuracy: 0.9562
348 | Epoch: 18 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 610.8782 	 Validation Accuracy: 0.9562
349 | Epoch: 18 	 Total Cost: 1.4653 	 Supervised Cost: 1.4653 	 Unsupervised Cost: 617.4746 	 Validation Accuracy: 0.9547
350 | Epoch: 18 	 Total Cost: 1.4654 	 Supervised Cost: 1.4654 	 Unsupervised Cost: 615.9659 	 Validation Accuracy: 0.9542
351 | Epoch: 18 	 Total Cost: 1.4655 	 Supervised Cost: 1.4655 	 Unsupervised Cost: 617.5388 	 Validation Accuracy: 0.9532
352 | Epoch: 18 	 Total Cost: 1.4658 	 Supervised Cost: 1.4658 	 Unsupervised Cost: 618.8958 	 Validation Accuracy: 0.9555
353 | Epoch: 18 	 Total Cost: 1.4657 	 Supervised Cost: 1.4657 	 Unsupervised Cost: 619.8357 	 Validation Accuracy: 0.9535
354 | Epoch: 18 	 Total Cost: 1.4662 	 Supervised Cost: 1.4662 	 Unsupervised Cost: 619.6097 	 Validation Accuracy: 0.9514
355 | Epoch: 18 	 Total Cost: 1.4664 	 Supervised Cost: 1.4664 	 Unsupervised Cost: 619.8222 	 Validation Accuracy: 0.9544
356 | Epoch: 18 	 Total Cost: 1.4664 	 Supervised Cost: 1.4664 	 Unsupervised Cost: 619.6141 	 Validation Accuracy: 0.9554
357 | Epoch: 18 	 Total Cost: 1.4662 	 Supervised Cost: 1.4662 	 Unsupervised Cost: 619.8137 	 Validation Accuracy: 0.9557
358 | Epoch: 18 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 620.1991 	 Validation Accuracy: 0.9526
359 | Epoch: 18 	 Total Cost: 1.4664 	 Supervised Cost: 1.4664 	 Unsupervised Cost: 620.3352 	 Validation Accuracy: 0.946
360 | Epoch: 18 	 Total Cost: 1.4665 	 Supervised Cost: 1.4665 	 Unsupervised Cost: 620.6739 	 Validation Accuracy: 0.9503
361 | Epoch: 18 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 620.3650 	 Validation Accuracy: 0.9478
362 | Epoch: 18 	 Total Cost: 1.4669 	 Supervised Cost: 1.4669 	 Unsupervised Cost: 619.9524 	 Validation Accuracy: 0.9484
363 | Epoch: 19 	 Total Cost: 1.4676 	 Supervised Cost: 1.4676 	 Unsupervised Cost: 618.4312 	 Validation Accuracy: 0.951
364 | Epoch: 19 	 Total Cost: 1.4668 	 Supervised Cost: 1.4668 	 Unsupervised Cost: 618.1408 	 Validation Accuracy: 0.9523
365 | Epoch: 19 	 Total Cost: 1.4657 	 Supervised Cost: 1.4657 	 Unsupervised Cost: 618.9991 	 Validation Accuracy: 0.9513
366 | Epoch: 19 	 Total Cost: 1.4657 	 Supervised Cost: 1.4657 	 Unsupervised Cost: 621.5109 	 Validation Accuracy: 0.9491
367 | Epoch: 19 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 620.0003 	 Validation Accuracy: 0.9507
368 | Epoch: 19 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 620.1013 	 Validation Accuracy: 0.9513
369 | Epoch: 19 	 Total Cost: 1.4664 	 Supervised Cost: 1.4664 	 Unsupervised Cost: 620.4848 	 Validation Accuracy: 0.9518
370 | Epoch: 19 	 Total Cost: 1.4665 	 Supervised Cost: 1.4665 	 Unsupervised Cost: 620.8880 	 Validation Accuracy: 0.9521
371 | Epoch: 19 	 Total Cost: 1.4664 	 Supervised Cost: 1.4664 	 Unsupervised Cost: 621.6099 	 Validation Accuracy: 0.9531
372 | Epoch: 19 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 621.0426 	 Validation Accuracy: 0.9555
373 | Epoch: 19 	 Total Cost: 1.4667 	 Supervised Cost: 1.4667 	 Unsupervised Cost: 620.7768 	 Validation Accuracy: 0.9501
374 | Epoch: 19 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 620.9622 	 Validation Accuracy: 0.9526
375 | Epoch: 19 	 Total Cost: 1.4665 	 Supervised Cost: 1.4665 	 Unsupervised Cost: 619.5601 	 Validation Accuracy: 0.9511
376 | Epoch: 19 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 619.7033 	 Validation Accuracy: 0.9521
377 | Epoch: 19 	 Total Cost: 1.4667 	 Supervised Cost: 1.4667 	 Unsupervised Cost: 619.4828 	 Validation Accuracy: 0.9548
378 | Epoch: 20 	 Total Cost: 1.4667 	 Supervised Cost: 1.4667 	 Unsupervised Cost: 616.5818 	 Validation Accuracy: 0.9499
379 | Epoch: 20 	 Total Cost: 1.4673 	 Supervised Cost: 1.4673 	 Unsupervised Cost: 616.6656 	 Validation Accuracy: 0.9525
380 | Epoch: 20 	 Total Cost: 1.4674 	 Supervised Cost: 1.4674 	 Unsupervised Cost: 617.3407 	 Validation Accuracy: 0.9502
381 | Epoch: 20 	 Total Cost: 1.4672 	 Supervised Cost: 1.4672 	 Unsupervised Cost: 620.1535 	 Validation Accuracy: 0.9525
382 | Epoch: 20 	 Total Cost: 1.4671 	 Supervised Cost: 1.4671 	 Unsupervised Cost: 619.8682 	 Validation Accuracy: 0.9519
383 | Epoch: 20 	 Total Cost: 1.4666 	 Supervised Cost: 1.4666 	 Unsupervised Cost: 619.3270 	 Validation Accuracy: 0.9522
384 | Epoch: 20 	 Total Cost: 1.4667 	 Supervised Cost: 1.4667 	 Unsupervised Cost: 620.2370 	 Validation Accuracy: 0.9533
385 | Epoch: 20 	 Total Cost: 1.4665 	 Supervised Cost: 1.4665 	 Unsupervised Cost: 620.0918 	 Validation Accuracy: 0.9556
386 | Epoch: 20 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 620.0182 	 Validation Accuracy: 0.953
387 | Epoch: 20 	 Total Cost: 1.4665 	 Supervised Cost: 1.4665 	 Unsupervised Cost: 619.2569 	 Validation Accuracy: 0.9546
388 | Epoch: 20 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 619.4088 	 Validation Accuracy: 0.9532
389 | Epoch: 20 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 619.4327 	 Validation Accuracy: 0.9526
390 | Epoch: 20 	 Total Cost: 1.4663 	 Supervised Cost: 1.4663 	 Unsupervised Cost: 619.7021 	 Validation Accuracy: 0.9513
391 | Epoch: 20 	 Total Cost: 1.4662 	 Supervised Cost: 1.4662 	 Unsupervised Cost: 620.0468 	 Validation Accuracy: 0.9545
392 | Epoch: 20 	 Total Cost: 1.4662 	 Supervised Cost: 1.4662 	 Unsupervised Cost: 619.8761 	 Validation Accuracy: 0.9564
393 | =====================
394 | 
395 | Done :)
396 | 


--------------------------------------------------------------------------------
/logs/ladder_supervised_unsupervised.log:
--------------------------------------------------------------------------------
  1 | =====================
  2 | BATCH SIZE: 100
  3 | EPOCHS: 20
  4 | NOISE STD: 0.2
  5 | CUDA: True
  6 | =====================
  7 | 
  8 | Loading Data
  9 | 
 10 | ========NETWORK=======
 11 | Ladder (
 12 |   (se): StackedEncoders (
 13 |     (encoders): Sequential (
 14 |       (encoder_0): Encoder (
 15 |         (linear): Linear (784 -> 1000)
 16 |         (bn_normalize_clean): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 17 |         (bn_normalize): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 18 |         (activation): ReLU ()
 19 |       )
 20 |       (encoder_1): Encoder (
 21 |         (linear): Linear (1000 -> 500)
 22 |         (bn_normalize_clean): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 23 |         (bn_normalize): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 24 |         (activation): ReLU ()
 25 |       )
 26 |       (encoder_2): Encoder (
 27 |         (linear): Linear (500 -> 250)
 28 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 29 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 30 |         (activation): ReLU ()
 31 |       )
 32 |       (encoder_3): Encoder (
 33 |         (linear): Linear (250 -> 250)
 34 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 35 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 36 |         (activation): ReLU ()
 37 |       )
 38 |       (encoder_4): Encoder (
 39 |         (linear): Linear (250 -> 250)
 40 |         (bn_normalize_clean): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 41 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 42 |         (activation): ReLU ()
 43 |       )
 44 |       (encoder_5): Encoder (
 45 |         (linear): Linear (250 -> 10)
 46 |         (bn_normalize_clean): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 47 |         (bn_normalize): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 48 |         (activation): Softmax ()
 49 |       )
 50 |     )
 51 |   )
 52 |   (de): StackedDecoders (
 53 |     (bn_u_top): BatchNorm1d(10, eps=1e-05, momentum=0.1, affine=False)
 54 |     (decoders): Sequential (
 55 |       (decoder_0): Decoder (
 56 |         (V): Linear (10 -> 250)
 57 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 58 |       )
 59 |       (decoder_1): Decoder (
 60 |         (V): Linear (250 -> 250)
 61 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 62 |       )
 63 |       (decoder_2): Decoder (
 64 |         (V): Linear (250 -> 250)
 65 |         (bn_normalize): BatchNorm1d(250, eps=1e-05, momentum=0.1, affine=False)
 66 |       )
 67 |       (decoder_3): Decoder (
 68 |         (V): Linear (250 -> 500)
 69 |         (bn_normalize): BatchNorm1d(500, eps=1e-05, momentum=0.1, affine=False)
 70 |       )
 71 |       (decoder_4): Decoder (
 72 |         (V): Linear (500 -> 1000)
 73 |         (bn_normalize): BatchNorm1d(1000, eps=1e-05, momentum=0.1, affine=False)
 74 |       )
 75 |       (decoder_5): Decoder (
 76 |         (V): Linear (1000 -> 784)
 77 |         (bn_normalize): BatchNorm1d(784, eps=1e-05, momentum=0.1, affine=False)
 78 |       )
 79 |     )
 80 |     (bottom_decoder): Decoder (
 81 |     )
 82 |   )
 83 |   (bn_image): BatchNorm1d(784, eps=1e-05, momentum=0.1, affine=False)
 84 | )
 85 | ======================
 86 | 
 87 | ==UNSUPERVISED-COSTS==
 88 | [0.1, 0.1, 0.1, 0.1, 0.1, 10.0, 1000.0]
 89 | 
 90 | =====================
 91 | TRAINING
 92 | 
 93 | Epoch: 1 	 Total Cost: 550.9735 	 Supervised Cost: 2.0765 	 Unsupervised Cost: 548.8970 	 Validation Accuracy: 0.7321
 94 | Epoch: 1 	 Total Cost: 427.1570 	 Supervised Cost: 2.0066 	 Unsupervised Cost: 425.1504 	 Validation Accuracy: 0.7991
 95 | Epoch: 1 	 Total Cost: 350.8330 	 Supervised Cost: 1.9685 	 Unsupervised Cost: 348.8644 	 Validation Accuracy: 0.7557
 96 | Epoch: 1 	 Total Cost: 298.2721 	 Supervised Cost: 1.9439 	 Unsupervised Cost: 296.3282 	 Validation Accuracy: 0.8086
 97 | Epoch: 1 	 Total Cost: 261.9606 	 Supervised Cost: 1.9291 	 Unsupervised Cost: 260.0314 	 Validation Accuracy: 0.7908
 98 | Epoch: 1 	 Total Cost: 238.2991 	 Supervised Cost: 1.9111 	 Unsupervised Cost: 236.3881 	 Validation Accuracy: 0.7832
 99 | Epoch: 1 	 Total Cost: 218.3870 	 Supervised Cost: 1.8966 	 Unsupervised Cost: 216.4904 	 Validation Accuracy: 0.8341
100 | Epoch: 1 	 Total Cost: 202.6910 	 Supervised Cost: 1.8836 	 Unsupervised Cost: 200.8074 	 Validation Accuracy: 0.7877
101 | Epoch: 1 	 Total Cost: 191.6323 	 Supervised Cost: 1.8859 	 Unsupervised Cost: 189.7464 	 Validation Accuracy: 0.5965
102 | Epoch: 1 	 Total Cost: 183.1738 	 Supervised Cost: 1.8982 	 Unsupervised Cost: 181.2756 	 Validation Accuracy: 0.5728
103 | Epoch: 1 	 Total Cost: 175.2309 	 Supervised Cost: 1.9097 	 Unsupervised Cost: 173.3212 	 Validation Accuracy: 0.5687
104 | Epoch: 1 	 Total Cost: 167.4628 	 Supervised Cost: 1.9245 	 Unsupervised Cost: 165.5382 	 Validation Accuracy: 0.4546
105 | Epoch: 1 	 Total Cost: 162.2987 	 Supervised Cost: 1.9351 	 Unsupervised Cost: 160.3636 	 Validation Accuracy: 0.4543
106 | Epoch: 1 	 Total Cost: 158.2400 	 Supervised Cost: 1.9421 	 Unsupervised Cost: 156.2979 	 Validation Accuracy: 0.5557
107 | Epoch: 1 	 Total Cost: 154.1734 	 Supervised Cost: 1.9473 	 Unsupervised Cost: 152.2261 	 Validation Accuracy: 0.5283
108 | Epoch: 2 	 Total Cost: 88.7336 	 Supervised Cost: 2.0193 	 Unsupervised Cost: 86.7143 	 Validation Accuracy: 0.5409
109 | Epoch: 2 	 Total Cost: 94.6015 	 Supervised Cost: 2.0189 	 Unsupervised Cost: 92.5826 	 Validation Accuracy: 0.4648
110 | Epoch: 2 	 Total Cost: 92.8685 	 Supervised Cost: 2.0469 	 Unsupervised Cost: 90.8216 	 Validation Accuracy: 0.3824
111 | Epoch: 2 	 Total Cost: 90.8853 	 Supervised Cost: 2.0813 	 Unsupervised Cost: 88.8041 	 Validation Accuracy: 0.2393
112 | Epoch: 2 	 Total Cost: 93.3638 	 Supervised Cost: 2.1112 	 Unsupervised Cost: 91.2526 	 Validation Accuracy: 0.2312
113 | Epoch: 2 	 Total Cost: 91.9885 	 Supervised Cost: 2.1292 	 Unsupervised Cost: 89.8593 	 Validation Accuracy: 0.2425
114 | Epoch: 2 	 Total Cost: 92.1342 	 Supervised Cost: 2.1401 	 Unsupervised Cost: 89.9940 	 Validation Accuracy: 0.3045
115 | Epoch: 2 	 Total Cost: 90.7103 	 Supervised Cost: 2.1468 	 Unsupervised Cost: 88.5634 	 Validation Accuracy: 0.3224
116 | Epoch: 2 	 Total Cost: 89.2156 	 Supervised Cost: 2.1512 	 Unsupervised Cost: 87.0644 	 Validation Accuracy: 0.322
117 | Epoch: 2 	 Total Cost: 87.9017 	 Supervised Cost: 2.1550 	 Unsupervised Cost: 85.7467 	 Validation Accuracy: 0.3363
118 | Epoch: 2 	 Total Cost: 86.5533 	 Supervised Cost: 2.1579 	 Unsupervised Cost: 84.3954 	 Validation Accuracy: 0.3222
119 | Epoch: 2 	 Total Cost: 86.2838 	 Supervised Cost: 2.1606 	 Unsupervised Cost: 84.1232 	 Validation Accuracy: 0.2631
120 | Epoch: 2 	 Total Cost: 84.2654 	 Supervised Cost: 2.1626 	 Unsupervised Cost: 82.1028 	 Validation Accuracy: 0.2629
121 | Epoch: 2 	 Total Cost: 83.5138 	 Supervised Cost: 2.1643 	 Unsupervised Cost: 81.3495 	 Validation Accuracy: 0.2571
122 | Epoch: 2 	 Total Cost: 82.2271 	 Supervised Cost: 2.1662 	 Unsupervised Cost: 80.0609 	 Validation Accuracy: 0.2515
123 | Epoch: 3 	 Total Cost: 73.1144 	 Supervised Cost: 2.1938 	 Unsupervised Cost: 70.9206 	 Validation Accuracy: 0.261
124 | Epoch: 3 	 Total Cost: 73.2463 	 Supervised Cost: 2.1933 	 Unsupervised Cost: 71.0529 	 Validation Accuracy: 0.2576
125 | Epoch: 3 	 Total Cost: 88.0400 	 Supervised Cost: 2.1954 	 Unsupervised Cost: 85.8446 	 Validation Accuracy: 0.2489
126 | Epoch: 3 	 Total Cost: 91.2248 	 Supervised Cost: 2.1973 	 Unsupervised Cost: 89.0275 	 Validation Accuracy: 0.2495
127 | Epoch: 3 	 Total Cost: 88.4062 	 Supervised Cost: 2.1991 	 Unsupervised Cost: 86.2071 	 Validation Accuracy: 0.246
128 | Epoch: 3 	 Total Cost: 86.3741 	 Supervised Cost: 2.1991 	 Unsupervised Cost: 84.1750 	 Validation Accuracy: 0.2558
129 | Epoch: 3 	 Total Cost: 86.7902 	 Supervised Cost: 2.1989 	 Unsupervised Cost: 84.5913 	 Validation Accuracy: 0.2586
130 | Epoch: 3 	 Total Cost: 83.9120 	 Supervised Cost: 2.1991 	 Unsupervised Cost: 81.7129 	 Validation Accuracy: 0.252
131 | Epoch: 3 	 Total Cost: 82.5111 	 Supervised Cost: 2.1990 	 Unsupervised Cost: 80.3121 	 Validation Accuracy: 0.2414
132 | Epoch: 3 	 Total Cost: 82.1774 	 Supervised Cost: 2.1992 	 Unsupervised Cost: 79.9781 	 Validation Accuracy: 0.2381
133 | Epoch: 3 	 Total Cost: 82.5350 	 Supervised Cost: 2.2000 	 Unsupervised Cost: 80.3351 	 Validation Accuracy: 0.2505
134 | Epoch: 3 	 Total Cost: 84.1461 	 Supervised Cost: 2.2001 	 Unsupervised Cost: 81.9460 	 Validation Accuracy: 0.2463
135 | Epoch: 3 	 Total Cost: 83.8969 	 Supervised Cost: 2.1999 	 Unsupervised Cost: 81.6970 	 Validation Accuracy: 0.2561
136 | Epoch: 3 	 Total Cost: 83.9111 	 Supervised Cost: 2.1995 	 Unsupervised Cost: 81.7117 	 Validation Accuracy: 0.2596
137 | Epoch: 3 	 Total Cost: 82.5549 	 Supervised Cost: 2.1988 	 Unsupervised Cost: 80.3560 	 Validation Accuracy: 0.2605
138 | Epoch: 4 	 Total Cost: 67.2953 	 Supervised Cost: 2.1860 	 Unsupervised Cost: 65.1093 	 Validation Accuracy: 0.2599
139 | Epoch: 4 	 Total Cost: 73.2184 	 Supervised Cost: 2.1872 	 Unsupervised Cost: 71.0313 	 Validation Accuracy: 0.2596
140 | Epoch: 4 	 Total Cost: 76.5452 	 Supervised Cost: 2.1866 	 Unsupervised Cost: 74.3586 	 Validation Accuracy: 0.2621
141 | Epoch: 4 	 Total Cost: 76.6005 	 Supervised Cost: 2.1864 	 Unsupervised Cost: 74.4141 	 Validation Accuracy: 0.2727
142 | Epoch: 4 	 Total Cost: 80.3451 	 Supervised Cost: 2.1859 	 Unsupervised Cost: 78.1592 	 Validation Accuracy: 0.2684
143 | Epoch: 4 	 Total Cost: 79.6779 	 Supervised Cost: 2.1854 	 Unsupervised Cost: 77.4924 	 Validation Accuracy: 0.2592
144 | Epoch: 4 	 Total Cost: 79.3560 	 Supervised Cost: 2.1855 	 Unsupervised Cost: 77.1706 	 Validation Accuracy: 0.2655
145 | Epoch: 4 	 Total Cost: 82.6309 	 Supervised Cost: 2.1851 	 Unsupervised Cost: 80.4458 	 Validation Accuracy: 0.2654
146 | Epoch: 4 	 Total Cost: 82.1727 	 Supervised Cost: 2.1848 	 Unsupervised Cost: 79.9879 	 Validation Accuracy: 0.2648
147 | Epoch: 4 	 Total Cost: 81.8601 	 Supervised Cost: 2.1852 	 Unsupervised Cost: 79.6749 	 Validation Accuracy: 0.268
148 | Epoch: 4 	 Total Cost: 81.4769 	 Supervised Cost: 2.1857 	 Unsupervised Cost: 79.2912 	 Validation Accuracy: 0.2716
149 | Epoch: 4 	 Total Cost: 81.4360 	 Supervised Cost: 2.1864 	 Unsupervised Cost: 79.2496 	 Validation Accuracy: 0.2667
150 | Epoch: 4 	 Total Cost: 80.5623 	 Supervised Cost: 2.1870 	 Unsupervised Cost: 78.3753 	 Validation Accuracy: 0.2733
151 | Epoch: 4 	 Total Cost: 79.6876 	 Supervised Cost: 2.1872 	 Unsupervised Cost: 77.5004 	 Validation Accuracy: 0.2695
152 | Epoch: 4 	 Total Cost: 79.0121 	 Supervised Cost: 2.1876 	 Unsupervised Cost: 76.8244 	 Validation Accuracy: 0.2715
153 | Epoch: 5 	 Total Cost: 87.9847 	 Supervised Cost: 2.1865 	 Unsupervised Cost: 85.7982 	 Validation Accuracy: 0.2728
154 | Epoch: 5 	 Total Cost: 83.1427 	 Supervised Cost: 2.2025 	 Unsupervised Cost: 80.9402 	 Validation Accuracy: 0.2187
155 | Epoch: 5 	 Total Cost: 77.0477 	 Supervised Cost: 2.2191 	 Unsupervised Cost: 74.8286 	 Validation Accuracy: 0.2254
156 | Epoch: 5 	 Total Cost: 73.4575 	 Supervised Cost: 2.2231 	 Unsupervised Cost: 71.2343 	 Validation Accuracy: 0.2374
157 | Epoch: 5 	 Total Cost: 73.2923 	 Supervised Cost: 2.2223 	 Unsupervised Cost: 71.0700 	 Validation Accuracy: 0.2409
158 | Epoch: 5 	 Total Cost: 75.7295 	 Supervised Cost: 2.2204 	 Unsupervised Cost: 73.5091 	 Validation Accuracy: 0.2852
159 | Epoch: 5 	 Total Cost: 73.5604 	 Supervised Cost: 2.2166 	 Unsupervised Cost: 71.3438 	 Validation Accuracy: 0.2973
160 | Epoch: 5 	 Total Cost: 73.3900 	 Supervised Cost: 2.2127 	 Unsupervised Cost: 71.1772 	 Validation Accuracy: 0.2828
161 | Epoch: 5 	 Total Cost: 73.4562 	 Supervised Cost: 2.2094 	 Unsupervised Cost: 71.2468 	 Validation Accuracy: 0.2758
162 | Epoch: 5 	 Total Cost: 72.3715 	 Supervised Cost: 2.2058 	 Unsupervised Cost: 70.1656 	 Validation Accuracy: 0.292
163 | Epoch: 5 	 Total Cost: 72.2904 	 Supervised Cost: 2.2033 	 Unsupervised Cost: 70.0872 	 Validation Accuracy: 0.319
164 | Epoch: 5 	 Total Cost: 72.0026 	 Supervised Cost: 2.2011 	 Unsupervised Cost: 69.8015 	 Validation Accuracy: 0.3193
165 | Epoch: 5 	 Total Cost: 71.1614 	 Supervised Cost: 2.1986 	 Unsupervised Cost: 68.9628 	 Validation Accuracy: 0.3277
166 | Epoch: 5 	 Total Cost: 72.7715 	 Supervised Cost: 2.1966 	 Unsupervised Cost: 70.5750 	 Validation Accuracy: 0.3223
167 | Epoch: 5 	 Total Cost: 74.2227 	 Supervised Cost: 2.1961 	 Unsupervised Cost: 72.0266 	 Validation Accuracy: 0.2753
168 | Epoch: 6 	 Total Cost: 65.8542 	 Supervised Cost: 2.2120 	 Unsupervised Cost: 63.6422 	 Validation Accuracy: 0.22
169 | Epoch: 6 	 Total Cost: 64.3261 	 Supervised Cost: 2.2253 	 Unsupervised Cost: 62.1009 	 Validation Accuracy: 0.201
170 | Epoch: 6 	 Total Cost: 67.8102 	 Supervised Cost: 2.2259 	 Unsupervised Cost: 65.5844 	 Validation Accuracy: 0.2149
171 | Epoch: 6 	 Total Cost: 70.3805 	 Supervised Cost: 2.2321 	 Unsupervised Cost: 68.1484 	 Validation Accuracy: 0.1908
172 | Epoch: 6 	 Total Cost: 68.2204 	 Supervised Cost: 2.2374 	 Unsupervised Cost: 65.9830 	 Validation Accuracy: 0.2342
173 | Epoch: 6 	 Total Cost: 69.4019 	 Supervised Cost: 2.2388 	 Unsupervised Cost: 67.1632 	 Validation Accuracy: 0.2479
174 | Epoch: 6 	 Total Cost: 67.5898 	 Supervised Cost: 2.2387 	 Unsupervised Cost: 65.3511 	 Validation Accuracy: 0.254
175 | Epoch: 6 	 Total Cost: 67.2521 	 Supervised Cost: 2.2374 	 Unsupervised Cost: 65.0148 	 Validation Accuracy: 0.2757
176 | Epoch: 6 	 Total Cost: 66.0649 	 Supervised Cost: 2.2344 	 Unsupervised Cost: 63.8306 	 Validation Accuracy: 0.3081
177 | Epoch: 6 	 Total Cost: 66.0666 	 Supervised Cost: 2.2300 	 Unsupervised Cost: 63.8366 	 Validation Accuracy: 0.3068
178 | Epoch: 6 	 Total Cost: 66.8038 	 Supervised Cost: 2.2253 	 Unsupervised Cost: 64.5784 	 Validation Accuracy: 0.3344
179 | Epoch: 6 	 Total Cost: 66.7807 	 Supervised Cost: 2.2208 	 Unsupervised Cost: 64.5599 	 Validation Accuracy: 0.3579
180 | Epoch: 6 	 Total Cost: 66.7902 	 Supervised Cost: 2.2175 	 Unsupervised Cost: 64.5727 	 Validation Accuracy: 0.3334
181 | Epoch: 6 	 Total Cost: 65.8723 	 Supervised Cost: 2.2158 	 Unsupervised Cost: 63.6565 	 Validation Accuracy: 0.3068
182 | Epoch: 6 	 Total Cost: 65.3281 	 Supervised Cost: 2.2135 	 Unsupervised Cost: 63.1146 	 Validation Accuracy: 0.3045
183 | Epoch: 7 	 Total Cost: 83.1965 	 Supervised Cost: 2.1730 	 Unsupervised Cost: 81.0234 	 Validation Accuracy: 0.332
184 | Epoch: 7 	 Total Cost: 93.2587 	 Supervised Cost: 2.1723 	 Unsupervised Cost: 91.0864 	 Validation Accuracy: 0.3326
185 | Epoch: 7 	 Total Cost: 87.3421 	 Supervised Cost: 2.1750 	 Unsupervised Cost: 85.1672 	 Validation Accuracy: 0.3375
186 | Epoch: 7 	 Total Cost: 84.1001 	 Supervised Cost: 2.1748 	 Unsupervised Cost: 81.9253 	 Validation Accuracy: 0.3364
187 | Epoch: 7 	 Total Cost: 84.1558 	 Supervised Cost: 2.1739 	 Unsupervised Cost: 81.9819 	 Validation Accuracy: 0.337
188 | Epoch: 7 	 Total Cost: 82.1699 	 Supervised Cost: 2.1719 	 Unsupervised Cost: 79.9980 	 Validation Accuracy: 0.3385
189 | Epoch: 7 	 Total Cost: 81.4186 	 Supervised Cost: 2.1690 	 Unsupervised Cost: 79.2495 	 Validation Accuracy: 0.3389
190 | Epoch: 7 	 Total Cost: 81.4082 	 Supervised Cost: 2.1658 	 Unsupervised Cost: 79.2425 	 Validation Accuracy: 0.345
191 | Epoch: 7 	 Total Cost: 82.1518 	 Supervised Cost: 2.1644 	 Unsupervised Cost: 79.9874 	 Validation Accuracy: 0.318
192 | Epoch: 7 	 Total Cost: 82.6243 	 Supervised Cost: 2.1659 	 Unsupervised Cost: 80.4584 	 Validation Accuracy: 0.2989
193 | Epoch: 7 	 Total Cost: 84.8085 	 Supervised Cost: 2.1690 	 Unsupervised Cost: 82.6395 	 Validation Accuracy: 0.2878
194 | Epoch: 7 	 Total Cost: 84.5991 	 Supervised Cost: 2.1720 	 Unsupervised Cost: 82.4271 	 Validation Accuracy: 0.2787
195 | Epoch: 7 	 Total Cost: 83.8585 	 Supervised Cost: 2.1763 	 Unsupervised Cost: 81.6822 	 Validation Accuracy: 0.2443
196 | Epoch: 7 	 Total Cost: 82.6279 	 Supervised Cost: 2.1807 	 Unsupervised Cost: 80.4472 	 Validation Accuracy: 0.247
197 | Epoch: 7 	 Total Cost: 82.7202 	 Supervised Cost: 2.1844 	 Unsupervised Cost: 80.5358 	 Validation Accuracy: 0.2592
198 | Epoch: 8 	 Total Cost: 71.3371 	 Supervised Cost: 2.2311 	 Unsupervised Cost: 69.1060 	 Validation Accuracy: 0.2615
199 | Epoch: 8 	 Total Cost: 68.3188 	 Supervised Cost: 2.2296 	 Unsupervised Cost: 66.0892 	 Validation Accuracy: 0.2668
200 | Epoch: 8 	 Total Cost: 70.1114 	 Supervised Cost: 2.2283 	 Unsupervised Cost: 67.8831 	 Validation Accuracy: 0.2649
201 | Epoch: 8 	 Total Cost: 71.6156 	 Supervised Cost: 2.2265 	 Unsupervised Cost: 69.3891 	 Validation Accuracy: 0.2693
202 | Epoch: 8 	 Total Cost: 72.9759 	 Supervised Cost: 2.2249 	 Unsupervised Cost: 70.7510 	 Validation Accuracy: 0.2555
203 | Epoch: 8 	 Total Cost: 73.5341 	 Supervised Cost: 2.2261 	 Unsupervised Cost: 71.3080 	 Validation Accuracy: 0.2126
204 | Epoch: 8 	 Total Cost: 73.0574 	 Supervised Cost: 2.2272 	 Unsupervised Cost: 70.8302 	 Validation Accuracy: 0.2158
205 | Epoch: 8 	 Total Cost: 74.0336 	 Supervised Cost: 2.2281 	 Unsupervised Cost: 71.8055 	 Validation Accuracy: 0.2166
206 | Epoch: 8 	 Total Cost: 75.1853 	 Supervised Cost: 2.2290 	 Unsupervised Cost: 72.9563 	 Validation Accuracy: 0.2162
207 | Epoch: 8 	 Total Cost: 74.8266 	 Supervised Cost: 2.2297 	 Unsupervised Cost: 72.5969 	 Validation Accuracy: 0.2203
208 | Epoch: 8 	 Total Cost: 74.9034 	 Supervised Cost: 2.2288 	 Unsupervised Cost: 72.6747 	 Validation Accuracy: 0.2532
209 | Epoch: 8 	 Total Cost: 73.3498 	 Supervised Cost: 2.2274 	 Unsupervised Cost: 71.1224 	 Validation Accuracy: 0.2499
210 | Epoch: 8 	 Total Cost: 71.5804 	 Supervised Cost: 2.2266 	 Unsupervised Cost: 69.3539 	 Validation Accuracy: 0.2511
211 | Epoch: 8 	 Total Cost: 71.0686 	 Supervised Cost: 2.2257 	 Unsupervised Cost: 68.8428 	 Validation Accuracy: 0.2545
212 | Epoch: 8 	 Total Cost: 70.1834 	 Supervised Cost: 2.2247 	 Unsupervised Cost: 67.9587 	 Validation Accuracy: 0.2639
213 | Epoch: 9 	 Total Cost: 75.0889 	 Supervised Cost: 2.2040 	 Unsupervised Cost: 72.8849 	 Validation Accuracy: 0.256
214 | Epoch: 9 	 Total Cost: 86.2165 	 Supervised Cost: 2.2011 	 Unsupervised Cost: 84.0154 	 Validation Accuracy: 0.2709
215 | Epoch: 9 	 Total Cost: 83.0000 	 Supervised Cost: 2.1984 	 Unsupervised Cost: 80.8016 	 Validation Accuracy: 0.267
216 | Epoch: 9 	 Total Cost: 82.4837 	 Supervised Cost: 2.1965 	 Unsupervised Cost: 80.2872 	 Validation Accuracy: 0.2654
217 | Epoch: 9 	 Total Cost: 81.4051 	 Supervised Cost: 2.1948 	 Unsupervised Cost: 79.2103 	 Validation Accuracy: 0.2744
218 | Epoch: 9 	 Total Cost: 78.3762 	 Supervised Cost: 2.1937 	 Unsupervised Cost: 76.1826 	 Validation Accuracy: 0.2797
219 | Epoch: 9 	 Total Cost: 75.0341 	 Supervised Cost: 2.1923 	 Unsupervised Cost: 72.8417 	 Validation Accuracy: 0.2793
220 | Epoch: 9 	 Total Cost: 72.9157 	 Supervised Cost: 2.1909 	 Unsupervised Cost: 70.7248 	 Validation Accuracy: 0.2834
221 | Epoch: 9 	 Total Cost: 72.0464 	 Supervised Cost: 2.1897 	 Unsupervised Cost: 69.8567 	 Validation Accuracy: 0.2838
222 | Epoch: 9 	 Total Cost: 71.2212 	 Supervised Cost: 2.1885 	 Unsupervised Cost: 69.0328 	 Validation Accuracy: 0.2839
223 | Epoch: 9 	 Total Cost: 71.4414 	 Supervised Cost: 2.1880 	 Unsupervised Cost: 69.2533 	 Validation Accuracy: 0.2814
224 | Epoch: 9 	 Total Cost: 70.4998 	 Supervised Cost: 2.1876 	 Unsupervised Cost: 68.3122 	 Validation Accuracy: 0.2845
225 | Epoch: 9 	 Total Cost: 70.1095 	 Supervised Cost: 2.1871 	 Unsupervised Cost: 67.9224 	 Validation Accuracy: 0.2812
226 | Epoch: 9 	 Total Cost: 69.9059 	 Supervised Cost: 2.1868 	 Unsupervised Cost: 67.7191 	 Validation Accuracy: 0.2816
227 | Epoch: 9 	 Total Cost: 69.8157 	 Supervised Cost: 2.1870 	 Unsupervised Cost: 67.6287 	 Validation Accuracy: 0.2564
228 | Epoch: 10 	 Total Cost: 61.6264 	 Supervised Cost: 2.2525 	 Unsupervised Cost: 59.3739 	 Validation Accuracy: 0.1798
229 | Epoch: 10 	 Total Cost: 55.6888 	 Supervised Cost: 2.2496 	 Unsupervised Cost: 53.4392 	 Validation Accuracy: 0.1832
230 | Epoch: 10 	 Total Cost: 55.3519 	 Supervised Cost: 2.2481 	 Unsupervised Cost: 53.1037 	 Validation Accuracy: 0.1855
231 | Epoch: 10 	 Total Cost: 54.4047 	 Supervised Cost: 2.2465 	 Unsupervised Cost: 52.1582 	 Validation Accuracy: 0.182
232 | Epoch: 10 	 Total Cost: 53.5091 	 Supervised Cost: 2.2462 	 Unsupervised Cost: 51.2628 	 Validation Accuracy: 0.1874
233 | Epoch: 10 	 Total Cost: 53.1895 	 Supervised Cost: 2.2457 	 Unsupervised Cost: 50.9438 	 Validation Accuracy: 0.1896
234 | Epoch: 10 	 Total Cost: 52.6875 	 Supervised Cost: 2.2451 	 Unsupervised Cost: 50.4425 	 Validation Accuracy: 0.188
235 | Epoch: 10 	 Total Cost: 52.6361 	 Supervised Cost: 2.2446 	 Unsupervised Cost: 50.3915 	 Validation Accuracy: 0.1913
236 | Epoch: 10 	 Total Cost: 52.1124 	 Supervised Cost: 2.2442 	 Unsupervised Cost: 49.8682 	 Validation Accuracy: 0.1859
237 | Epoch: 10 	 Total Cost: 54.1949 	 Supervised Cost: 2.2440 	 Unsupervised Cost: 51.9509 	 Validation Accuracy: 0.1906
238 | Epoch: 10 	 Total Cost: 54.1232 	 Supervised Cost: 2.2438 	 Unsupervised Cost: 51.8794 	 Validation Accuracy: 0.1919
239 | Epoch: 10 	 Total Cost: 53.8018 	 Supervised Cost: 2.2436 	 Unsupervised Cost: 51.5583 	 Validation Accuracy: 0.1894
240 | Epoch: 10 	 Total Cost: 53.3252 	 Supervised Cost: 2.2432 	 Unsupervised Cost: 51.0820 	 Validation Accuracy: 0.1944
241 | Epoch: 10 	 Total Cost: 55.9798 	 Supervised Cost: 2.2425 	 Unsupervised Cost: 53.7373 	 Validation Accuracy: 0.1978
242 | Epoch: 10 	 Total Cost: 56.4136 	 Supervised Cost: 2.2421 	 Unsupervised Cost: 54.1715 	 Validation Accuracy: 0.199
243 | Epoch: 11 	 Total Cost: 52.9115 	 Supervised Cost: 2.2357 	 Unsupervised Cost: 50.6758 	 Validation Accuracy: 0.1922
244 | Epoch: 11 	 Total Cost: 57.2202 	 Supervised Cost: 2.2355 	 Unsupervised Cost: 54.9848 	 Validation Accuracy: 0.2178
245 | Epoch: 11 	 Total Cost: 58.7296 	 Supervised Cost: 2.2339 	 Unsupervised Cost: 56.4957 	 Validation Accuracy: 0.2245
246 | Epoch: 11 	 Total Cost: 59.1785 	 Supervised Cost: 2.2327 	 Unsupervised Cost: 56.9458 	 Validation Accuracy: 0.2267
247 | Epoch: 11 	 Total Cost: 58.8833 	 Supervised Cost: 2.2317 	 Unsupervised Cost: 56.6515 	 Validation Accuracy: 0.216
248 | Epoch: 11 	 Total Cost: 60.3379 	 Supervised Cost: 2.2309 	 Unsupervised Cost: 58.1070 	 Validation Accuracy: 0.2202
249 | Epoch: 11 	 Total Cost: 61.6377 	 Supervised Cost: 2.2296 	 Unsupervised Cost: 59.4081 	 Validation Accuracy: 0.2221
250 | Epoch: 11 	 Total Cost: 61.9218 	 Supervised Cost: 2.2284 	 Unsupervised Cost: 59.6934 	 Validation Accuracy: 0.2387
251 | Epoch: 11 	 Total Cost: 63.8965 	 Supervised Cost: 2.2268 	 Unsupervised Cost: 61.6697 	 Validation Accuracy: 0.2358
252 | Epoch: 11 	 Total Cost: 62.9224 	 Supervised Cost: 2.2260 	 Unsupervised Cost: 60.6964 	 Validation Accuracy: 0.2343
253 | Epoch: 11 	 Total Cost: 62.2254 	 Supervised Cost: 2.2255 	 Unsupervised Cost: 59.9998 	 Validation Accuracy: 0.2502
254 | Epoch: 11 	 Total Cost: 61.9019 	 Supervised Cost: 2.2245 	 Unsupervised Cost: 59.6775 	 Validation Accuracy: 0.2475
255 | Epoch: 11 	 Total Cost: 61.8140 	 Supervised Cost: 2.2232 	 Unsupervised Cost: 59.5908 	 Validation Accuracy: 0.292
256 | Epoch: 11 	 Total Cost: 61.3285 	 Supervised Cost: 2.2218 	 Unsupervised Cost: 59.1067 	 Validation Accuracy: 0.2604
257 | Epoch: 11 	 Total Cost: 60.8978 	 Supervised Cost: 2.2209 	 Unsupervised Cost: 58.6769 	 Validation Accuracy: 0.2003
258 | Epoch: 12 	 Total Cost: 68.7332 	 Supervised Cost: 2.2391 	 Unsupervised Cost: 66.4941 	 Validation Accuracy: 0.2025
259 | Epoch: 12 	 Total Cost: 62.9622 	 Supervised Cost: 2.2394 	 Unsupervised Cost: 60.7228 	 Validation Accuracy: 0.2039
260 | Epoch: 12 	 Total Cost: 69.5101 	 Supervised Cost: 2.2384 	 Unsupervised Cost: 67.2718 	 Validation Accuracy: 0.2037
261 | Epoch: 12 	 Total Cost: 65.6796 	 Supervised Cost: 2.2387 	 Unsupervised Cost: 63.4409 	 Validation Accuracy: 0.2049
262 | Epoch: 12 	 Total Cost: 66.7399 	 Supervised Cost: 2.2382 	 Unsupervised Cost: 64.5017 	 Validation Accuracy: 0.2048
263 | Epoch: 12 	 Total Cost: 64.3115 	 Supervised Cost: 2.2381 	 Unsupervised Cost: 62.0734 	 Validation Accuracy: 0.2031
264 | Epoch: 12 	 Total Cost: 62.4230 	 Supervised Cost: 2.2379 	 Unsupervised Cost: 60.1851 	 Validation Accuracy: 0.2052
265 | Epoch: 12 	 Total Cost: 60.3900 	 Supervised Cost: 2.2373 	 Unsupervised Cost: 58.1527 	 Validation Accuracy: 0.2075
266 | Epoch: 12 	 Total Cost: 58.5694 	 Supervised Cost: 2.2369 	 Unsupervised Cost: 56.3324 	 Validation Accuracy: 0.2087
267 | Epoch: 12 	 Total Cost: 58.1173 	 Supervised Cost: 2.2365 	 Unsupervised Cost: 55.8808 	 Validation Accuracy: 0.209
268 | Epoch: 12 	 Total Cost: 58.3700 	 Supervised Cost: 2.2362 	 Unsupervised Cost: 56.1338 	 Validation Accuracy: 0.2036
269 | Epoch: 12 	 Total Cost: 58.2055 	 Supervised Cost: 2.2363 	 Unsupervised Cost: 55.9692 	 Validation Accuracy: 0.2038
270 | Epoch: 12 	 Total Cost: 57.8816 	 Supervised Cost: 2.2360 	 Unsupervised Cost: 55.6456 	 Validation Accuracy: 0.2044
271 | Epoch: 12 	 Total Cost: 58.0788 	 Supervised Cost: 2.2359 	 Unsupervised Cost: 55.8429 	 Validation Accuracy: 0.2041
272 | Epoch: 12 	 Total Cost: 57.5359 	 Supervised Cost: 2.2358 	 Unsupervised Cost: 55.3000 	 Validation Accuracy: 0.2067
273 | Epoch: 13 	 Total Cost: 72.6216 	 Supervised Cost: 2.2339 	 Unsupervised Cost: 70.3877 	 Validation Accuracy: 0.2094
274 | Epoch: 13 	 Total Cost: 72.7142 	 Supervised Cost: 2.2326 	 Unsupervised Cost: 70.4816 	 Validation Accuracy: 0.211
275 | Epoch: 13 	 Total Cost: 68.2693 	 Supervised Cost: 2.2319 	 Unsupervised Cost: 66.0374 	 Validation Accuracy: 0.2159
276 | Epoch: 13 	 Total Cost: 65.6347 	 Supervised Cost: 2.2309 	 Unsupervised Cost: 63.4038 	 Validation Accuracy: 0.2114
277 | Epoch: 13 	 Total Cost: 64.7060 	 Supervised Cost: 2.2304 	 Unsupervised Cost: 62.4756 	 Validation Accuracy: 0.213
278 | Epoch: 13 	 Total Cost: 63.0191 	 Supervised Cost: 2.2299 	 Unsupervised Cost: 60.7892 	 Validation Accuracy: 0.2193
279 | Epoch: 13 	 Total Cost: 61.2665 	 Supervised Cost: 2.2295 	 Unsupervised Cost: 59.0370 	 Validation Accuracy: 0.2191
280 | Epoch: 13 	 Total Cost: 61.8577 	 Supervised Cost: 2.2291 	 Unsupervised Cost: 59.6286 	 Validation Accuracy: 0.2137
281 | Epoch: 13 	 Total Cost: 61.3955 	 Supervised Cost: 2.2294 	 Unsupervised Cost: 59.1661 	 Validation Accuracy: 0.209
282 | Epoch: 13 	 Total Cost: 61.1257 	 Supervised Cost: 2.2298 	 Unsupervised Cost: 58.8958 	 Validation Accuracy: 0.2123
283 | Epoch: 13 	 Total Cost: 61.0087 	 Supervised Cost: 2.2300 	 Unsupervised Cost: 58.7788 	 Validation Accuracy: 0.1993
284 | Epoch: 13 	 Total Cost: 61.0160 	 Supervised Cost: 2.2303 	 Unsupervised Cost: 58.7857 	 Validation Accuracy: 0.1974
285 | Epoch: 13 	 Total Cost: 60.4915 	 Supervised Cost: 2.2307 	 Unsupervised Cost: 58.2608 	 Validation Accuracy: 0.1987
286 | Epoch: 13 	 Total Cost: 59.4273 	 Supervised Cost: 2.2308 	 Unsupervised Cost: 57.1966 	 Validation Accuracy: 0.1983
287 | Epoch: 13 	 Total Cost: 59.5587 	 Supervised Cost: 2.2309 	 Unsupervised Cost: 57.3278 	 Validation Accuracy: 0.2012
288 | Epoch: 14 	 Total Cost: 66.7181 	 Supervised Cost: 2.2273 	 Unsupervised Cost: 64.4908 	 Validation Accuracy: 0.1932
289 | Epoch: 14 	 Total Cost: 80.8125 	 Supervised Cost: 2.2282 	 Unsupervised Cost: 78.5844 	 Validation Accuracy: 0.1984
290 | Epoch: 14 	 Total Cost: 72.0786 	 Supervised Cost: 2.2286 	 Unsupervised Cost: 69.8499 	 Validation Accuracy: 0.197
291 | Epoch: 14 	 Total Cost: 68.1743 	 Supervised Cost: 2.2300 	 Unsupervised Cost: 65.9443 	 Validation Accuracy: 0.1995
292 | Epoch: 14 	 Total Cost: 65.5049 	 Supervised Cost: 2.2319 	 Unsupervised Cost: 63.2730 	 Validation Accuracy: 0.1999
293 | Epoch: 14 	 Total Cost: 64.2896 	 Supervised Cost: 2.2321 	 Unsupervised Cost: 62.0576 	 Validation Accuracy: 0.1988
294 | Epoch: 14 	 Total Cost: 62.4804 	 Supervised Cost: 2.2326 	 Unsupervised Cost: 60.2477 	 Validation Accuracy: 0.1949
295 | Epoch: 14 	 Total Cost: 61.2478 	 Supervised Cost: 2.2335 	 Unsupervised Cost: 59.0144 	 Validation Accuracy: 0.1946
296 | Epoch: 14 	 Total Cost: 59.6653 	 Supervised Cost: 2.2343 	 Unsupervised Cost: 57.4309 	 Validation Accuracy: 0.1975
297 | Epoch: 14 	 Total Cost: 58.2248 	 Supervised Cost: 2.2348 	 Unsupervised Cost: 55.9900 	 Validation Accuracy: 0.1936
298 | Epoch: 14 	 Total Cost: 58.3709 	 Supervised Cost: 2.2351 	 Unsupervised Cost: 56.1358 	 Validation Accuracy: 0.1977
299 | Epoch: 14 	 Total Cost: 58.1186 	 Supervised Cost: 2.2353 	 Unsupervised Cost: 55.8833 	 Validation Accuracy: 0.2002
300 | Epoch: 14 	 Total Cost: 57.6126 	 Supervised Cost: 2.2354 	 Unsupervised Cost: 55.3772 	 Validation Accuracy: 0.2021
301 | Epoch: 14 	 Total Cost: 57.5781 	 Supervised Cost: 2.2354 	 Unsupervised Cost: 55.3428 	 Validation Accuracy: 0.206
302 | Epoch: 14 	 Total Cost: 57.4099 	 Supervised Cost: 2.2356 	 Unsupervised Cost: 55.1744 	 Validation Accuracy: 0.1875
303 | Epoch: 15 	 Total Cost: 85.3538 	 Supervised Cost: 2.2464 	 Unsupervised Cost: 83.1075 	 Validation Accuracy: 0.1579
304 | Epoch: 15 	 Total Cost: 76.8993 	 Supervised Cost: 2.2576 	 Unsupervised Cost: 74.6417 	 Validation Accuracy: 0.1395
305 | Epoch: 15 	 Total Cost: 69.6807 	 Supervised Cost: 2.2613 	 Unsupervised Cost: 67.4194 	 Validation Accuracy: 0.1422
306 | Epoch: 15 	 Total Cost: 64.3913 	 Supervised Cost: 2.2627 	 Unsupervised Cost: 62.1286 	 Validation Accuracy: 0.15
307 | Epoch: 15 	 Total Cost: 64.2598 	 Supervised Cost: 2.2626 	 Unsupervised Cost: 61.9972 	 Validation Accuracy: 0.1492
308 | Epoch: 15 	 Total Cost: 68.8505 	 Supervised Cost: 2.2620 	 Unsupervised Cost: 66.5885 	 Validation Accuracy: 0.1537
309 | Epoch: 15 	 Total Cost: 73.6140 	 Supervised Cost: 2.2623 	 Unsupervised Cost: 71.3517 	 Validation Accuracy: 0.1695
310 | Epoch: 15 	 Total Cost: 75.8000 	 Supervised Cost: 2.2628 	 Unsupervised Cost: 73.5372 	 Validation Accuracy: 0.168
311 | Epoch: 15 	 Total Cost: 75.2754 	 Supervised Cost: 2.2633 	 Unsupervised Cost: 73.0121 	 Validation Accuracy: 0.1633
312 | Epoch: 15 	 Total Cost: 73.7759 	 Supervised Cost: 2.2636 	 Unsupervised Cost: 71.5123 	 Validation Accuracy: 0.1713
313 | Epoch: 15 	 Total Cost: 70.9970 	 Supervised Cost: 2.2635 	 Unsupervised Cost: 68.7334 	 Validation Accuracy: 0.1756
314 | Epoch: 15 	 Total Cost: 69.5395 	 Supervised Cost: 2.2632 	 Unsupervised Cost: 67.2763 	 Validation Accuracy: 0.1804
315 | Epoch: 15 	 Total Cost: 69.4039 	 Supervised Cost: 2.2634 	 Unsupervised Cost: 67.1405 	 Validation Accuracy: 0.1748
316 | Epoch: 15 	 Total Cost: 70.7192 	 Supervised Cost: 2.2636 	 Unsupervised Cost: 68.4556 	 Validation Accuracy: 0.1772
317 | Epoch: 15 	 Total Cost: 70.4895 	 Supervised Cost: 2.2637 	 Unsupervised Cost: 68.2258 	 Validation Accuracy: 0.1738
318 | Epoch: 16 	 Total Cost: 67.1379 	 Supervised Cost: 2.2678 	 Unsupervised Cost: 64.8701 	 Validation Accuracy: 0.1739
319 | Epoch: 16 	 Total Cost: 59.0976 	 Supervised Cost: 2.2694 	 Unsupervised Cost: 56.8282 	 Validation Accuracy: 0.1709
320 | Epoch: 16 	 Total Cost: 56.2306 	 Supervised Cost: 2.2721 	 Unsupervised Cost: 53.9585 	 Validation Accuracy: 0.1593
321 | Epoch: 16 	 Total Cost: 53.4383 	 Supervised Cost: 2.2737 	 Unsupervised Cost: 51.1647 	 Validation Accuracy: 0.1606
322 | Epoch: 16 	 Total Cost: 53.6414 	 Supervised Cost: 2.2733 	 Unsupervised Cost: 51.3681 	 Validation Accuracy: 0.168
323 | Epoch: 16 	 Total Cost: 53.1597 	 Supervised Cost: 2.2723 	 Unsupervised Cost: 50.8874 	 Validation Accuracy: 0.1791
324 | Epoch: 16 	 Total Cost: 51.9200 	 Supervised Cost: 2.2716 	 Unsupervised Cost: 49.6484 	 Validation Accuracy: 0.1848
325 | Epoch: 16 	 Total Cost: 50.3218 	 Supervised Cost: 2.2711 	 Unsupervised Cost: 48.0507 	 Validation Accuracy: 0.1836
326 | Epoch: 16 	 Total Cost: 50.3889 	 Supervised Cost: 2.2705 	 Unsupervised Cost: 48.1184 	 Validation Accuracy: 0.1847
327 | Epoch: 16 	 Total Cost: 50.5346 	 Supervised Cost: 2.2695 	 Unsupervised Cost: 48.2651 	 Validation Accuracy: 0.2005
328 | Epoch: 16 	 Total Cost: 51.0243 	 Supervised Cost: 2.2688 	 Unsupervised Cost: 48.7554 	 Validation Accuracy: 0.2022
329 | Epoch: 16 	 Total Cost: 51.3565 	 Supervised Cost: 2.2681 	 Unsupervised Cost: 49.0884 	 Validation Accuracy: 0.202
330 | Epoch: 16 	 Total Cost: 51.1049 	 Supervised Cost: 2.2674 	 Unsupervised Cost: 48.8375 	 Validation Accuracy: 0.2051
331 | Epoch: 16 	 Total Cost: 50.6423 	 Supervised Cost: 2.2666 	 Unsupervised Cost: 48.3758 	 Validation Accuracy: 0.2119
332 | Epoch: 16 	 Total Cost: 50.2281 	 Supervised Cost: 2.2657 	 Unsupervised Cost: 47.9624 	 Validation Accuracy: 0.2168
333 | Epoch: 17 	 Total Cost: 38.7718 	 Supervised Cost: 2.2524 	 Unsupervised Cost: 36.5194 	 Validation Accuracy: 0.2061
334 | Epoch: 17 	 Total Cost: 44.7239 	 Supervised Cost: 2.2502 	 Unsupervised Cost: 42.4737 	 Validation Accuracy: 0.2004
335 | Epoch: 17 	 Total Cost: 46.3951 	 Supervised Cost: 2.2496 	 Unsupervised Cost: 44.1455 	 Validation Accuracy: 0.2008
336 | Epoch: 17 	 Total Cost: 46.7857 	 Supervised Cost: 2.2492 	 Unsupervised Cost: 44.5365 	 Validation Accuracy: 0.2031
337 | Epoch: 17 	 Total Cost: 47.8203 	 Supervised Cost: 2.2493 	 Unsupervised Cost: 45.5711 	 Validation Accuracy: 0.2053
338 | Epoch: 17 	 Total Cost: 47.0747 	 Supervised Cost: 2.2492 	 Unsupervised Cost: 44.8255 	 Validation Accuracy: 0.1885
339 | Epoch: 17 	 Total Cost: 47.6868 	 Supervised Cost: 2.2487 	 Unsupervised Cost: 45.4381 	 Validation Accuracy: 0.1914
340 | Epoch: 17 	 Total Cost: 47.0504 	 Supervised Cost: 2.2485 	 Unsupervised Cost: 44.8019 	 Validation Accuracy: 0.1984
341 | Epoch: 17 	 Total Cost: 47.7598 	 Supervised Cost: 2.2484 	 Unsupervised Cost: 45.5114 	 Validation Accuracy: 0.1998
342 | Epoch: 17 	 Total Cost: 47.7522 	 Supervised Cost: 2.2480 	 Unsupervised Cost: 45.5042 	 Validation Accuracy: 0.2027
343 | Epoch: 17 	 Total Cost: 48.3329 	 Supervised Cost: 2.2479 	 Unsupervised Cost: 46.0850 	 Validation Accuracy: 0.2084
344 | Epoch: 17 	 Total Cost: 47.9188 	 Supervised Cost: 2.2471 	 Unsupervised Cost: 45.6717 	 Validation Accuracy: 0.2089
345 | Epoch: 17 	 Total Cost: 47.7605 	 Supervised Cost: 2.2466 	 Unsupervised Cost: 45.5140 	 Validation Accuracy: 0.214
346 | Epoch: 17 	 Total Cost: 47.8824 	 Supervised Cost: 2.2460 	 Unsupervised Cost: 45.6364 	 Validation Accuracy: 0.2122
347 | Epoch: 17 	 Total Cost: 47.3258 	 Supervised Cost: 2.2455 	 Unsupervised Cost: 45.0802 	 Validation Accuracy: 0.2165
348 | Epoch: 18 	 Total Cost: 42.5688 	 Supervised Cost: 2.2331 	 Unsupervised Cost: 40.3357 	 Validation Accuracy: 0.2162
349 | Epoch: 18 	 Total Cost: 41.4103 	 Supervised Cost: 2.2334 	 Unsupervised Cost: 39.1770 	 Validation Accuracy: 0.217
350 | Epoch: 18 	 Total Cost: 40.5049 	 Supervised Cost: 2.2325 	 Unsupervised Cost: 38.2724 	 Validation Accuracy: 0.2219
351 | Epoch: 18 	 Total Cost: 40.6927 	 Supervised Cost: 2.2330 	 Unsupervised Cost: 38.4596 	 Validation Accuracy: 0.226
352 | Epoch: 18 	 Total Cost: 40.0919 	 Supervised Cost: 2.2324 	 Unsupervised Cost: 37.8594 	 Validation Accuracy: 0.2225
353 | Epoch: 18 	 Total Cost: 43.7068 	 Supervised Cost: 2.2316 	 Unsupervised Cost: 41.4752 	 Validation Accuracy: 0.224
354 | Epoch: 18 	 Total Cost: 45.7846 	 Supervised Cost: 2.2310 	 Unsupervised Cost: 43.5537 	 Validation Accuracy: 0.2361
355 | Epoch: 18 	 Total Cost: 48.7456 	 Supervised Cost: 2.2296 	 Unsupervised Cost: 46.5160 	 Validation Accuracy: 0.2411
356 | Epoch: 18 	 Total Cost: 50.7264 	 Supervised Cost: 2.2292 	 Unsupervised Cost: 48.4972 	 Validation Accuracy: 0.2068
357 | Epoch: 18 	 Total Cost: 51.8030 	 Supervised Cost: 2.2298 	 Unsupervised Cost: 49.5732 	 Validation Accuracy: 0.1928
358 | Epoch: 18 	 Total Cost: 52.2031 	 Supervised Cost: 2.2313 	 Unsupervised Cost: 49.9718 	 Validation Accuracy: 0.1871
359 | Epoch: 18 	 Total Cost: 51.9626 	 Supervised Cost: 2.2322 	 Unsupervised Cost: 49.7304 	 Validation Accuracy: 0.1884
360 | Epoch: 18 	 Total Cost: 52.7679 	 Supervised Cost: 2.2329 	 Unsupervised Cost: 50.5351 	 Validation Accuracy: 0.1871
361 | Epoch: 18 	 Total Cost: 52.7767 	 Supervised Cost: 2.2334 	 Unsupervised Cost: 50.5433 	 Validation Accuracy: 0.1931
362 | Epoch: 18 	 Total Cost: 53.4755 	 Supervised Cost: 2.2335 	 Unsupervised Cost: 51.2419 	 Validation Accuracy: 0.1946
363 | Epoch: 19 	 Total Cost: 56.4926 	 Supervised Cost: 2.2217 	 Unsupervised Cost: 54.2709 	 Validation Accuracy: 0.2304
364 | Epoch: 19 	 Total Cost: 54.7131 	 Supervised Cost: 2.2187 	 Unsupervised Cost: 52.4944 	 Validation Accuracy: 0.2418
365 | Epoch: 19 	 Total Cost: 52.3653 	 Supervised Cost: 2.2150 	 Unsupervised Cost: 50.1503 	 Validation Accuracy: 0.2508
366 | Epoch: 19 	 Total Cost: 49.0046 	 Supervised Cost: 2.2118 	 Unsupervised Cost: 46.7929 	 Validation Accuracy: 0.2568
367 | Epoch: 19 	 Total Cost: 47.6941 	 Supervised Cost: 2.2082 	 Unsupervised Cost: 45.4859 	 Validation Accuracy: 0.2618
368 | Epoch: 19 	 Total Cost: 47.6684 	 Supervised Cost: 2.2042 	 Unsupervised Cost: 45.4642 	 Validation Accuracy: 0.267
369 | Epoch: 19 	 Total Cost: 47.2864 	 Supervised Cost: 2.2005 	 Unsupervised Cost: 45.0859 	 Validation Accuracy: 0.2709
370 | Epoch: 19 	 Total Cost: 48.0051 	 Supervised Cost: 2.1972 	 Unsupervised Cost: 45.8080 	 Validation Accuracy: 0.2783
371 | Epoch: 19 	 Total Cost: 47.8645 	 Supervised Cost: 2.1938 	 Unsupervised Cost: 45.6707 	 Validation Accuracy: 0.2846
372 | Epoch: 19 	 Total Cost: 47.9490 	 Supervised Cost: 2.1903 	 Unsupervised Cost: 45.7588 	 Validation Accuracy: 0.2856
373 | Epoch: 19 	 Total Cost: 48.5783 	 Supervised Cost: 2.1874 	 Unsupervised Cost: 46.3909 	 Validation Accuracy: 0.2871
374 | Epoch: 19 	 Total Cost: 49.2751 	 Supervised Cost: 2.1848 	 Unsupervised Cost: 47.0903 	 Validation Accuracy: 0.2944
375 | Epoch: 19 	 Total Cost: 50.2333 	 Supervised Cost: 2.1823 	 Unsupervised Cost: 48.0510 	 Validation Accuracy: 0.3105
376 | Epoch: 19 	 Total Cost: 49.9106 	 Supervised Cost: 2.1799 	 Unsupervised Cost: 47.7307 	 Validation Accuracy: 0.332
377 | Epoch: 19 	 Total Cost: 50.1634 	 Supervised Cost: 2.1776 	 Unsupervised Cost: 47.9858 	 Validation Accuracy: 0.3679
378 | Epoch: 20 	 Total Cost: 50.1488 	 Supervised Cost: 2.1205 	 Unsupervised Cost: 48.0284 	 Validation Accuracy: 0.3971
379 | Epoch: 20 	 Total Cost: 50.1923 	 Supervised Cost: 2.1164 	 Unsupervised Cost: 48.0759 	 Validation Accuracy: 0.4054
380 | Epoch: 20 	 Total Cost: 48.7265 	 Supervised Cost: 2.1126 	 Unsupervised Cost: 46.6139 	 Validation Accuracy: 0.4032
381 | Epoch: 20 	 Total Cost: 51.8325 	 Supervised Cost: 2.1099 	 Unsupervised Cost: 49.7225 	 Validation Accuracy: 0.4143
382 | Epoch: 20 	 Total Cost: 51.7776 	 Supervised Cost: 2.1054 	 Unsupervised Cost: 49.6722 	 Validation Accuracy: 0.4198
383 | Epoch: 20 	 Total Cost: 52.3343 	 Supervised Cost: 2.1013 	 Unsupervised Cost: 50.2330 	 Validation Accuracy: 0.4308
384 | Epoch: 20 	 Total Cost: 51.0337 	 Supervised Cost: 2.0970 	 Unsupervised Cost: 48.9367 	 Validation Accuracy: 0.4481
385 | Epoch: 20 	 Total Cost: 53.6136 	 Supervised Cost: 2.0926 	 Unsupervised Cost: 51.5210 	 Validation Accuracy: 0.4583
386 | Epoch: 20 	 Total Cost: 55.1519 	 Supervised Cost: 2.0892 	 Unsupervised Cost: 53.0628 	 Validation Accuracy: 0.4635
387 | Epoch: 20 	 Total Cost: 54.6093 	 Supervised Cost: 2.0860 	 Unsupervised Cost: 52.5233 	 Validation Accuracy: 0.4857
388 | Epoch: 20 	 Total Cost: 53.8451 	 Supervised Cost: 2.0828 	 Unsupervised Cost: 51.7623 	 Validation Accuracy: 0.4827
389 | Epoch: 20 	 Total Cost: 52.9703 	 Supervised Cost: 2.0802 	 Unsupervised Cost: 50.8901 	 Validation Accuracy: 0.4944
390 | Epoch: 20 	 Total Cost: 52.1631 	 Supervised Cost: 2.0773 	 Unsupervised Cost: 50.0858 	 Validation Accuracy: 0.5067
391 | Epoch: 20 	 Total Cost: 52.0492 	 Supervised Cost: 2.0747 	 Unsupervised Cost: 49.9745 	 Validation Accuracy: 0.5104
392 | Epoch: 20 	 Total Cost: 51.3769 	 Supervised Cost: 2.0722 	 Unsupervised Cost: 49.3047 	 Validation Accuracy: 0.5148
393 | =====================
394 | 
395 | Done :)
396 | 


--------------------------------------------------------------------------------
/stacked_denoising_autoencoder/autoencoder.py:
--------------------------------------------------------------------------------
 1 | from  __future__ import print_function
 2 | 
 3 | import numpy as np
 4 | 
 5 | import torch
 6 | from torch.autograd import Variable
 7 | from torch.nn.parameter import Parameter
 8 | 
 9 | 
10 | class Autoencoder(torch.nn.Module):
11 |     """Denoising Autoencoder
12 |     Encodes the data preferably in a lower dimension.
13 |     During training the input is encoded and then reconstructed using the latent embedding.
14 |     The matrix used for reconstruction is the transpose of matrix used for encoding.
15 | 
16 |     For building a Denoising Autoencoder noise has to be added to the training data
17 |     before it is passed through the network. 'corrupt' method allows you to
18 |     add noise to the data.
19 | 
20 |     reference: http://www.deeplearning.net/tutorial/dA.html#da
21 |     """
22 | 
23 |     # TODO: Adapt autoencoder for DataLoader.
24 | 
25 |     def __init__(self, d_in, d_hidden, batch_size, corruption=0.2):
26 |         super(Autoencoder, self).__init__()
27 |         self.d_in = d_in
28 |         self.d_hidden = d_hidden
29 |         self.batch_size = batch_size
30 |         self.corruption = corruption
31 |         self.W = Parameter(torch.FloatTensor(d_in, d_hidden), requires_grad=True)
32 |         self.W.data.uniform_(-4. * np.sqrt(6. / (d_hidden + d_in)),
33 |                              4. * np.sqrt(6. / (d_hidden + d_in)))
34 |         self.b = Parameter(torch.zeros(1, d_hidden), requires_grad=True)
35 |         self.sigmoid1 = torch.nn.Sigmoid()
36 |         self.b_prime = Parameter(torch.zeros(1, d_in), requires_grad=True)
37 |         self.sigmoid2 = torch.nn.Sigmoid()
38 | 
39 |     def corrupt(self, x):
40 |         noise = torch.FloatTensor(np.random.binomial(1, 1.0 - self.corruption, size=x.data.size()))
41 |         result = x.clone()
42 |         result.data *= noise
43 |         return result
44 | 
45 |     def encode(self, x, add_noise=False):
46 |         if add_noise:
47 |             tilde_x = self.corrupt(x)
48 |         else:
49 |             tilde_x = x.clone()
50 |         ones = Parameter(torch.ones(self.batch_size, 1))
51 |         t = tilde_x.mm(self.W)
52 |         t = t + ones.mm(self.b)
53 |         t = self.sigmoid1.forward(t)
54 |         return t
55 | 
56 |     def decode(self, x):
57 |         ones = Parameter(torch.ones(self.batch_size, 1))
58 |         t = x.mm(self.W.transpose(1, 0)) + ones.mm(self.b_prime)
59 |         t = self.sigmoid2.forward(t)
60 |         return t
61 | 
62 |     def forward(self, x):
63 |         t = self.encode(x)
64 |         return t
65 | 
66 |     def train_ae(self, train_X, optimizer, epochs, verbose=True):
67 |         N = train_X.data.size()[0]
68 |         num_batches = N / self.batch_size
69 |         for e in range(epochs):
70 |             agg_cost = 0.
71 |             for k in range(num_batches):
72 |                 start, end = k * self.batch_size, (k + 1) * self.batch_size
73 |                 bX = train_X[start:end]
74 |                 optimizer.zero_grad()
75 |                 Z = self.forward(bX)
76 |                 Z = self.decode(Z)
77 |                 loss = -torch.sum(bX * torch.log(Z) + (1.0 - bX) * torch.log(1.0 - Z), 1)
78 |                 cost = torch.mean(loss)
79 |                 cost.backward()
80 |                 optimizer.step()
81 |                 agg_cost += cost
82 |             agg_cost /= num_batches
83 |             if verbose:
84 |                 print("Epoch:", e, "cost:", agg_cost.data[0])
85 | 
86 | 
87 | def main():
88 |     pass
89 | 
90 | 
91 | if __name__ == "__main__":
92 |     main()
93 | 


--------------------------------------------------------------------------------
/stacked_denoising_autoencoder/sda.py:
--------------------------------------------------------------------------------
  1 | from  __future__ import print_function
  2 | 
  3 | import sys
  4 | sys.path.append("/Users/abhishekkadian/Documents/Github/jaa-dl/assignment-1/")
  5 | 
  6 | import numpy as np
  7 | 
  8 | import torch
  9 | from torch.autograd import Variable
 10 | from torch.optim import SGD
 11 | 
 12 | import autoencoder as ae
 13 | import convnet
 14 | 
 15 | 
 16 | class SDA(torch.nn.Module):
 17 |     """Stacked Denoising Autoencoder
 18 | 
 19 |     reference: http://www.deeplearning.net/tutorial/SdA.html,
 20 |                http://www.jmlr.org/papers/volume11/vincent10a/vincent10a.pdf
 21 |     """
 22 |     def __init__(self, d_input, d_hidden_autoencoders, d_out,
 23 |                  corruptions, batch_size, pre_lr=0.001, ft_lr=0.1):
 24 |         super(SDA, self).__init__()
 25 |         self.d_input = d_input
 26 |         self.d_hidden_autoencoders = list(d_hidden_autoencoders)
 27 |         self.d_out = d_out
 28 |         self.corruptions = corruptions
 29 |         self.batch_size = batch_size
 30 |         self.pre_lr = pre_lr
 31 |         self.ft_lr = ft_lr
 32 | 
 33 |         # Create one sequential module containing all autoencoders and logistic layer
 34 |         self.sequential = torch.nn.Sequential()
 35 | 
 36 |         # Create the Autoencoders
 37 |         self.autoencoders_ref = []
 38 |         for i, (d, c) in enumerate(zip(d_hidden_autoencoders, corruptions)):
 39 |             if i == 0:
 40 |                 curr_input = d_input
 41 |             else:
 42 |                 curr_input = d_hidden_autoencoders[i - 1]
 43 |             dna = ae.Autoencoder(curr_input, d, batch_size, corruption=c)
 44 |             self.autoencoders_ref.append("autoencoder_" + str(i))
 45 |             self.sequential.add_module(self.autoencoders_ref[-1], dna)
 46 | 
 47 |         # Create the Logistic Layer
 48 |         self.sequential.add_module("top_linear1", torch.nn.Linear(d_hidden_autoencoders[-1], d_out, bias=True))
 49 |         self.sequential.top_linear1 = torch.nn.Linear(d_hidden_autoencoders[-1], d_out, bias=True)
 50 |         self.sequential.top_linear1.weight.data = torch.zeros(self.sequential.top_linear1.weight.data.size())
 51 |         self.sequential.top_linear1.bias.data = torch.zeros(d_out)
 52 |         self.sequential.add_module("softmax", torch.nn.LogSoftmax())
 53 | 
 54 |     def pretrain(self, x, pt_epochs, verbose=True):
 55 |         n = x.data.size()[0]
 56 |         num_batches = n / self.batch_size
 57 |         t = x
 58 | 
 59 |         # Pre-train 1 autoencoder at a time
 60 |         for i, ae_re in enumerate(self.autoencoders_ref):
 61 |             # Get the current autoencoder
 62 |             ae = getattr(self.sequential, ae_re)
 63 | 
 64 |             # Getting encoded output from the previous autoencoder
 65 |             if i > 0:
 66 |                 # Set the requires_grad to False so that backprop doesn't
 67 |                 # travel all the way back to the previous autoencoder
 68 |                 temp = Variable(torch.FloatTensor(n, ae.d_in), requires_grad=False)
 69 |                 for k in range(num_batches):
 70 |                     start, end = k * self.batch_size, (k + 1) * self.batch_size
 71 |                     prev_ae = getattr(self.sequential, self.autoencoders_ref[i - 1])
 72 |                     temp.data[start:end] = prev_ae.encode(t[start:end], add_noise=False).data
 73 |                 t = temp
 74 |             optimizer = SGD(ae.parameters(), lr=self.pre_lr)
 75 | 
 76 |             # Pre-training
 77 |             print("Pre-training Autoencoder:", i)
 78 |             for ep in range(pt_epochs):
 79 |                 agg_cost = 0.
 80 |                 for k in range(num_batches):
 81 |                     start, end = k * self.batch_size, (k + 1) * self.batch_size
 82 |                     bt = t[start:end]
 83 |                     optimizer.zero_grad()
 84 |                     z = ae.encode(bt, add_noise=True)
 85 |                     z = ae.decode(z)
 86 |                     loss = -torch.sum(bt * torch.log(z) + (1.0 - bt) * torch.log(1.0 - z), 1)
 87 |                     cost = torch.mean(loss)
 88 |                     cost.backward()
 89 |                     optimizer.step()
 90 |                     agg_cost += cost
 91 |                 agg_cost /= num_batches
 92 |                 if verbose:
 93 |                     print("Pre-training Autoencoder:", i, "Epoch:", ep, "Cost:", agg_cost.data[0])
 94 | 
 95 |     def forward(self, x):
 96 |         t = self.sequential.forward(x)
 97 |         return t
 98 | 
 99 |     def finetune(self, train_X, train_y, valid_X, valid_y,
100 |                  valid_actual_size, ft_epochs, verbose=True):
101 |         n = train_X.data.size()[0]
102 |         num_batches = n / self.batch_size
103 |         n_v = valid_X.data.size()[0]
104 |         num_batches_v = n_v / self.batch_size
105 |         optimizer = SGD(self.parameters(), lr=self.ft_lr)
106 |         loss = torch.nn.NLLLoss()
107 | 
108 |         for ef in range(ft_epochs):
109 |             agg_cost = 0
110 |             for k in range(num_batches):
111 |                 start, end = k * self.batch_size, (k + 1) * self.batch_size
112 |                 bX = train_X[start:end]
113 |                 by = train_y[start:end]
114 |                 optimizer.zero_grad()
115 |                 p = self.forward(bX)
116 |                 cost = loss.forward(p, by)
117 |                 agg_cost += cost
118 |                 cost.backward()
119 |                 optimizer.step()
120 |             agg_cost /= num_batches
121 |             preds = np.zeros((n_v, self.d_out))
122 | 
123 |             # Calculate accuracy on Validation set
124 |             for k in range(num_batches_v):
125 |                 start, end = k * self.batch_size, (k + 1) * self.batch_size
126 |                 bX = valid_X[start:end]
127 |                 p = self.forward(bX).data.numpy()
128 |                 preds[start:end] = p
129 |             correct = 0
130 |             for actual, prediction in zip(valid_y[:valid_actual_size], preds[:valid_actual_size]):
131 |                 ind = np.argmax(prediction)
132 |                 actual = actual.data.numpy()
133 |                 if ind == actual:
134 |                     correct += 1
135 | 
136 |             if verbose:
137 |                 print("Fine-tuning Epoch:", ef, "Cost:", agg_cost.data[0],
138 |                       "Validation Accuracy:", "{0:.4f}".format(correct / float(valid_actual_size)))
139 | 
140 | 
141 | def main():
142 |     # Load data
143 |     trX, teX, trY, teY = convnet.load_mnist(onehot=False)
144 |     trX = np.array([x.flatten() for x in trX])
145 |     teX = np.array([x.flatten() for x in teX])
146 |     trX = Variable(torch.from_numpy(trX).float())
147 |     teX = Variable(torch.from_numpy(teX).float())
148 |     trY = Variable(torch.from_numpy(trY).long())
149 |     teY = Variable(torch.from_numpy(teY).long())
150 | 
151 |     batch_size = 64
152 | 
153 |     # Pad the validation set
154 |     actual_size = teX.size()[0]
155 |     padded_size = (actual_size / batch_size + 1) * batch_size
156 |     teX_padded = Variable(torch.FloatTensor(padded_size, teX.size()[1]))
157 |     teY_padded = Variable(torch.LongTensor(padded_size) * 0)
158 |     teX_padded[:actual_size] = teX
159 |     teY_padded[:actual_size] = teY
160 | 
161 |     sda = SDA(d_input=784,
162 |               d_hidden_autoencoders=[1000, 1000, 1000],
163 |               d_out=10,
164 |               corruptions=[.1, .2, .3],
165 |               batch_size=batch_size)
166 | 
167 |     sda.pretrain(trX, pt_epochs=15)
168 | 
169 |     sda.finetune(trX, trY, teX_padded, teY_padded,
170 |                  valid_actual_size=actual_size, ft_epochs=36)
171 | 
172 | 
173 | if __name__ == "__main__":
174 |     main()


--------------------------------------------------------------------------------
/utils/mnist_data.py:
--------------------------------------------------------------------------------
  1 | from __future__ import print_function
  2 | import os
  3 | import urllib
  4 | import random
  5 | import numpy as np
  6 | import gzip
  7 | from collections import defaultdict
  8 | import pickle
  9 | import argparse
 10 | 
 11 | 
 12 | def get_data(filename, directory,
 13 |              data_url="http://yann.lecun.com/exdb/mnist/",
 14 |              verbose=True):
 15 |     if not os.path.exists(directory):
 16 |         os.mkdir(directory)
 17 |     filepath = os.path.join(directory, filename)
 18 |     if not os.path.exists(filepath):
 19 |         filepath, _ = urllib.urlretrieve(data_url + filename, filepath)
 20 |         statinfo = os.stat(filepath)
 21 |         if verbose:
 22 |             print('Succesfully downloaded', filename, statinfo.st_size, 'bytes.')
 23 |     return filepath
 24 | 
 25 | 
 26 | def _read32(bytestream):
 27 |     dt = np.dtype(np.uint32).newbyteorder('>')
 28 |     return np.frombuffer(bytestream.read(4), dtype=dt)[0]
 29 | 
 30 | 
 31 | def extract_images(filename, verbose=True):
 32 |     """Extract the images into a 4D uint8 numpy array [index, y, x, depth]."""
 33 |     if verbose:
 34 |         print('Extracting', filename)
 35 |     with gzip.open(filename) as bytestream:
 36 |         magic = _read32(bytestream)
 37 |         if magic != 2051:
 38 |             raise ValueError(
 39 |               'Invalid magic number %d in MNIST image file: %s' %
 40 |               (magic, filename))
 41 |         num_images = _read32(bytestream)
 42 |         rows = _read32(bytestream)
 43 |         cols = _read32(bytestream)
 44 |         buf = bytestream.read(rows * cols * num_images)
 45 |         data = np.frombuffer(buf, dtype=np.uint8)
 46 |         data = data.reshape(num_images, rows, cols, 1)
 47 |         return data
 48 | 
 49 | 
 50 | def extract_labels(filename, verbose=True):
 51 |     """Extract the labels into a 1D uint8 numpy array [index]."""
 52 |     if verbose:
 53 |         print('Extracting', filename)
 54 |     with gzip.open(filename) as bytestream:
 55 |         magic = _read32(bytestream)
 56 |         if magic != 2049:
 57 |             raise ValueError(
 58 |               'Invalid magic number %d in MNIST label file: %s' %
 59 |               (magic, filename))
 60 |         num_items = _read32(bytestream)
 61 |         buf = bytestream.read(num_items)
 62 |         labels = np.frombuffer(buf, dtype=np.uint8)
 63 |         return labels
 64 | 
 65 | 
 66 | def shuffle_images_labels(images, labels):
 67 |     assert images.shape[0] == labels.shape[0]
 68 |     randomize = np.arange(images.shape[0])
 69 |     np.random.shuffle(randomize)
 70 |     return images[randomize], labels[randomize]
 71 | 
 72 | 
 73 | def dump_pickle(filepath, d):
 74 |     with open(filepath, "wb") as f:
 75 |         pickle.dump(d, f)
 76 | 
 77 | 
 78 | def main():
 79 |     # command line arguments
 80 |     parser = argparse.ArgumentParser(description="Parser for MNIST data generation")
 81 |     parser.add_argument("--num_labelled", type=int, default=100)
 82 |     args = parser.parse_args()
 83 | 
 84 |     n_labelled = args.num_labelled
 85 |     random.seed(42)
 86 |     np.random.seed(42)
 87 |     data_dir = "data/"
 88 |     mnist_train_images_gz = 'train-images-idx3-ubyte.gz'
 89 |     mnist_train_labels_gz = 'train-labels-idx1-ubyte.gz'
 90 |     mnist_test_images_gz = 't10k-images-idx3-ubyte.gz'
 91 |     mnist_test_labels_gz = 't10k-labels-idx1-ubyte.gz'
 92 | 
 93 |     mnist_train_images = get_data(mnist_train_images_gz, data_dir)
 94 |     mnist_train_images = extract_images(mnist_train_images)
 95 |     mnist_train_labels = get_data(mnist_train_labels_gz, data_dir)
 96 |     mnist_train_labels = extract_labels(mnist_train_labels)
 97 |     mnist_test_images = get_data(mnist_test_images_gz, data_dir)
 98 |     mnist_test_images = extract_images(mnist_test_images)
 99 |     mnist_test_labels = get_data(mnist_test_labels_gz, data_dir)
100 |     mnist_test_labels = extract_labels(mnist_test_labels)
101 | 
102 |     train_data_shuffle = [(x, y) for x, y in zip(mnist_train_images, mnist_train_labels)]
103 |     random.shuffle(train_data_shuffle)
104 |     mnist_shuffled_train_images = np.array([x[0] for x in train_data_shuffle])
105 |     mnist_shuffled_train_labels = np.array([x[1] for x in train_data_shuffle])
106 | 
107 |     validation_size = 10000
108 |     train_size = mnist_train_images.shape[0] - validation_size
109 | 
110 |     train_images = mnist_shuffled_train_images[:train_size].copy()
111 |     train_labels = mnist_shuffled_train_labels[:train_size].copy()
112 | 
113 |     validation_images = mnist_shuffled_train_images[train_size:].copy()
114 |     validation_labels = mnist_shuffled_train_labels[train_size:].copy()
115 | 
116 |     test_images = mnist_test_images
117 |     test_labels = mnist_test_labels
118 | 
119 |     train_data_label_buckets = defaultdict(list)
120 | 
121 |     for image, label in zip(train_images, train_labels):
122 |         train_data_label_buckets[label].append((image, label))
123 | 
124 |     num_labels = len(train_data_label_buckets)
125 | 
126 |     train_labelled_data_images = []
127 |     train_labelled_data_labels = []
128 |     train_unlabelled_data_images = []
129 |     train_unlabelled_data_labels = []
130 | 
131 |     for label, label_data in train_data_label_buckets.items():
132 |         count = n_labelled / num_labels
133 |         for v in label_data[:count]:
134 |             train_labelled_data_images.append(v[0])
135 |             train_labelled_data_labels.append(v[1])
136 |         for v in label_data[count:]:
137 |             train_unlabelled_data_images.append(v[0])
138 |             # dummy label
139 |             train_unlabelled_data_labels.append(-1)
140 | 
141 |     train_labelled_images = np.array(train_labelled_data_images)
142 |     train_labelled_labels = np.array(train_labelled_data_labels)
143 | 
144 |     train_unlabelled_images = np.array(train_unlabelled_data_images)
145 |     train_unlabelled_labels = np.array(train_unlabelled_data_labels)
146 | 
147 |     train_labelled_images = train_labelled_images[:, :, :, 0]
148 |     train_unlabelled_images = train_unlabelled_images[:, :, :, 0]
149 |     validation_images = validation_images[:, :, :, 0]
150 |     test_images = test_images[:, :, :, 0]
151 | 
152 |     train_labelled_images, train_labelled_labels = shuffle_images_labels(train_labelled_images, train_labelled_labels)
153 | 
154 |     # normalizing
155 |     train_labelled_images = np.multiply(train_labelled_images, 1./255.)
156 |     train_unlabelled_images = np.multiply(train_unlabelled_images, 1./255.)
157 |     validation_images = np.multiply(validation_images, 1./255.)
158 |     test_images = np.multiply(test_images, 1./255,)
159 | 
160 |     print("=" * 50)
161 |     print("train_labelled_images shape:", train_labelled_images.shape)
162 |     print("train_labelled_labels shape:", train_labelled_labels.shape)
163 |     print()
164 |     print("train_unlabelled_images shape:", train_unlabelled_images.shape)
165 |     print("train_unlabelled_labels shape:", train_unlabelled_labels.shape)
166 |     print()
167 |     print("validation_images shape:", validation_images.shape)
168 |     print("validation_labels shape:", validation_labels.shape)
169 |     print()
170 |     print("test_images shape:", test_images.shape)
171 |     print("test_labels shape:", test_labels.shape)
172 |     print("=" * 50)
173 | 
174 |     print("Dumping pickles")
175 | 
176 |     dump_pickle(data_dir + "train_labelled_images.p", train_labelled_images)
177 |     dump_pickle(data_dir + "train_labelled_labels.p", train_labelled_labels)
178 |     dump_pickle(data_dir + "train_unlabelled_images.p", train_unlabelled_images)
179 |     dump_pickle(data_dir + "train_unlabelled_labels.p", train_unlabelled_labels)
180 |     dump_pickle(data_dir + "validation_images.p", validation_images)
181 |     dump_pickle(data_dir + "validation_labels.p", validation_labels)
182 |     dump_pickle(data_dir + "test_images.p", test_images)
183 |     dump_pickle(data_dir + "test_labels.p", test_labels)
184 | 
185 |     print("MNIST dataset successfully created")
186 | 
187 | 
188 | if __name__ == "__main__":
189 |     main()


--------------------------------------------------------------------------------