├── source
    ├── system
    │   ├── Uncopyable.h
    │   ├── ComputeProgram.h
    │   ├── ComputeSystem.h
    │   ├── ComputeProgram.cpp
    │   └── ComputeSystem.cpp
    ├── HTFE.i
    ├── setup.py
    ├── example.py
    ├── benchmark1.py
    ├── htfe
    │   ├── HTFE.h
    │   └── HTFE.cpp
    └── htfe.py
├── LICENSE.md
├── README.md
└── resources
    └── htfe.cl


/source/system/Uncopyable.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | // Inherit from this class to make the class uncopyable
 4 | namespace sys {
 5 | 	class Uncopyable {
 6 | 	protected:
 7 | 		Uncopyable() {}
 8 | 		virtual ~Uncopyable() {}
 9 | 	private:
10 | 		Uncopyable(const Uncopyable &);
11 | 		Uncopyable &operator=(const Uncopyable &);
12 | 	};
13 | }


--------------------------------------------------------------------------------
/source/system/ComputeProgram.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "ComputeSystem.h"
 4 | 
 5 | #include <assert.h>
 6 | 
 7 | namespace sys {
 8 | 	class ComputeProgram {
 9 | 	private:
10 | 		cl::Program _program;
11 | 
12 | 	public:
13 | 		bool loadFromFile(const std::string &name, ComputeSystem &cs);
14 | 
15 | 		cl::Program &getProgram() {
16 | 			return _program;
17 | 		}
18 | 	};
19 | }


--------------------------------------------------------------------------------
/source/HTFE.i:
--------------------------------------------------------------------------------
 1 | %begin %{
 2 | #include <cmath>
 3 | #include <iostream>
 4 | %}
 5 | %module htfe
 6 | 
 7 | %{
 8 | #include "htfe/HTFE.h"
 9 | %}
10 | 
11 | %include "std_string.i"
12 | %include "std_vector.i"
13 | 
14 | namespace std {
15 |    %template(vectorld) vector<htfe::LayerDesc>;
16 | };
17 | 
18 | %include "htfe/HTFE.h"
19 | %include "system/ComputeSystem.h"
20 | %include "system/ComputeProgram.h"


--------------------------------------------------------------------------------
/source/setup.py:
--------------------------------------------------------------------------------
1 | from distutils.core import setup, Extension
2 | 
3 | # Change these to point to the right locations
4 | clIncludeDir = "C:/Program Files (x86)/AMD APP SDK/3.0-0-Beta/include/"
5 | clLibDir = "C:/Program Files (x86)/AMD APP SDK/3.0-0-Beta/lib/x86_64/"
6 | 
7 | extension_mod = Extension(name="_htfe", sources=["HTFE.i", "system/ComputeSystem.cpp", "system/ComputeProgram.cpp", "htfe/HTFE.cpp"], swig_opts=["-c++"], language=["c++"], include_dirs=[clIncludeDir, "./"], library_dirs=[clLibDir], libraries=["OpenCL"])
8 | 
9 | setup(name = "htfe", version="1.0", ext_modules=[extension_mod], package_data={"htfe": ["../resources/*.cl"]})


--------------------------------------------------------------------------------
/source/system/ComputeSystem.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "Uncopyable.h"
 4 | 
 5 | #include <CL/cl.hpp>
 6 | 
 7 | #define SYS_ALLOW_CL_GL_CONTEXT 0
 8 | 
 9 | namespace sys {
10 | 	enum DeviceType {
11 | 		_cpu, _gpu, _all, _none
12 | 	};
13 | 
14 | 	class ComputeSystem {
15 | 	private:
16 | 		cl::Platform _platform;
17 | 		cl::Device _device;
18 | 		cl::Context _context;
19 | 		cl::CommandQueue _queue;
20 | 
21 | 	public:
22 | 		bool create(DeviceType type, bool createFromGLContext = false);
23 | 
24 | 		cl::Platform &getPlatform() {
25 | 			return _platform;
26 | 		}
27 | 
28 | 		cl::Device &getDevice() {
29 | 			return _device;
30 | 		}
31 | 
32 | 		cl::Context &getContext() {
33 | 			return _context;
34 | 		}
35 | 
36 | 		cl::CommandQueue &getQueue() {
37 | 			return _queue;
38 | 		}
39 | 	};
40 | }


--------------------------------------------------------------------------------
/source/system/ComputeProgram.cpp:
--------------------------------------------------------------------------------
 1 | #include "ComputeProgram.h"
 2 | 
 3 | #include <fstream>
 4 | #include <iostream>
 5 | 
 6 | using namespace sys;
 7 | 
 8 | bool ComputeProgram::loadFromFile(const std::string &name, ComputeSystem &cs) {
 9 | 	std::ifstream fromFile(name);
10 | 
11 | 	if (!fromFile.is_open()) {
12 | #ifdef SYS_DEBUG
13 | 		std::cerr << "Could not open file " << name << "!" << std::endl;
14 | #endif
15 | 		return false;
16 | 	}
17 | 
18 | 	std::string source = "";
19 | 
20 | 	while (!fromFile.eof() && fromFile.good()) {
21 | 		std::string line; 
22 | 
23 | 		std::getline(fromFile, line);
24 | 
25 | 		source += line + "\n";
26 | 	}
27 | 
28 | 	_program = cl::Program(cs.getContext(), source);
29 | 
30 | 	if (_program.build(std::vector<cl::Device>(1, cs.getDevice())) != CL_SUCCESS) {
31 | #ifdef SYS_DEBUG
32 | 		std::cerr << "Error building: " << _program.getBuildInfo<CL_PROGRAM_BUILD_LOG>(cs.getDevice()) << std::endl;
33 | #endif
34 | 		return false;
35 | 	}
36 | 
37 | 	return true;
38 | }


--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
 1 | pyHTFE
 2 | Copyright (C) 2015 Eric Laukien
 3 | 
 4 | This software is provided 'as-is', without any express or implied
 5 | warranty.  In no event will the authors be held liable for any damages
 6 | arising from the use of this software.
 7 | 
 8 | Permission is granted to anyone to use this software for any purpose,
 9 | including commercial applications, and to alter it and redistribute it
10 | freely, subject to the following restrictions:
11 | 
12 | 1. The origin of this software must not be misrepresented; you must not
13 | 	claim that you wrote the original software. If you use this software
14 | 	in a product, an acknowledgment in the product documentation would be
15 | 	appreciated but is not required.
16 | 2. Altered source versions must be plainly marked as such, and must not be
17 | 	misrepresented as being the original software.
18 | 3. This notice may not be removed or altered from any source distribution.
19 | 
20 | ------------------------------------------------------------------------------
21 | 
22 | pyHTFE uses the following external libraries:
23 | 
24 | OpenCL


--------------------------------------------------------------------------------
/source/example.py:
--------------------------------------------------------------------------------
 1 | import htfe as ht
 2 | 
 3 | cs = ht.ComputeSystem()
 4 | 
 5 | cs.create(ht._gpu)
 6 | 
 7 | prog = ht.ComputeProgram()
 8 | 
 9 | if not prog.loadFromFile("htfe.cl", cs):
10 |     print("Could not load program!")
11 | 
12 | h = ht.HTFE()
13 | 
14 | inputWidth = 4
15 | inputHeight = 4
16 | minInitWeight = -0.1
17 | maxInitWeight = 0.1
18 | 
19 | layerDescs = []
20 | 
21 | l1 = ht.LayerDesc()
22 | l1._width = 16
23 | l1._height = 16
24 | 
25 | l2 = ht.LayerDesc()
26 | l2._width = 12
27 | l2._height = 12
28 | 
29 | l3 = ht.LayerDesc()
30 | l3._width = 8
31 | l3._height = 8
32 | 
33 | layerDescs.append(l1)
34 | layerDescs.append(l2)
35 | layerDescs.append(l3)
36 | 
37 | h.createRandom(cs, prog, inputWidth, inputHeight, layerDescs, minInitWeight, maxInitWeight)
38 | 
39 | sequence = [
40 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
41 |         [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 ],
42 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
43 |         [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 ],
44 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
45 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
46 |         [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 ],
47 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
48 |         [ 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 1 ],
49 |         [ 0, 0, 1, 1, 0, 1, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1 ]
50 |     ]
51 | 
52 | for p in range(0, 100):
53 |     for i in range(0, len(sequence)):
54 |         for j in range(0, 16):
55 |             h.setInput(j, sequence[i][j])
56 | 
57 |         h.activate(cs)
58 |         h.learn(cs)
59 |         h.stepEnd()
60 | 
61 | #h.clearMemory(cs)
62 | 
63 | for i in range(0, len(sequence)):
64 |     for j in range(0, 16):
65 |         h.setInput(j, sequence[i][j])
66 | 
67 |     h.activate(cs)
68 |     
69 |     t = ""
70 | 
71 |     for j in range(0, 16):
72 |         if h.getPrediction(j) > 0.5:
73 |             t += "1 "
74 |         else:
75 |             t += "0 "
76 | 
77 |     print(t)
78 | 
79 |     h.stepEnd()
80 | 
81 | print("Test complete")


--------------------------------------------------------------------------------
/source/system/ComputeSystem.cpp:
--------------------------------------------------------------------------------
 1 | #include "ComputeSystem.h"
 2 | 
 3 | #include <iostream>
 4 | 
 5 | using namespace sys;
 6 | 
 7 | bool ComputeSystem::create(DeviceType type, bool createFromGLContext) {
 8 | 	if (type == _none) {
 9 | #ifdef SYS_DEBUG
10 | 		std::cout << "No OpenCL context created." << std::endl;
11 | #endif
12 | 		return true;
13 | 	}
14 | 
15 | 	std::vector<cl::Platform> allPlatforms;
16 | 	cl::Platform::get(&allPlatforms);
17 | 
18 | 	if (allPlatforms.empty()) {
19 | #ifdef SYS_DEBUG
20 | 		std::cout << "No platforms found. Check your OpenCL installation." << std::endl;
21 | #endif
22 | 		return false;
23 | 	}
24 | 
25 | 	_platform = allPlatforms.front();
26 | 
27 | #ifdef SYS_DEBUG
28 | 	std::cout << "Using platform: " << _platform.getInfo<CL_PLATFORM_NAME>() << std::endl;
29 | #endif
30 | 
31 | 	std::vector<cl::Device> allDevices;
32 | 
33 | 	switch (type) {
34 | 	case _cpu:
35 | 		_platform.getDevices(CL_DEVICE_TYPE_CPU, &allDevices);
36 | 		break;
37 | 	case _gpu:
38 | 		_platform.getDevices(CL_DEVICE_TYPE_GPU, &allDevices);
39 | 		break;
40 | 	case _all:
41 | 		_platform.getDevices(CL_DEVICE_TYPE_ALL, &allDevices);
42 | 		break;
43 | 	}
44 | 
45 | 	if (allDevices.empty()) {
46 | #ifdef SYS_DEBUG
47 | 		std::cout << "No devices found. Check your OpenCL installation." << std::endl;
48 | #endif
49 | 		return false;
50 | 	}
51 | 
52 | 	_device = allDevices.front();
53 | 
54 | #ifdef SYS_DEBUG
55 | 	std::cout << "Using device: " << _device.getInfo<CL_DEVICE_NAME>() << std::endl;
56 | #endif
57 | 	
58 | #if(SYS_ALLOW_CL_GL_CONTEXT)
59 | 	if (createFromGLContext) {
60 | #if defined (__APPLE__) || defined(MACOSX)
61 | 		CGLContextObj kCGLContext = CGLGetCurrentContext();
62 | 		CGLShareGroupObj kCGLShareGroup = CGLGetShareGroup(kCGLContext);
63 | 		cl_context_properties props[] = {
64 | 			CL_CONTEXT_PROPERTY_USE_CGL_SHAREGROUP_APPLE, (cl_context_properties)kCGLShareGroup,
65 | 			0
66 | 		};
67 | #else
68 | #if defined WIN32
69 | 		cl_context_properties props[] = {
70 | 			CL_GL_CONTEXT_KHR, (cl_context_properties)wglGetCurrentContext(),
71 | 			CL_WGL_HDC_KHR, (cl_context_properties)wglGetCurrentDC(),
72 | 			CL_CONTEXT_PLATFORM, (cl_context_properties)static_cast<cl_platform_id>(_platform()),
73 | 			0
74 | 		};
75 | #else
76 | 		cl_context_properties props[] = {
77 | 			CL_GL_CONTEXT_KHR, (cl_context_properties)glXGetCurrentContext(),
78 | 			CL_GLX_DISPLAY_KHR, (cl_context_properties)glXGetCurrentDisplay(),
79 | 			CL_CONTEXT_PLATFORM, (cl_context_properties)static_cast<cl_platform_id>(_platform()),
80 | 			0
81 | 		};
82 | #endif
83 | #endif
84 | 
85 | 		_context = cl::Context(_device, props);
86 | 	}
87 | 	else
88 | #endif
89 | 		_context = _device;
90 | 
91 | 	_queue = cl::CommandQueue(_context, _device);
92 | 
93 | 	return true;
94 | }


--------------------------------------------------------------------------------
/source/benchmark1.py:
--------------------------------------------------------------------------------
  1 | import htfe as ht
  2 | import pickle as pic
  3 | import math
  4 | import random
  5 | 
  6 | ############################## HTFE Init ##############################
  7 | 
  8 | cs = ht.ComputeSystem()
  9 | 
 10 | cs.create(ht._gpu)
 11 | 
 12 | prog = ht.ComputeProgram()
 13 | 
 14 | if not prog.loadFromFile("htfe.cl", cs):
 15 |     print("Could not load program!")
 16 | 
 17 | h = ht.HTFE()
 18 | 
 19 | minNote = 21 # Inclusive
 20 | maxNote = 109 # Exclusive
 21 | numNotes = maxNote - minNote
 22 | 
 23 | numNotesRoot = int(math.ceil(math.sqrt(numNotes)))
 24 | 
 25 | inputSize = numNotesRoot * numNotesRoot
 26 | 
 27 | inputWidth = numNotesRoot
 28 | inputHeight = numNotesRoot
 29 | minInitWeight = -0.1
 30 | maxInitWeight = 0.1
 31 | 
 32 | layerDescs = []
 33 | 
 34 | l1 = ht.LayerDesc()
 35 | l1._width = 64
 36 | l1._height = 64
 37 | 
 38 | l2 = ht.LayerDesc()
 39 | l2._width = 44
 40 | l2._height = 44
 41 | 
 42 | l3 = ht.LayerDesc()
 43 | l3._width = 32
 44 | l3._height = 32
 45 | 
 46 | l4 = ht.LayerDesc()
 47 | l4._width = 22
 48 | l4._height = 22
 49 | 
 50 | layerDescs.append(l1)
 51 | layerDescs.append(l2)
 52 | layerDescs.append(l3)
 53 | layerDescs.append(l4)
 54 | 
 55 | h.createRandom(cs, prog, inputWidth, inputHeight, layerDescs, minInitWeight, maxInitWeight)
 56 | 
 57 | ############################## Training ##############################
 58 | 
 59 | f = open("Piano-midi.de.pickle", "rb")
 60 | dataset = pic.load(f)
 61 | 
 62 | numSequencesUse = min(4, len(dataset["train"]))
 63 | 
 64 | trainIterations = 1
 65 | 
 66 | for i in range(0, trainIterations):
 67 |     for seq in range(0, numSequencesUse):
 68 |         for j in range(0, len(dataset["train"][seq])):
 69 |             for k in range(0, numNotes):
 70 |                 h.setInput(k, 0.0)
 71 | 
 72 |             for k in dataset["train"][seq][j]:
 73 |                 h.setInput(int(k) - minNote, 1.0)
 74 | 
 75 |             h.activate(cs)
 76 |             h.learn(cs)
 77 |             h.stepEnd()
 78 | 
 79 |         h.clearMemory(cs)
 80 | 
 81 |         print("Training sequence " + str(seq + 1) + " out of " + str(numSequencesUse) + " completed.")
 82 | 
 83 |     print("Training iteration " + str(i + 1) + " out of " + str(trainIterations) + " completed.")
 84 | 
 85 | ############################## Testing Predictions ##############################
 86 | 
 87 | errorCount = 0.0
 88 | totalCount = 0.0
 89 | 
 90 | for seq in range(0, numSequencesUse):
 91 |     prediction = []
 92 | 
 93 |     for j in range(0, len(dataset["train"][seq])):
 94 |         currentInput = []
 95 | 
 96 |         for k in range(0, numNotes):
 97 |             h.setInput(k, 0.0)
 98 |             currentInput.append(0.0)
 99 | 
100 |         for k in dataset["train"][seq][j]:
101 |             h.setInput(int(k) - minNote, 1.0)
102 |             currentInput[int(k) - minNote] = 1.0
103 | 
104 |         if j > 0:
105 |             # Compare prediction to input
106 |             for k in range(0, numNotes):
107 |                 if (prediction[k] > 0.5) != (currentInput[k] > 0.5):
108 |                     errorCount += 1
109 | 
110 |                 totalCount += 1
111 |         
112 |         h.activate(cs)
113 | 
114 |         h.stepEnd()
115 | 
116 |         prediction = []
117 | 
118 |         for k in range(0, numNotes):
119 |             prediction.append(h.getPrediction(k))
120 | 
121 |     h.clearMemory(cs)
122 | 
123 |     print("Test sequence " + str(seq + 1) + " out of " + str(numSequencesUse) + " tested.")
124 | 
125 | print("Error percent: " + str(errorCount / totalCount * 100) + "%")


--------------------------------------------------------------------------------
/source/htfe/HTFE.h:
--------------------------------------------------------------------------------
  1 | #pragma once
  2 | 
  3 | #include "../system/ComputeSystem.h"
  4 | #include "../system/ComputeProgram.h"
  5 | 
  6 | #include <vector>
  7 | #include <list>
  8 | 
  9 | #include <random>
 10 | 
 11 | #include <memory>
 12 | 
 13 | namespace htfe {
 14 | 	struct LayerDesc {
 15 | 		int _width, _height;
 16 | 
 17 | 		int _receptiveFieldRadius;
 18 | 		int _reconstructionRadius;
 19 | 		int _lateralConnectionRadius;
 20 | 		int _inhibitionRadius;
 21 | 		int _feedBackConnectionRadius;
 22 | 
 23 | 		float _sparsity;
 24 | 
 25 | 		float _dutyCycleDecay;
 26 | 		float _feedForwardAlpha;
 27 | 		float _lateralAlpha;
 28 | 		float _feedBackAlpha;
 29 | 		float _hiddenBiasAlpha;
 30 | 		float _reconstructionAlpha;
 31 | 		float _gamma;
 32 | 		float _lateralScalar;
 33 | 		float _feedBackScalar;
 34 | 		float _weightDecay;
 35 | 
 36 | 		LayerDesc()
 37 | 			: _width(16), _height(16), _receptiveFieldRadius(5), _reconstructionRadius(8), _lateralConnectionRadius(7), _inhibitionRadius(4), _feedBackConnectionRadius(6),
 38 | 			_sparsity(1.01f / 81.0f), _dutyCycleDecay(0.01f),
 39 | 			_feedForwardAlpha(0.05f), _lateralAlpha(0.05f), _feedBackAlpha(0.05f), _hiddenBiasAlpha(0.05f), _reconstructionAlpha(0.05f),
 40 | 			_gamma(0.0f), _lateralScalar(0.1f), _feedBackScalar(0.1f), _weightDecay(0.001f)
 41 | 		{}
 42 | 	};
 43 | 
 44 | 	struct Layer {
 45 | 		cl::Image2D _hiddenFeedForwardActivations;
 46 | 		cl::Image2D _hiddenFeedBackActivations;
 47 | 		cl::Image2D _hiddenFeedBackActivationsPrev;
 48 | 
 49 | 		cl::Image2D _hiddenStatesFeedForward;
 50 | 		cl::Image2D _hiddenStatesFeedForwardPrev;
 51 | 
 52 | 		cl::Image2D _hiddenStatesFeedBack;
 53 | 		cl::Image2D _hiddenStatesFeedBackPrev;
 54 | 		cl::Image2D _hiddenStatesFeedBackPrevPrev;
 55 | 
 56 | 		cl::Image3D _feedForwardWeights;
 57 | 		cl::Image3D _feedForwardWeightsPrev;
 58 | 
 59 | 		cl::Image3D _reconstructionWeights;
 60 | 		cl::Image3D _reconstructionWeightsPrev;
 61 | 
 62 | 		cl::Image2D _visibleBiases;
 63 | 		cl::Image2D _visibleBiasesPrev;
 64 | 
 65 | 		cl::Image2D _hiddenBiases;
 66 | 		cl::Image2D _hiddenBiasesPrev;
 67 | 
 68 | 		cl::Image3D _lateralWeights;
 69 | 		cl::Image3D _lateralWeightsPrev;
 70 | 
 71 | 		cl::Image3D _feedBackWeights;
 72 | 		cl::Image3D _feedBackWeightsPrev;
 73 | 
 74 | 		cl::Image2D _visibleReconstruction;
 75 | 		cl::Image2D _visibleReconstructionPrev;
 76 | 	};
 77 | 		
 78 | 	class HTFE {
 79 | 	private:
 80 | 		int _inputWidth, _inputHeight;
 81 | 
 82 | 		std::vector<LayerDesc> _layerDescs;
 83 | 		std::vector<Layer> _layers;
 84 | 
 85 | 		cl::Kernel _layerHiddenFeedForwardActivateKernel;
 86 | 		cl::Kernel _layerHiddenFeedBackActivateKernel;
 87 | 		cl::Kernel _layerHiddenInhibitKernel;
 88 | 		cl::Kernel _layerVisibleReconstructKernel;
 89 | 		cl::Kernel _layerHiddenWeightUpdateKernel;
 90 | 		cl::Kernel _layerHiddenWeightUpdateLastKernel;
 91 | 		cl::Kernel _layerVisibleWeightUpdateKernel;
 92 | 		cl::Kernel _layerUpdateQKernel;
 93 | 
 94 | 		std::vector<float> _input;
 95 | 		std::vector<float> _prediction;
 96 | 
 97 | 		cl::Image2D _inputImage;
 98 | 		cl::Image2D _inputImagePrev;
 99 | 
100 | 	public:
101 | 		void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, int inputWidth, int inputHeight, const std::vector<LayerDesc> &layerDescs, float minInitWeight, float maxInitWeight);
102 | 	
103 | 		void activate(sys::ComputeSystem &cs);
104 | 		void learn(sys::ComputeSystem &cs);
105 | 		void stepEnd();
106 | 
107 | 		int getInputWidth() const {
108 | 			return _inputWidth;
109 | 		}
110 | 
111 | 		int getInputHeight() const {
112 | 			return _inputHeight;
113 | 		}
114 | 
115 | 		const std::vector<LayerDesc> &getLayerDescs() const {
116 | 			return _layerDescs;
117 | 		}
118 | 
119 | 		const std::vector<Layer> &getLayers() const {
120 | 			return _layers;
121 | 		}
122 | 
123 | 		const cl::Image2D &getInputImage() const {
124 | 			return _inputImage;
125 | 		}
126 | 
127 | 		void setInput(int i, float value) {
128 | 			_input[i] = value;
129 | 		}
130 | 
131 | 		void setInput(int x, int y, float value) {
132 | 			setInput(x + y * _inputWidth, value);
133 | 		}
134 | 
135 | 		float getPrediction(int i) const {
136 | 			return _prediction[i];
137 | 		}
138 | 
139 | 		float getPrediction(int x, int y) const {
140 | 			return getPrediction(x + y * _inputWidth);
141 | 		}
142 | 
143 | 		void clearMemory(sys::ComputeSystem &cs);
144 | 	};
145 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # ![pyHTFE Logo](http://i1218.photobucket.com/albums/dd401/222464/PYHTFELOGOSMALL.png)
 2 | 
 3 | pyHTFE
 4 | =======
 5 | 
 6 | HTFE is a derivative of HTM (hierarchical temporal memory) geared towards high-performance GPU sequence prediction.
 7 | 
 8 | Overview
 9 | -----------
10 | 
11 | HTFE stands for hierarchical temporal free energy. It can be thought of as a echo state network with a self-optimizing hierarchical reservoir.
12 | It can also be thought of as a stack of recurrent sparse autoencoders with local connectivity.
13 | 
14 | The main benefit of this particular architecture is scalability. The hierarchical and fully online nature of the algorithm makes it perfect for large-scale real-time sequence prediction tasks.
15 | 
16 | HTFE exists as both a pure C++ library and this Python binding.
17 | 
18 | Algorithm Description
19 | -----------
20 | 
21 | HTFE is a stack of autoencoders.
22 | 
23 | These autoencoders are not trained to reconstruct their current input, but rather the input of the next timestep.
24 | 
25 | The hidden representation of the autoencoders is kept sparse with explicit local lateral inhibition. This means that the resulting sparse distributed representations (SDRs) are local in the sense that they preserve the local topology of the input. This also vastly reduces forgetting, and means that no stochastic sampling is necessary. It also has recurrent hidden to hidden node connections, so it can handle partial observability.
26 | 
27 | A single layer of HTFE looks like this:
28 | 
29 | # ![HTFE layer](http://i1218.photobucket.com/albums/dd401/222464/HTFERLUnitImage_2.png)
30 | 
31 | This particular autoencoder, as with many autoencoders, can be thought of as a back-propagation network. It backpropagates errors between the reconstruction and next timestep inputs back to hidden nodes, which propagate back to themselves through recurrent connections as well as to the current timestep inputs.
32 | 
33 | These individual autoencoders are then stacked, but are also given additional feed-back connections from higher layers. This allows information to flow both up and down: Representations are features are formed when going up, and predictions are made going down. This has the benefit the features at the highest levels do not need to be very descriptive, since the information is always available in more detail in the layers below it.
34 | 
35 | HTFE has a feature known as temporal pooling: It can group previous events into conceptual "bins", as a by-product of the way the spatial pooling interacts with the hidden to hidden recurrent connections. The temporal pooling also allows the system to function very similarly to an echo state network, where the reservoir optimizes itself (since the SDRs change rather slowly).
36 | 
37 | All together this leads to an extremely scalable system. Below is a rendering of a 5 layer HTFE with over 400000 hidden nodes, running at 60 frames per second on a AMD r290.
38 | 
39 | # ![HTFE hierarchy](http://i1218.photobucket.com/albums/dd401/222464/HTFERLSIZE.png)
40 | 
41 | Install
42 | -----------
43 | 
44 | pyHTFE relies on OpenCL.
45 | 
46 | To get OpenCL, refer to your graphics hardware vendor website (for AMD and Nvidia), or CPU vendor (e.g. the Intel OpenCL SDK).
47 | Works best with AMD cards (best OpenCL support).
48 | 
49 | Once OpenCL is installed, navigate to the setup.py file contained in the source directory. Run the following: 
50 | 
51 | ```python
52 | python setup.py install
53 | ```
54 | 
55 | If this doesn't immediately work, it is possible that the OpenCL SDK installed to an unexpected location. If this is the case, open setup.py and change the following lines to match the location of the OpenCL SDK, and try installing again:
56 | 
57 | ```python
58 | clIncludeDir = "C:/Program Files (x86)/AMD APP SDK/3.0-0-Beta/include/"
59 | clLibDir = "C:/Program Files (x86)/AMD APP SDK/3.0-0-Beta/lib/x86_64/"
60 | ```
61 | 
62 | Usage
63 | -----------
64 | 
65 | More detailed usage instructions will come. For now, see the example.py for usage.
66 | 
67 | License
68 | -----------
69 | 
70 | pyHTFE
71 | Copyright (C) 2015 Eric Laukien
72 | 
73 | This software is provided 'as-is', without any express or implied
74 | warranty.  In no event will the authors be held liable for any damages
75 | arising from the use of this software.
76 | 
77 | Permission is granted to anyone to use this software for any purpose,
78 | including commercial applications, and to alter it and redistribute it
79 | freely, subject to the following restrictions:
80 | 
81 | 1. The origin of this software must not be misrepresented; you must not
82 | 	claim that you wrote the original software. If you use this software
83 | 	in a product, an acknowledgment in the product documentation would be
84 | 	appreciated but is not required.
85 | 2. Altered source versions must be plainly marked as such, and must not be
86 | 	misrepresented as being the original software.
87 | 3. This notice may not be removed or altered from any source distribution.
88 | 
89 | ------------------------------------------------------------------------------
90 | 
91 | pyHTFE uses the following external libraries:
92 | 
93 | OpenCL
94 | 
95 | 


--------------------------------------------------------------------------------
/resources/htfe.cl:
--------------------------------------------------------------------------------
  1 | constant sampler_t normalizedClampedNearestSampler = CLK_NORMALIZED_COORDS_TRUE |
  2 | CLK_ADDRESS_CLAMP |
  3 | CLK_FILTER_NEAREST;
  4 | 
  5 | constant sampler_t normalizedClampedToEdgeNearestSampler = CLK_NORMALIZED_COORDS_TRUE |
  6 | CLK_ADDRESS_CLAMP_TO_EDGE |
  7 | CLK_FILTER_NEAREST;
  8 | 
  9 | constant sampler_t unnormalizedClampedNearestSampler = CLK_NORMALIZED_COORDS_FALSE |
 10 | CLK_ADDRESS_CLAMP |
 11 | CLK_FILTER_NEAREST;
 12 | 
 13 | constant sampler_t defaultNormalizedSampler = CLK_NORMALIZED_COORDS_TRUE |
 14 | CLK_ADDRESS_CLAMP_TO_EDGE |
 15 | CLK_FILTER_NEAREST;
 16 | 
 17 | constant sampler_t defaultUnnormalizedSampler = CLK_NORMALIZED_COORDS_FALSE |
 18 | CLK_ADDRESS_CLAMP_TO_EDGE |
 19 | CLK_FILTER_NEAREST;
 20 | 
 21 | float randFloat(uint2* state) {
 22 | 	const float invMaxInt = 1.0f / 4294967296.0f;
 23 | 	uint x = (*state).x * 17 + (*state).y * 13123;
 24 | 	(*state).x = (x << 13) ^ x;
 25 | 	(*state).y ^= (x << 7);
 26 | 
 27 | 	uint tmp = x * (x * x * 15731 + 74323) + 871483;
 28 | 
 29 | 	return convert_float(tmp) * invMaxInt;
 30 | }
 31 | 
 32 | float sigmoid(float x) {
 33 | 	return 1.0f / (1.0f + exp(-x));
 34 | }
 35 | 
 36 | float boostFunction(float trace, float threshold) {
 37 | 	return fmin(1.0f, fmax(0.0f, threshold - trace) / threshold);
 38 | }
 39 | 
 40 | void kernel initializeLayerHidden(write_only image2d_t hiddenFeedForwardActivations,
 41 | 	write_only image2d_t hiddenFeedBackActivations,
 42 | 	write_only image2d_t hiddenStates,
 43 | 	write_only image3d_t feedForwardWeights,
 44 | 	write_only image2d_t hiddenBiases,
 45 | 	write_only image3d_t lateralWeights,
 46 | 	write_only image3d_t feedBackWeights,
 47 | 	int feedForwardSize, int lateralSize, int feedBackSize,
 48 | 	uint2 seed, float sparsity, float lateralScalar, float feedBackScalar, float minWeight, float maxWeight)
 49 | {
 50 | 	uint2 seedValue = seed + (uint2)(get_global_id(0) * 29 + 12, get_global_id(1) * 16 + 23) * 36;
 51 | 
 52 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
 53 | 
 54 | 	write_imagef(hiddenFeedForwardActivations, hiddenPosition, (float4)(0.0f, 0.0f, 0.0f, 0.0f));
 55 | 	write_imagef(hiddenFeedBackActivations, hiddenPosition, (float4)(0.0f, 0.0f, 0.0f, 0.0f));
 56 | 	write_imagef(hiddenStates, hiddenPosition, (float4)(0.0f, 0.0f, 0.0f, 0.0f));
 57 | 
 58 | 	float hiddenBias = randFloat(&seedValue) * (maxWeight - minWeight) + minWeight;
 59 | 
 60 | 	write_imagef(hiddenBiases, hiddenPosition, (float4)(hiddenBias, 0.0f, 0.0f, 0.0f));
 61 | 
 62 | 	for (int wi = 0; wi < feedForwardSize; wi++) {
 63 | 		int4 weightPosition = (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0);
 64 | 
 65 | 		float feedForwardWeight = randFloat(&seedValue) * (maxWeight - minWeight) + minWeight;
 66 | 
 67 | 		write_imagef(feedForwardWeights, weightPosition, (float4)(feedForwardWeight, 0.0f, 0.0f, 0.0f));
 68 | 	}
 69 | 
 70 | 	for (int wi = 0; wi < lateralSize; wi++) {
 71 | 		int4 weightPosition = (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0);
 72 | 
 73 | 		float lateralWeight = lateralScalar * (randFloat(&seedValue) * (maxWeight - minWeight) + minWeight);
 74 | 
 75 | 		write_imagef(lateralWeights, weightPosition, (float4)(lateralWeight, 0.0f, 0.0f, 0.0f));
 76 | 	}
 77 | 
 78 | 	for (int wi = 0; wi < feedBackSize; wi++) {
 79 | 		int4 weightPosition = (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0);
 80 | 
 81 | 		float feedBackWeight = feedBackScalar * (randFloat(&seedValue) * (maxWeight - minWeight) + minWeight);
 82 | 
 83 | 		write_imagef(feedBackWeights, weightPosition, (float4)(feedBackWeight, 0.0f, 0.0f, 0.0f));
 84 | 	}
 85 | }
 86 | 
 87 | void kernel initializeLayerVisible(write_only image2d_t visibleBiases, write_only image2d_t visibleReconstruction, write_only image3d_t reconstructionWeights,
 88 | 	int reconstructionSize, uint2 seed, float minWeight, float maxWeight)
 89 | {
 90 | 	uint2 seedValue = seed + (uint2)(get_global_id(0) * 64 + 11, get_global_id(1) * 16 + 4) * 2;
 91 | 
 92 | 	int2 visiblePosition = (int2)(get_global_id(0), get_global_id(1));
 93 | 
 94 | 	float bias = randFloat(&seedValue) * (maxWeight - minWeight) + minWeight;
 95 | 
 96 | 	write_imagef(visibleBiases, visiblePosition, (float4)(bias, 0.0f, 0.0f, 0.0f));
 97 | 
 98 | 	for (int wi = 0; wi < reconstructionSize; wi++) {
 99 | 		float weight = randFloat(&seedValue) * (maxWeight - minWeight) + minWeight;
100 | 
101 | 		write_imagef(reconstructionWeights, (int4)(visiblePosition.x, visiblePosition.y, wi, 0), (float4)(weight, 0.0f, 0.0f, 0.0f));
102 | 	}
103 | 
104 | 	write_imagef(visibleReconstruction, visiblePosition, (float4)(0.0f, 0.0f, 0.0f, 0.0f));
105 | }
106 | 
107 | void kernel layerHiddenFeedForwardActivate(read_only image2d_t inputs, read_only image2d_t hiddenStatesPrev, read_only image3d_t feedForwardWeights, read_only image3d_t lateralWeights, read_only image2d_t hiddenBiases, write_only image2d_t hiddenFeedForwardActivations,
108 | 	int2 layerSize, float2 layerSizeMinusOneInv, int2 inputSize, int2 inputSizeMinusOne, int receptiveFieldRadius, int lateralConnectionRadius)
109 | {
110 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
111 | 
112 | 	float2 inputCenterPositionNormalized = (float2)(hiddenPosition.x * layerSizeMinusOneInv.x, hiddenPosition.y * layerSizeMinusOneInv.y);
113 | 	int2 inputCenterPosition = (int2)(inputCenterPositionNormalized.x * inputSizeMinusOne.x, inputCenterPositionNormalized.y * inputSizeMinusOne.y);
114 | 
115 | 	float sum = 0.0f;
116 | 
117 | 	int wi = 0;
118 | 
119 | 	for (int dx = -receptiveFieldRadius; dx <= receptiveFieldRadius; dx++)
120 | 		for (int dy = -receptiveFieldRadius; dy <= receptiveFieldRadius; dy++) {
121 | 			int2 inputPosition = (int2)(inputCenterPosition.x + dx, inputCenterPosition.y + dy);
122 | 
123 | 			if (inputPosition.x >= 0 && inputPosition.x < inputSize.x && inputPosition.y >= 0 && inputPosition.y < inputSize.y) {
124 | 				float input = read_imagef(inputs, inputPosition).x;
125 | 
126 | 				float weight = read_imagef(feedForwardWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
127 | 
128 | 				sum += weight * input;
129 | 			}
130 | 
131 | 			wi++;
132 | 		}
133 | 
134 | 	wi = 0;
135 | 
136 | 	for (int dx = -lateralConnectionRadius; dx <= lateralConnectionRadius; dx++)
137 | 		for (int dy = -lateralConnectionRadius; dy <= lateralConnectionRadius; dy++) {
138 | 			int2 layerPosition = (int2)(hiddenPosition.x + dx, hiddenPosition.y + dy);
139 | 
140 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
141 | 				float state = read_imagef(hiddenStatesPrev, layerPosition).x;
142 | 
143 | 				float weight = read_imagef(lateralWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
144 | 
145 | 				sum += weight * state;
146 | 			}
147 | 
148 | 			wi++;
149 | 		}
150 | 
151 | 	// Bias
152 | 	float bias = read_imagef(hiddenBiases, hiddenPosition).x;
153 | 
154 | 	sum += bias;
155 | 
156 | 	write_imagef(hiddenFeedForwardActivations, hiddenPosition, (float4)(sigmoid(sum), sum, 0.0f, 0.0f));
157 | }
158 | 
159 | void kernel layerHiddenFeedBackActivate(read_only image2d_t hiddenFeedForwardActivations, read_only image2d_t nextLayerHiddenStates, read_only image3d_t feedBackWeights, write_only image2d_t hiddenFeedBackActivations,
160 | 	int2 layerSize, float2 layerSizeMinusOneInv, int2 nextSize, int2 nextSizeMinusOne, int feedBackRadius)
161 | {
162 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
163 | 
164 | 	float2 nextCenterPositionNormalized = (float2)(hiddenPosition.x * layerSizeMinusOneInv.x, hiddenPosition.y * layerSizeMinusOneInv.y);
165 | 	int2 nextCenterPosition = (int2)(nextCenterPositionNormalized.x * nextSizeMinusOne.x, nextCenterPositionNormalized.y * nextSizeMinusOne.y);
166 | 
167 | 	float feedForwardActivation = read_imagef(hiddenFeedForwardActivations, hiddenPosition).y;
168 | 
169 | 	float sum = feedForwardActivation;
170 | 
171 | 	int wi = 0;
172 | 
173 | 	for (int dx = -feedBackRadius; dx <= feedBackRadius; dx++)
174 | 		for (int dy = -feedBackRadius; dy <= feedBackRadius; dy++) {
175 | 			int2 nextPosition = (int2)(nextCenterPosition.x + dx, nextCenterPosition.y + dy);
176 | 
177 | 			if (nextPosition.x >= 0 && nextPosition.x < nextSize.x && nextPosition.y >= 0 && nextPosition.y < nextSize.y) {
178 | 				float next = read_imagef(nextLayerHiddenStates, nextPosition).x;
179 | 
180 | 				float weight = read_imagef(feedBackWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
181 | 
182 | 				sum += weight * next;
183 | 			}
184 | 
185 | 			wi++;
186 | 		}
187 | 
188 | 	write_imagef(hiddenFeedBackActivations, hiddenPosition, (float4)(sigmoid(sum), 0.0f, 0.0f, 0.0f));
189 | }
190 | 
191 | void kernel layerHiddenInhibit(read_only image2d_t hiddenActivations, read_only image2d_t hiddenStatesPrev, write_only image2d_t hiddenStates,
192 | 	int2 layerSize, int inhibitionRadius, float localActivity)
193 | {
194 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
195 | 
196 | 	float thisActivation = read_imagef(hiddenActivations, hiddenPosition).x;
197 | 
198 | 	float numHigher = 0.0f;
199 | 
200 | 	for (int dx = -inhibitionRadius; dx <= inhibitionRadius; dx++)
201 | 		for (int dy = -inhibitionRadius; dy <= inhibitionRadius; dy++) {
202 | 			if (dx == 0 && dy == 0)
203 | 				continue;
204 | 
205 | 			int2 layerPosition = (int2)(hiddenPosition.x + dx, hiddenPosition.y + dy);
206 | 
207 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
208 | 				float activation = read_imagef(hiddenActivations, layerPosition).x;
209 | 
210 | 				numHigher += activation >= thisActivation ? 1.0f : 0.0f;
211 | 			}
212 | 		}
213 | 
214 | 	float newState = numHigher < localActivity ? 1.0f : 0.0f;
215 | 
216 | 	write_imagef(hiddenStates, hiddenPosition, (float4)(newState, 0.0f, 0.0f, 0.0f));
217 | }
218 | 
219 | void kernel layerVisibleReconstruct(read_only image2d_t hiddenStates, read_only image3d_t reconstructionWeights, read_only image2d_t visibleBiases, write_only image2d_t visibleReconstruction,
220 | 	int reconstructionReceptiveRadius, int2 inputSizeMinusOne, float2 inputSizeMinusOneInv, int2 layerSize, int2 layerSizeMinusOne, float2 layerSizeMinusOneInv)
221 | {
222 | 	int2 visiblePosition = (int2)(get_global_id(0), get_global_id(1));
223 | 	float2 layerPositionNormalized = (float2)(visiblePosition.x * inputSizeMinusOneInv.x, visiblePosition.y * inputSizeMinusOneInv.y);
224 | 	int2 layerPositionCenter = (int2)(layerPositionNormalized.x * layerSizeMinusOne.x, layerPositionNormalized.y * layerSizeMinusOne.y);
225 | 
226 | 	float sum = 0.0f;
227 | 
228 | 	int wi = 0;
229 | 
230 | 	for (int dx = -reconstructionReceptiveRadius; dx <= reconstructionReceptiveRadius; dx++)
231 | 		for (int dy = -reconstructionReceptiveRadius; dy <= reconstructionReceptiveRadius; dy++) {
232 | 			int2 layerPosition = (int2)(layerPositionCenter.x + dx, layerPositionCenter.y + dy);
233 | 
234 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
235 | 				float source = read_imagef(hiddenStates, layerPosition).x;
236 | 
237 | 				float weight = read_imagef(reconstructionWeights, (int4)(visiblePosition.x, visiblePosition.y, wi, 0)).x;
238 | 
239 | 				sum += source * weight;
240 | 			}
241 | 
242 | 			wi++;
243 | 		}
244 | 
245 | 	//float bias = read_imagef(visibleBiases, visiblePosition).x;
246 | 
247 | 	//sum += bias;
248 | 
249 | 	write_imagef(visibleReconstruction, visiblePosition, (float4)(sum, 0.0f, 0.0f, 0.0f));
250 | }
251 | 
252 | void kernel layerHiddenWeightUpdate(read_only image2d_t visibleReconstruction, read_only image2d_t inputs, read_only image2d_t inputsPrev, read_only image2d_t feedBackActivationsPrev, read_only image2d_t hiddenStatesPrev, read_only image2d_t hiddenStatesPrevPrev, read_only image2d_t nextLayerHiddenStatesPrev,
253 | 	read_only image3d_t reconstructionWeightsPrev, read_only image3d_t feedForwardWeightsPrev, read_only image3d_t lateralWeightsPrev, read_only image2d_t hiddenBiasesPrev, read_only image3d_t feedBackWeightsPrev,
254 | 	write_only image3d_t feedForwardWeights, write_only image3d_t lateralWeights, write_only image2d_t hiddenBiases, write_only image3d_t feedBackWeights,
255 | 	int2 layerSize, int2 layerSizeMinusOne, float2 layerSizeMinusOneInv, int2 inputSize, int2 inputSizeMinusOne, float2 inputSizeMinusOneInv, int2 nextSize, int2 nextSizeMinusOne, int receptiveFieldRadius, int lateralConnectionRadius, int feedBackRadius, int reconstructionReceptiveRadius, float sparsity, float4 alpha, float weightDecay)
256 | {
257 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
258 | 
259 | 	float2 inputCenterPositionNormalized = (float2)(hiddenPosition.x * layerSizeMinusOneInv.x, hiddenPosition.y * layerSizeMinusOneInv.y);
260 | 	int2 inputCenterPosition = (int2)(inputCenterPositionNormalized.x * inputSizeMinusOne.x, inputCenterPositionNormalized.y * inputSizeMinusOne.y);
261 | 
262 | 	int2 nextCenterPosition = (int2)(inputCenterPositionNormalized.x * nextSizeMinusOne.x, inputCenterPositionNormalized.y * nextSizeMinusOne.y);
263 | 
264 | 	float thisHiddenStatePrev = read_imagef(hiddenStatesPrev, hiddenPosition).x;
265 | 	float thisHiddenStatePrevPrev = read_imagef(hiddenStatesPrevPrev, hiddenPosition).x;
266 | 	float thisActivation = read_imagef(feedBackActivationsPrev, hiddenPosition).x;
267 | 
268 | 	// --------------------------------- Collect Error -------------------------------------
269 | 
270 | 	float sum = 0.0f;
271 | 
272 | 	for (int dx = -receptiveFieldRadius; dx <= receptiveFieldRadius; dx++)
273 | 		for (int dy = -receptiveFieldRadius; dy <= receptiveFieldRadius; dy++) {
274 | 			int2 inputPosition = (int2)(inputCenterPosition.x + dx, inputCenterPosition.y + dy);
275 | 
276 | 			if (inputPosition.x >= 0 && inputPosition.x < inputSize.x && inputPosition.y >= 0 && inputPosition.y < inputSize.y) {
277 | 				// Next layer node's receptive field
278 | 				int2 fieldCenter = (int2)(inputPosition.x * inputSizeMinusOneInv.x * layerSizeMinusOne.x, inputPosition.y * inputSizeMinusOneInv.y * layerSizeMinusOne.y);
279 | 
280 | 				int2 fieldLowerBounds = fieldCenter - (int2)(reconstructionReceptiveRadius);
281 | 				int2 fieldUpperBounds = fieldCenter + (int2)(reconstructionReceptiveRadius);
282 | 
283 | 				// Check for containment
284 | 				if (hiddenPosition.x >= fieldLowerBounds.x && hiddenPosition.x <= fieldUpperBounds.x && hiddenPosition.y >= fieldLowerBounds.y && hiddenPosition.y <= fieldUpperBounds.y) {
285 | 					int rdx = hiddenPosition.x - fieldLowerBounds.x;
286 | 					int rdy = hiddenPosition.y - fieldLowerBounds.y;
287 | 
288 | 					float input = read_imagef(inputs, inputPosition).x;
289 | 					float recon = read_imagef(visibleReconstruction, inputPosition).x;
290 | 
291 | 					int weightIndex = rdy + rdx * (reconstructionReceptiveRadius * 2 + 1);
292 | 
293 | 					float weight = read_imagef(reconstructionWeightsPrev, (int4)(inputPosition.x, inputPosition.y, weightIndex, 0)).x;
294 | 
295 | 					sum += (input - recon) * weight;
296 | 				}
297 | 			}
298 | 		}
299 | 
300 | 	float learn = thisHiddenStatePrev * (1.0f - thisHiddenStatePrevPrev);
301 | 	float error = learn * thisActivation * (1.0f - thisActivation) * sum;
302 | 
303 | 	// --------------------------------- Update on Error ---------------------------------
304 | 
305 | 	int wi = 0;
306 | 
307 | 	for (int dx = -receptiveFieldRadius; dx <= receptiveFieldRadius; dx++)
308 | 		for (int dy = -receptiveFieldRadius; dy <= receptiveFieldRadius; dy++) {
309 | 			int2 inputPosition = (int2)(inputCenterPosition.x + dx, inputCenterPosition.y + dy);
310 | 
311 | 			if (inputPosition.x >= 0 && inputPosition.x < inputSize.x && inputPosition.y >= 0 && inputPosition.y < inputSize.y) {
312 | 				float input = read_imagef(inputsPrev, inputPosition).x;
313 | 
314 | 				float eligibility = error * input;
315 | 
316 | 				float prevWeight = read_imagef(feedForwardWeightsPrev, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
317 | 
318 | 				float newWeight = (1.0f - weightDecay * thisHiddenStatePrev) * prevWeight + alpha.x * eligibility;
319 | 
320 | 				write_imagef(feedForwardWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
321 | 			}
322 | 
323 | 			wi++;
324 | 		}
325 | 
326 | 	wi = 0;
327 | 
328 | 	for (int dx = -lateralConnectionRadius; dx <= lateralConnectionRadius; dx++)
329 | 		for (int dy = -lateralConnectionRadius; dy <= lateralConnectionRadius; dy++) {
330 | 			int2 layerPosition = (int2)(hiddenPosition.x + dx, hiddenPosition.y + dy);
331 | 
332 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
333 | 				float input = read_imagef(hiddenStatesPrevPrev, layerPosition).x;
334 | 
335 | 				float eligibility = error * input;
336 | 
337 | 				float prevWeight = read_imagef(lateralWeightsPrev, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
338 | 
339 | 				float newWeight = (1.0f - weightDecay * thisHiddenStatePrev) * prevWeight + alpha.y * eligibility;
340 | 
341 | 				write_imagef(lateralWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
342 | 			}
343 | 
344 | 			wi++;
345 | 		}
346 | 
347 | 	wi = 0;
348 | 
349 | 	for (int dx = -feedBackRadius; dx <= feedBackRadius; dx++)
350 | 		for (int dy = -feedBackRadius; dy <= feedBackRadius; dy++) {
351 | 			int2 nextPosition = (int2)(nextCenterPosition.x + dx, nextCenterPosition.y + dy);
352 | 
353 | 			if (nextPosition.x >= 0 && nextPosition.x < nextSize.x && nextPosition.y >= 0 && nextPosition.y < nextSize.y) {
354 | 				float next = read_imagef(nextLayerHiddenStatesPrev, nextPosition).x;
355 | 
356 | 				float eligibility = error * next;
357 | 
358 | 				float prevWeight = read_imagef(feedBackWeightsPrev, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
359 | 
360 | 				float newWeight = (1.0f - weightDecay * thisHiddenStatePrev) * prevWeight + alpha.z * eligibility;
361 | 
362 | 				write_imagef(feedBackWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
363 | 			}
364 | 
365 | 			wi++;
366 | 		}
367 | 
368 | 	float eligibility = error;
369 | 
370 | 	float prevBias = read_imagef(hiddenBiasesPrev, hiddenPosition).x;
371 | 
372 | 	float newBias = (1.0f - weightDecay * thisHiddenStatePrev) * prevBias + alpha.w * eligibility;
373 | 
374 | 	write_imagef(hiddenBiases, hiddenPosition, (float4)(newBias, 0.0f, 0.0f, 0.0f));
375 | }
376 | 
377 | void kernel layerHiddenWeightUpdateLast(read_only image2d_t visibleReconstruction, read_only image2d_t inputs, read_only image2d_t inputsPrev, read_only image2d_t feedBackActivationsPrev, read_only image2d_t hiddenStatesPrev, read_only image2d_t hiddenStatesPrevPrev,
378 | 	read_only image3d_t reconstructionWeightsPrev, read_only image3d_t feedForwardWeightsPrev, read_only image3d_t lateralWeightsPrev, read_only image2d_t hiddenBiasesPrev,
379 | 	write_only image3d_t feedForwardWeights, write_only image3d_t lateralWeights, write_only image2d_t hiddenBiases,
380 | 	int2 layerSize, int2 layerSizeMinusOne, float2 layerSizeMinusOneInv, int2 inputSize, int2 inputSizeMinusOne, float2 inputSizeMinusOneInv, int receptiveFieldRadius, int lateralConnectionRadius, int reconstructionReceptiveRadius, float sparsity, float4 alpha, float weightDecay)
381 | {
382 | 	int2 hiddenPosition = (int2)(get_global_id(0), get_global_id(1));
383 | 
384 | 	float2 inputCenterPositionNormalized = (float2)(hiddenPosition.x * layerSizeMinusOneInv.x, hiddenPosition.y * layerSizeMinusOneInv.y);
385 | 	int2 inputCenterPosition = (int2)(inputCenterPositionNormalized.x * inputSizeMinusOne.x, inputCenterPositionNormalized.y * inputSizeMinusOne.y);
386 | 
387 | 	float thisHiddenStatePrev = read_imagef(hiddenStatesPrev, hiddenPosition).x;
388 | 	float thisHiddenStatePrevPrev = read_imagef(hiddenStatesPrevPrev, hiddenPosition).x;
389 | 	float thisActivation = read_imagef(feedBackActivationsPrev, hiddenPosition).x;
390 | 
391 | 	// --------------------------------- Collect Error -------------------------------------
392 | 
393 | 	float sum = 0.0f;
394 | 
395 | 	for (int dx = -receptiveFieldRadius; dx <= receptiveFieldRadius; dx++)
396 | 		for (int dy = -receptiveFieldRadius; dy <= receptiveFieldRadius; dy++) {
397 | 			int2 inputPosition = (int2)(inputCenterPosition.x + dx, inputCenterPosition.y + dy);
398 | 
399 | 			if (inputPosition.x >= 0 && inputPosition.x < inputSize.x && inputPosition.y >= 0 && inputPosition.y < inputSize.y) {
400 | 				// Next layer node's receptive field
401 | 				int2 fieldCenter = (int2)(inputPosition.x * inputSizeMinusOneInv.x * layerSizeMinusOne.x, inputPosition.y * inputSizeMinusOneInv.y * layerSizeMinusOne.y);
402 | 
403 | 				int2 fieldLowerBounds = fieldCenter - (int2)(reconstructionReceptiveRadius);
404 | 				int2 fieldUpperBounds = fieldCenter + (int2)(reconstructionReceptiveRadius);
405 | 
406 | 				// Check for containment
407 | 				if (hiddenPosition.x >= fieldLowerBounds.x && hiddenPosition.x <= fieldUpperBounds.x && hiddenPosition.y >= fieldLowerBounds.y && hiddenPosition.y <= fieldUpperBounds.y) {
408 | 					int rdx = hiddenPosition.x - fieldLowerBounds.x;
409 | 					int rdy = hiddenPosition.y - fieldLowerBounds.y;
410 | 
411 | 					float input = read_imagef(inputs, inputPosition).x;
412 | 					float recon = read_imagef(visibleReconstruction, inputPosition).x;
413 | 
414 | 					int weightIndex = rdy + rdx * (reconstructionReceptiveRadius * 2 + 1);
415 | 
416 | 					float weight = read_imagef(reconstructionWeightsPrev, (int4)(inputPosition.x, inputPosition.y, weightIndex, 0)).x;
417 | 
418 | 					sum += (input - recon) * weight;
419 | 				}
420 | 			}
421 | 		}
422 | 
423 | 	float learn = thisHiddenStatePrev * (1.0f - thisHiddenStatePrevPrev);
424 | 	float error = learn * thisActivation * (1.0f - thisActivation) * sum;
425 | 
426 | 	// --------------------------------- Update on Error ---------------------------------
427 | 
428 | 	int wi = 0;
429 | 
430 | 	for (int dx = -receptiveFieldRadius; dx <= receptiveFieldRadius; dx++)
431 | 		for (int dy = -receptiveFieldRadius; dy <= receptiveFieldRadius; dy++) {
432 | 			int2 inputPosition = (int2)(inputCenterPosition.x + dx, inputCenterPosition.y + dy);
433 | 
434 | 			if (inputPosition.x >= 0 && inputPosition.x < inputSize.x && inputPosition.y >= 0 && inputPosition.y < inputSize.y) {
435 | 				float input = read_imagef(inputsPrev, inputPosition).x;
436 | 
437 | 				float eligibility = error * input;
438 | 
439 | 				float prevWeight = read_imagef(feedForwardWeightsPrev, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
440 | 
441 | 				float newWeight = (1.0f - weightDecay * thisHiddenStatePrev) * prevWeight + alpha.x * eligibility;
442 | 
443 | 				write_imagef(feedForwardWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
444 | 			}
445 | 
446 | 			wi++;
447 | 		}
448 | 
449 | 	wi = 0;
450 | 
451 | 	for (int dx = -lateralConnectionRadius; dx <= lateralConnectionRadius; dx++)
452 | 		for (int dy = -lateralConnectionRadius; dy <= lateralConnectionRadius; dy++) {
453 | 			int2 layerPosition = (int2)(hiddenPosition.x + dx, hiddenPosition.y + dy);
454 | 
455 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
456 | 				float input = read_imagef(hiddenStatesPrevPrev, layerPosition).x;
457 | 
458 | 				float eligibility = error * input;
459 | 
460 | 				float prevWeight = read_imagef(lateralWeightsPrev, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0)).x;
461 | 
462 | 				float newWeight = (1.0f - weightDecay * thisHiddenStatePrev) * prevWeight + alpha.y * eligibility;
463 | 
464 | 				write_imagef(lateralWeights, (int4)(hiddenPosition.x, hiddenPosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
465 | 			}
466 | 
467 | 			wi++;
468 | 		}
469 | 
470 | 	float eligibility = error;
471 | 
472 | 	float prevBias = read_imagef(hiddenBiasesPrev, hiddenPosition).x;
473 | 
474 | 	float newBias = (1.0f - weightDecay * thisHiddenStatePrev) * prevBias + alpha.w * eligibility;
475 | 
476 | 	write_imagef(hiddenBiases, hiddenPosition, (float4)(newBias, 0.0f, 0.0f, 0.0f));
477 | }
478 | 
479 | void kernel layerVisibleWeightUpdate(read_only image2d_t visibleReconstruction, read_only image2d_t inputs, read_only image2d_t hiddenStatesPrev, read_only image3d_t reconstructionWeightsPrev, read_only image2d_t visibleBiasesPrev, write_only image3d_t reconstructionWeights, write_only image2d_t visibleBiases,
480 | 	int reconstructionReceptiveRadius, int2 inputSizeMinusOne, float2 inputSizeMinusOneInv, int2 layerSize, int2 layerSizeMinusOne, float2 layerSizeMinusOneInv, float alpha)
481 | {
482 | 	int2 visiblePosition = (int2)(get_global_id(0), get_global_id(1));
483 | 	float2 layerPositionNormalized = (float2)(visiblePosition.x * inputSizeMinusOneInv.x, visiblePosition.y * inputSizeMinusOneInv.y);
484 | 	int2 layerPositionCenter = (int2)(layerPositionNormalized.x * layerSizeMinusOne.x, layerPositionNormalized.y * layerSizeMinusOne.y);
485 | 
486 | 	float input = read_imagef(inputs, visiblePosition).x;
487 | 	float recon = read_imagef(visibleReconstruction, visiblePosition).x;
488 | 
489 | 	float error = input - recon;
490 | 
491 | 	int wi = 0;
492 | 
493 | 	for (int dx = -reconstructionReceptiveRadius; dx <= reconstructionReceptiveRadius; dx++)
494 | 		for (int dy = -reconstructionReceptiveRadius; dy <= reconstructionReceptiveRadius; dy++) {
495 | 			int2 layerPosition = (int2)(layerPositionCenter.x + dx, layerPositionCenter.y + dy);
496 | 
497 | 			if (layerPosition.x >= 0 && layerPosition.x < layerSize.x && layerPosition.y >= 0 && layerPosition.y < layerSize.y) {
498 | 				float source = read_imagef(hiddenStatesPrev, layerPosition).x;
499 | 
500 | 				float eligibility = error * source;
501 | 
502 | 				float prevWeight = read_imagef(reconstructionWeightsPrev, (int4)(visiblePosition.x, visiblePosition.y, wi, 0)).x;
503 | 
504 | 				float newWeight = prevWeight + alpha * eligibility;
505 | 
506 | 				write_imagef(reconstructionWeights, (int4)(visiblePosition.x, visiblePosition.y, wi, 0), (float4)(newWeight, 0.0f, 0.0f, 0.0f));
507 | 			}
508 | 
509 | 			wi++;
510 | 		}
511 | 
512 | 	float eligibility = error;
513 | 
514 | 	float prevBias = read_imagef(visibleBiasesPrev, visiblePosition).x;
515 | 
516 | 	float newBias = prevBias + alpha * eligibility;
517 | 
518 | 	write_imagef(visibleBiases, visiblePosition, (float4)(newBias, 0.0f, 0.0f, 0.0f));
519 | }


--------------------------------------------------------------------------------
/source/htfe.py:
--------------------------------------------------------------------------------
  1 | # This file was automatically generated by SWIG (http://www.swig.org).
  2 | # Version 2.0.12
  3 | #
  4 | # Do not make changes to this file unless you know what you are doing--modify
  5 | # the SWIG interface file instead.
  6 | 
  7 | 
  8 | 
  9 | 
 10 | 
 11 | from sys import version_info
 12 | if version_info >= (2,6,0):
 13 |     def swig_import_helper():
 14 |         from os.path import dirname
 15 |         import imp
 16 |         fp = None
 17 |         try:
 18 |             fp, pathname, description = imp.find_module('_htfe', [dirname(__file__)])
 19 |         except ImportError:
 20 |             import _htfe
 21 |             return _htfe
 22 |         if fp is not None:
 23 |             try:
 24 |                 _mod = imp.load_module('_htfe', fp, pathname, description)
 25 |             finally:
 26 |                 fp.close()
 27 |             return _mod
 28 |     _htfe = swig_import_helper()
 29 |     del swig_import_helper
 30 | else:
 31 |     import _htfe
 32 | del version_info
 33 | try:
 34 |     _swig_property = property
 35 | except NameError:
 36 |     pass # Python < 2.2 doesn't have 'property'.
 37 | def _swig_setattr_nondynamic(self,class_type,name,value,static=1):
 38 |     if (name == "thisown"): return self.this.own(value)
 39 |     if (name == "this"):
 40 |         if type(value).__name__ == 'SwigPyObject':
 41 |             self.__dict__[name] = value
 42 |             return
 43 |     method = class_type.__swig_setmethods__.get(name,None)
 44 |     if method: return method(self,value)
 45 |     if (not static):
 46 |         self.__dict__[name] = value
 47 |     else:
 48 |         raise AttributeError("You cannot add attributes to %s" % self)
 49 | 
 50 | def _swig_setattr(self,class_type,name,value):
 51 |     return _swig_setattr_nondynamic(self,class_type,name,value,0)
 52 | 
 53 | def _swig_getattr(self,class_type,name):
 54 |     if (name == "thisown"): return self.this.own()
 55 |     method = class_type.__swig_getmethods__.get(name,None)
 56 |     if method: return method(self)
 57 |     raise AttributeError(name)
 58 | 
 59 | def _swig_repr(self):
 60 |     try: strthis = "proxy of " + self.this.__repr__()
 61 |     except: strthis = ""
 62 |     return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
 63 | 
 64 | try:
 65 |     _object = object
 66 |     _newclass = 1
 67 | except AttributeError:
 68 |     class _object : pass
 69 |     _newclass = 0
 70 | 
 71 | 
 72 | class SwigPyIterator(_object):
 73 |     __swig_setmethods__ = {}
 74 |     __setattr__ = lambda self, name, value: _swig_setattr(self, SwigPyIterator, name, value)
 75 |     __swig_getmethods__ = {}
 76 |     __getattr__ = lambda self, name: _swig_getattr(self, SwigPyIterator, name)
 77 |     def __init__(self, *args, **kwargs): raise AttributeError("No constructor defined - class is abstract")
 78 |     __repr__ = _swig_repr
 79 |     __swig_destroy__ = _htfe.delete_SwigPyIterator
 80 |     __del__ = lambda self : None;
 81 |     def value(self): return _htfe.SwigPyIterator_value(self)
 82 |     def incr(self, n=1): return _htfe.SwigPyIterator_incr(self, n)
 83 |     def decr(self, n=1): return _htfe.SwigPyIterator_decr(self, n)
 84 |     def distance(self, *args): return _htfe.SwigPyIterator_distance(self, *args)
 85 |     def equal(self, *args): return _htfe.SwigPyIterator_equal(self, *args)
 86 |     def copy(self): return _htfe.SwigPyIterator_copy(self)
 87 |     def next(self): return _htfe.SwigPyIterator_next(self)
 88 |     def __next__(self): return _htfe.SwigPyIterator___next__(self)
 89 |     def previous(self): return _htfe.SwigPyIterator_previous(self)
 90 |     def advance(self, *args): return _htfe.SwigPyIterator_advance(self, *args)
 91 |     def __eq__(self, *args): return _htfe.SwigPyIterator___eq__(self, *args)
 92 |     def __ne__(self, *args): return _htfe.SwigPyIterator___ne__(self, *args)
 93 |     def __iadd__(self, *args): return _htfe.SwigPyIterator___iadd__(self, *args)
 94 |     def __isub__(self, *args): return _htfe.SwigPyIterator___isub__(self, *args)
 95 |     def __add__(self, *args): return _htfe.SwigPyIterator___add__(self, *args)
 96 |     def __sub__(self, *args): return _htfe.SwigPyIterator___sub__(self, *args)
 97 |     def __iter__(self): return self
 98 | SwigPyIterator_swigregister = _htfe.SwigPyIterator_swigregister
 99 | SwigPyIterator_swigregister(SwigPyIterator)
100 | 
101 | class vectorld(_object):
102 |     __swig_setmethods__ = {}
103 |     __setattr__ = lambda self, name, value: _swig_setattr(self, vectorld, name, value)
104 |     __swig_getmethods__ = {}
105 |     __getattr__ = lambda self, name: _swig_getattr(self, vectorld, name)
106 |     __repr__ = _swig_repr
107 |     def iterator(self): return _htfe.vectorld_iterator(self)
108 |     def __iter__(self): return self.iterator()
109 |     def __nonzero__(self): return _htfe.vectorld___nonzero__(self)
110 |     def __bool__(self): return _htfe.vectorld___bool__(self)
111 |     def __len__(self): return _htfe.vectorld___len__(self)
112 |     def pop(self): return _htfe.vectorld_pop(self)
113 |     def __getslice__(self, *args): return _htfe.vectorld___getslice__(self, *args)
114 |     def __setslice__(self, *args): return _htfe.vectorld___setslice__(self, *args)
115 |     def __delslice__(self, *args): return _htfe.vectorld___delslice__(self, *args)
116 |     def __delitem__(self, *args): return _htfe.vectorld___delitem__(self, *args)
117 |     def __getitem__(self, *args): return _htfe.vectorld___getitem__(self, *args)
118 |     def __setitem__(self, *args): return _htfe.vectorld___setitem__(self, *args)
119 |     def append(self, *args): return _htfe.vectorld_append(self, *args)
120 |     def empty(self): return _htfe.vectorld_empty(self)
121 |     def size(self): return _htfe.vectorld_size(self)
122 |     def clear(self): return _htfe.vectorld_clear(self)
123 |     def swap(self, *args): return _htfe.vectorld_swap(self, *args)
124 |     def get_allocator(self): return _htfe.vectorld_get_allocator(self)
125 |     def begin(self): return _htfe.vectorld_begin(self)
126 |     def end(self): return _htfe.vectorld_end(self)
127 |     def rbegin(self): return _htfe.vectorld_rbegin(self)
128 |     def rend(self): return _htfe.vectorld_rend(self)
129 |     def pop_back(self): return _htfe.vectorld_pop_back(self)
130 |     def erase(self, *args): return _htfe.vectorld_erase(self, *args)
131 |     def __init__(self, *args): 
132 |         this = _htfe.new_vectorld(*args)
133 |         try: self.this.append(this)
134 |         except: self.this = this
135 |     def push_back(self, *args): return _htfe.vectorld_push_back(self, *args)
136 |     def front(self): return _htfe.vectorld_front(self)
137 |     def back(self): return _htfe.vectorld_back(self)
138 |     def assign(self, *args): return _htfe.vectorld_assign(self, *args)
139 |     def resize(self, *args): return _htfe.vectorld_resize(self, *args)
140 |     def insert(self, *args): return _htfe.vectorld_insert(self, *args)
141 |     def reserve(self, *args): return _htfe.vectorld_reserve(self, *args)
142 |     def capacity(self): return _htfe.vectorld_capacity(self)
143 |     __swig_destroy__ = _htfe.delete_vectorld
144 |     __del__ = lambda self : None;
145 | vectorld_swigregister = _htfe.vectorld_swigregister
146 | vectorld_swigregister(vectorld)
147 | 
148 | class LayerDesc(_object):
149 |     __swig_setmethods__ = {}
150 |     __setattr__ = lambda self, name, value: _swig_setattr(self, LayerDesc, name, value)
151 |     __swig_getmethods__ = {}
152 |     __getattr__ = lambda self, name: _swig_getattr(self, LayerDesc, name)
153 |     __repr__ = _swig_repr
154 |     __swig_setmethods__["_spatialWidth"] = _htfe.LayerDesc__spatialWidth_set
155 |     __swig_getmethods__["_spatialWidth"] = _htfe.LayerDesc__spatialWidth_get
156 |     if _newclass:_spatialWidth = _swig_property(_htfe.LayerDesc__spatialWidth_get, _htfe.LayerDesc__spatialWidth_set)
157 |     __swig_setmethods__["_spatialHeight"] = _htfe.LayerDesc__spatialHeight_set
158 |     __swig_getmethods__["_spatialHeight"] = _htfe.LayerDesc__spatialHeight_get
159 |     if _newclass:_spatialHeight = _swig_property(_htfe.LayerDesc__spatialHeight_get, _htfe.LayerDesc__spatialHeight_set)
160 |     __swig_setmethods__["_temporalWidth"] = _htfe.LayerDesc__temporalWidth_set
161 |     __swig_getmethods__["_temporalWidth"] = _htfe.LayerDesc__temporalWidth_get
162 |     if _newclass:_temporalWidth = _swig_property(_htfe.LayerDesc__temporalWidth_get, _htfe.LayerDesc__temporalWidth_set)
163 |     __swig_setmethods__["_temporalHeight"] = _htfe.LayerDesc__temporalHeight_set
164 |     __swig_getmethods__["_temporalHeight"] = _htfe.LayerDesc__temporalHeight_get
165 |     if _newclass:_temporalHeight = _swig_property(_htfe.LayerDesc__temporalHeight_get, _htfe.LayerDesc__temporalHeight_set)
166 |     __swig_setmethods__["_receptiveFieldRadius"] = _htfe.LayerDesc__receptiveFieldRadius_set
167 |     __swig_getmethods__["_receptiveFieldRadius"] = _htfe.LayerDesc__receptiveFieldRadius_get
168 |     if _newclass:_receptiveFieldRadius = _swig_property(_htfe.LayerDesc__receptiveFieldRadius_get, _htfe.LayerDesc__receptiveFieldRadius_set)
169 |     __swig_setmethods__["_reconstructionRadius"] = _htfe.LayerDesc__reconstructionRadius_set
170 |     __swig_getmethods__["_reconstructionRadius"] = _htfe.LayerDesc__reconstructionRadius_get
171 |     if _newclass:_reconstructionRadius = _swig_property(_htfe.LayerDesc__reconstructionRadius_get, _htfe.LayerDesc__reconstructionRadius_set)
172 |     __swig_setmethods__["_predictiveRadius"] = _htfe.LayerDesc__predictiveRadius_set
173 |     __swig_getmethods__["_predictiveRadius"] = _htfe.LayerDesc__predictiveRadius_get
174 |     if _newclass:_predictiveRadius = _swig_property(_htfe.LayerDesc__predictiveRadius_get, _htfe.LayerDesc__predictiveRadius_set)
175 |     __swig_setmethods__["_lateralConnectionRadius"] = _htfe.LayerDesc__lateralConnectionRadius_set
176 |     __swig_getmethods__["_lateralConnectionRadius"] = _htfe.LayerDesc__lateralConnectionRadius_get
177 |     if _newclass:_lateralConnectionRadius = _swig_property(_htfe.LayerDesc__lateralConnectionRadius_get, _htfe.LayerDesc__lateralConnectionRadius_set)
178 |     __swig_setmethods__["_spatialInhibitionRadius"] = _htfe.LayerDesc__spatialInhibitionRadius_set
179 |     __swig_getmethods__["_spatialInhibitionRadius"] = _htfe.LayerDesc__spatialInhibitionRadius_get
180 |     if _newclass:_spatialInhibitionRadius = _swig_property(_htfe.LayerDesc__spatialInhibitionRadius_get, _htfe.LayerDesc__spatialInhibitionRadius_set)
181 |     __swig_setmethods__["_temporalInhibitionRadius"] = _htfe.LayerDesc__temporalInhibitionRadius_set
182 |     __swig_getmethods__["_temporalInhibitionRadius"] = _htfe.LayerDesc__temporalInhibitionRadius_get
183 |     if _newclass:_temporalInhibitionRadius = _swig_property(_htfe.LayerDesc__temporalInhibitionRadius_get, _htfe.LayerDesc__temporalInhibitionRadius_set)
184 |     __swig_setmethods__["_feedBackConnectionRadius"] = _htfe.LayerDesc__feedBackConnectionRadius_set
185 |     __swig_getmethods__["_feedBackConnectionRadius"] = _htfe.LayerDesc__feedBackConnectionRadius_get
186 |     if _newclass:_feedBackConnectionRadius = _swig_property(_htfe.LayerDesc__feedBackConnectionRadius_get, _htfe.LayerDesc__feedBackConnectionRadius_set)
187 |     __swig_setmethods__["_spatialSparsity"] = _htfe.LayerDesc__spatialSparsity_set
188 |     __swig_getmethods__["_spatialSparsity"] = _htfe.LayerDesc__spatialSparsity_get
189 |     if _newclass:_spatialSparsity = _swig_property(_htfe.LayerDesc__spatialSparsity_get, _htfe.LayerDesc__spatialSparsity_set)
190 |     __swig_setmethods__["_temporalSparsity"] = _htfe.LayerDesc__temporalSparsity_set
191 |     __swig_getmethods__["_temporalSparsity"] = _htfe.LayerDesc__temporalSparsity_get
192 |     if _newclass:_temporalSparsity = _swig_property(_htfe.LayerDesc__temporalSparsity_get, _htfe.LayerDesc__temporalSparsity_set)
193 |     __swig_setmethods__["_dutyCycleDecay"] = _htfe.LayerDesc__dutyCycleDecay_set
194 |     __swig_getmethods__["_dutyCycleDecay"] = _htfe.LayerDesc__dutyCycleDecay_get
195 |     if _newclass:_dutyCycleDecay = _swig_property(_htfe.LayerDesc__dutyCycleDecay_get, _htfe.LayerDesc__dutyCycleDecay_set)
196 |     __swig_setmethods__["_spatialAlpha"] = _htfe.LayerDesc__spatialAlpha_set
197 |     __swig_getmethods__["_spatialAlpha"] = _htfe.LayerDesc__spatialAlpha_get
198 |     if _newclass:_spatialAlpha = _swig_property(_htfe.LayerDesc__spatialAlpha_get, _htfe.LayerDesc__spatialAlpha_set)
199 |     __swig_setmethods__["_predictiveAlpha"] = _htfe.LayerDesc__predictiveAlpha_set
200 |     __swig_getmethods__["_predictiveAlpha"] = _htfe.LayerDesc__predictiveAlpha_get
201 |     if _newclass:_predictiveAlpha = _swig_property(_htfe.LayerDesc__predictiveAlpha_get, _htfe.LayerDesc__predictiveAlpha_set)
202 |     __swig_setmethods__["_lateralAlpha"] = _htfe.LayerDesc__lateralAlpha_set
203 |     __swig_getmethods__["_lateralAlpha"] = _htfe.LayerDesc__lateralAlpha_get
204 |     if _newclass:_lateralAlpha = _swig_property(_htfe.LayerDesc__lateralAlpha_get, _htfe.LayerDesc__lateralAlpha_set)
205 |     __swig_setmethods__["_feedBackAlpha"] = _htfe.LayerDesc__feedBackAlpha_set
206 |     __swig_getmethods__["_feedBackAlpha"] = _htfe.LayerDesc__feedBackAlpha_get
207 |     if _newclass:_feedBackAlpha = _swig_property(_htfe.LayerDesc__feedBackAlpha_get, _htfe.LayerDesc__feedBackAlpha_set)
208 |     __swig_setmethods__["_reconstructionAlpha"] = _htfe.LayerDesc__reconstructionAlpha_set
209 |     __swig_getmethods__["_reconstructionAlpha"] = _htfe.LayerDesc__reconstructionAlpha_get
210 |     if _newclass:_reconstructionAlpha = _swig_property(_htfe.LayerDesc__reconstructionAlpha_get, _htfe.LayerDesc__reconstructionAlpha_set)
211 |     __swig_setmethods__["_spatialLambda"] = _htfe.LayerDesc__spatialLambda_set
212 |     __swig_getmethods__["_spatialLambda"] = _htfe.LayerDesc__spatialLambda_get
213 |     if _newclass:_spatialLambda = _swig_property(_htfe.LayerDesc__spatialLambda_get, _htfe.LayerDesc__spatialLambda_set)
214 |     __swig_setmethods__["_temporalLambda"] = _htfe.LayerDesc__temporalLambda_set
215 |     __swig_getmethods__["_temporalLambda"] = _htfe.LayerDesc__temporalLambda_get
216 |     if _newclass:_temporalLambda = _swig_property(_htfe.LayerDesc__temporalLambda_get, _htfe.LayerDesc__temporalLambda_set)
217 |     __swig_setmethods__["_spatialMomentum"] = _htfe.LayerDesc__spatialMomentum_set
218 |     __swig_getmethods__["_spatialMomentum"] = _htfe.LayerDesc__spatialMomentum_get
219 |     if _newclass:_spatialMomentum = _swig_property(_htfe.LayerDesc__spatialMomentum_get, _htfe.LayerDesc__spatialMomentum_set)
220 |     __swig_setmethods__["_predictiveMomentum"] = _htfe.LayerDesc__predictiveMomentum_set
221 |     __swig_getmethods__["_predictiveMomentum"] = _htfe.LayerDesc__predictiveMomentum_get
222 |     if _newclass:_predictiveMomentum = _swig_property(_htfe.LayerDesc__predictiveMomentum_get, _htfe.LayerDesc__predictiveMomentum_set)
223 |     __swig_setmethods__["_lateralMomentum"] = _htfe.LayerDesc__lateralMomentum_set
224 |     __swig_getmethods__["_lateralMomentum"] = _htfe.LayerDesc__lateralMomentum_get
225 |     if _newclass:_lateralMomentum = _swig_property(_htfe.LayerDesc__lateralMomentum_get, _htfe.LayerDesc__lateralMomentum_set)
226 |     __swig_setmethods__["_feedBackMomentum"] = _htfe.LayerDesc__feedBackMomentum_set
227 |     __swig_getmethods__["_feedBackMomentum"] = _htfe.LayerDesc__feedBackMomentum_get
228 |     if _newclass:_feedBackMomentum = _swig_property(_htfe.LayerDesc__feedBackMomentum_get, _htfe.LayerDesc__feedBackMomentum_set)
229 |     __swig_setmethods__["_reconstructionMomentum"] = _htfe.LayerDesc__reconstructionMomentum_set
230 |     __swig_getmethods__["_reconstructionMomentum"] = _htfe.LayerDesc__reconstructionMomentum_get
231 |     if _newclass:_reconstructionMomentum = _swig_property(_htfe.LayerDesc__reconstructionMomentum_get, _htfe.LayerDesc__reconstructionMomentum_set)
232 |     __swig_setmethods__["_lateralScalar"] = _htfe.LayerDesc__lateralScalar_set
233 |     __swig_getmethods__["_lateralScalar"] = _htfe.LayerDesc__lateralScalar_get
234 |     if _newclass:_lateralScalar = _swig_property(_htfe.LayerDesc__lateralScalar_get, _htfe.LayerDesc__lateralScalar_set)
235 |     __swig_setmethods__["_feedBackScalar"] = _htfe.LayerDesc__feedBackScalar_set
236 |     __swig_getmethods__["_feedBackScalar"] = _htfe.LayerDesc__feedBackScalar_get
237 |     if _newclass:_feedBackScalar = _swig_property(_htfe.LayerDesc__feedBackScalar_get, _htfe.LayerDesc__feedBackScalar_set)
238 |     __swig_setmethods__["_blurKernelWidth"] = _htfe.LayerDesc__blurKernelWidth_set
239 |     __swig_getmethods__["_blurKernelWidth"] = _htfe.LayerDesc__blurKernelWidth_get
240 |     if _newclass:_blurKernelWidth = _swig_property(_htfe.LayerDesc__blurKernelWidth_get, _htfe.LayerDesc__blurKernelWidth_set)
241 |     __swig_setmethods__["_numBlurPasses"] = _htfe.LayerDesc__numBlurPasses_set
242 |     __swig_getmethods__["_numBlurPasses"] = _htfe.LayerDesc__numBlurPasses_get
243 |     if _newclass:_numBlurPasses = _swig_property(_htfe.LayerDesc__numBlurPasses_get, _htfe.LayerDesc__numBlurPasses_set)
244 |     __swig_setmethods__["_gaussianNoise"] = _htfe.LayerDesc__gaussianNoise_set
245 |     __swig_getmethods__["_gaussianNoise"] = _htfe.LayerDesc__gaussianNoise_get
246 |     if _newclass:_gaussianNoise = _swig_property(_htfe.LayerDesc__gaussianNoise_get, _htfe.LayerDesc__gaussianNoise_set)
247 |     __swig_setmethods__["_minDerivative"] = _htfe.LayerDesc__minDerivative_set
248 |     __swig_getmethods__["_minDerivative"] = _htfe.LayerDesc__minDerivative_get
249 |     if _newclass:_minDerivative = _swig_property(_htfe.LayerDesc__minDerivative_get, _htfe.LayerDesc__minDerivative_set)
250 |     def __init__(self): 
251 |         this = _htfe.new_LayerDesc()
252 |         try: self.this.append(this)
253 |         except: self.this = this
254 |     __swig_destroy__ = _htfe.delete_LayerDesc
255 |     __del__ = lambda self : None;
256 | LayerDesc_swigregister = _htfe.LayerDesc_swigregister
257 | LayerDesc_swigregister(LayerDesc)
258 | 
259 | class Layer(_object):
260 |     __swig_setmethods__ = {}
261 |     __setattr__ = lambda self, name, value: _swig_setattr(self, Layer, name, value)
262 |     __swig_getmethods__ = {}
263 |     __getattr__ = lambda self, name: _swig_getattr(self, Layer, name)
264 |     __repr__ = _swig_repr
265 |     __swig_setmethods__["_hiddenActivationsSpatial"] = _htfe.Layer__hiddenActivationsSpatial_set
266 |     __swig_getmethods__["_hiddenActivationsSpatial"] = _htfe.Layer__hiddenActivationsSpatial_get
267 |     if _newclass:_hiddenActivationsSpatial = _swig_property(_htfe.Layer__hiddenActivationsSpatial_get, _htfe.Layer__hiddenActivationsSpatial_set)
268 |     __swig_setmethods__["_hiddenStatesSpatial"] = _htfe.Layer__hiddenStatesSpatial_set
269 |     __swig_getmethods__["_hiddenStatesSpatial"] = _htfe.Layer__hiddenStatesSpatial_get
270 |     if _newclass:_hiddenStatesSpatial = _swig_property(_htfe.Layer__hiddenStatesSpatial_get, _htfe.Layer__hiddenStatesSpatial_set)
271 |     __swig_setmethods__["_hiddenStatesSpatialPrev"] = _htfe.Layer__hiddenStatesSpatialPrev_set
272 |     __swig_getmethods__["_hiddenStatesSpatialPrev"] = _htfe.Layer__hiddenStatesSpatialPrev_get
273 |     if _newclass:_hiddenStatesSpatialPrev = _swig_property(_htfe.Layer__hiddenStatesSpatialPrev_get, _htfe.Layer__hiddenStatesSpatialPrev_set)
274 |     __swig_setmethods__["_hiddenActivationsTemporal"] = _htfe.Layer__hiddenActivationsTemporal_set
275 |     __swig_getmethods__["_hiddenActivationsTemporal"] = _htfe.Layer__hiddenActivationsTemporal_get
276 |     if _newclass:_hiddenActivationsTemporal = _swig_property(_htfe.Layer__hiddenActivationsTemporal_get, _htfe.Layer__hiddenActivationsTemporal_set)
277 |     __swig_setmethods__["_hiddenStatesTemporal"] = _htfe.Layer__hiddenStatesTemporal_set
278 |     __swig_getmethods__["_hiddenStatesTemporal"] = _htfe.Layer__hiddenStatesTemporal_get
279 |     if _newclass:_hiddenStatesTemporal = _swig_property(_htfe.Layer__hiddenStatesTemporal_get, _htfe.Layer__hiddenStatesTemporal_set)
280 |     __swig_setmethods__["_hiddenStatesTemporalPrev"] = _htfe.Layer__hiddenStatesTemporalPrev_set
281 |     __swig_getmethods__["_hiddenStatesTemporalPrev"] = _htfe.Layer__hiddenStatesTemporalPrev_get
282 |     if _newclass:_hiddenStatesTemporalPrev = _swig_property(_htfe.Layer__hiddenStatesTemporalPrev_get, _htfe.Layer__hiddenStatesTemporalPrev_set)
283 |     __swig_setmethods__["_hiddenStatesTemporalPrevPrev"] = _htfe.Layer__hiddenStatesTemporalPrevPrev_set
284 |     __swig_getmethods__["_hiddenStatesTemporalPrevPrev"] = _htfe.Layer__hiddenStatesTemporalPrevPrev_get
285 |     if _newclass:_hiddenStatesTemporalPrevPrev = _swig_property(_htfe.Layer__hiddenStatesTemporalPrevPrev_get, _htfe.Layer__hiddenStatesTemporalPrevPrev_set)
286 |     __swig_setmethods__["_spatialWeights"] = _htfe.Layer__spatialWeights_set
287 |     __swig_getmethods__["_spatialWeights"] = _htfe.Layer__spatialWeights_get
288 |     if _newclass:_spatialWeights = _swig_property(_htfe.Layer__spatialWeights_get, _htfe.Layer__spatialWeights_set)
289 |     __swig_setmethods__["_spatialWeightsPrev"] = _htfe.Layer__spatialWeightsPrev_set
290 |     __swig_getmethods__["_spatialWeightsPrev"] = _htfe.Layer__spatialWeightsPrev_get
291 |     if _newclass:_spatialWeightsPrev = _swig_property(_htfe.Layer__spatialWeightsPrev_get, _htfe.Layer__spatialWeightsPrev_set)
292 |     __swig_setmethods__["_spatialPredictiveReconstructionWeights"] = _htfe.Layer__spatialPredictiveReconstructionWeights_set
293 |     __swig_getmethods__["_spatialPredictiveReconstructionWeights"] = _htfe.Layer__spatialPredictiveReconstructionWeights_get
294 |     if _newclass:_spatialPredictiveReconstructionWeights = _swig_property(_htfe.Layer__spatialPredictiveReconstructionWeights_get, _htfe.Layer__spatialPredictiveReconstructionWeights_set)
295 |     __swig_setmethods__["_spatialPredictiveReconstructionWeightsPrev"] = _htfe.Layer__spatialPredictiveReconstructionWeightsPrev_set
296 |     __swig_getmethods__["_spatialPredictiveReconstructionWeightsPrev"] = _htfe.Layer__spatialPredictiveReconstructionWeightsPrev_get
297 |     if _newclass:_spatialPredictiveReconstructionWeightsPrev = _swig_property(_htfe.Layer__spatialPredictiveReconstructionWeightsPrev_get, _htfe.Layer__spatialPredictiveReconstructionWeightsPrev_set)
298 |     __swig_setmethods__["_predictiveWeights"] = _htfe.Layer__predictiveWeights_set
299 |     __swig_getmethods__["_predictiveWeights"] = _htfe.Layer__predictiveWeights_get
300 |     if _newclass:_predictiveWeights = _swig_property(_htfe.Layer__predictiveWeights_get, _htfe.Layer__predictiveWeights_set)
301 |     __swig_setmethods__["_predictiveWeightsPrev"] = _htfe.Layer__predictiveWeightsPrev_set
302 |     __swig_getmethods__["_predictiveWeightsPrev"] = _htfe.Layer__predictiveWeightsPrev_get
303 |     if _newclass:_predictiveWeightsPrev = _swig_property(_htfe.Layer__predictiveWeightsPrev_get, _htfe.Layer__predictiveWeightsPrev_set)
304 |     __swig_setmethods__["_lateralWeights"] = _htfe.Layer__lateralWeights_set
305 |     __swig_getmethods__["_lateralWeights"] = _htfe.Layer__lateralWeights_get
306 |     if _newclass:_lateralWeights = _swig_property(_htfe.Layer__lateralWeights_get, _htfe.Layer__lateralWeights_set)
307 |     __swig_setmethods__["_lateralWeightsPrev"] = _htfe.Layer__lateralWeightsPrev_set
308 |     __swig_getmethods__["_lateralWeightsPrev"] = _htfe.Layer__lateralWeightsPrev_get
309 |     if _newclass:_lateralWeightsPrev = _swig_property(_htfe.Layer__lateralWeightsPrev_get, _htfe.Layer__lateralWeightsPrev_set)
310 |     __swig_setmethods__["_feedBackWeights"] = _htfe.Layer__feedBackWeights_set
311 |     __swig_getmethods__["_feedBackWeights"] = _htfe.Layer__feedBackWeights_get
312 |     if _newclass:_feedBackWeights = _swig_property(_htfe.Layer__feedBackWeights_get, _htfe.Layer__feedBackWeights_set)
313 |     __swig_setmethods__["_feedBackWeightsPrev"] = _htfe.Layer__feedBackWeightsPrev_set
314 |     __swig_getmethods__["_feedBackWeightsPrev"] = _htfe.Layer__feedBackWeightsPrev_get
315 |     if _newclass:_feedBackWeightsPrev = _swig_property(_htfe.Layer__feedBackWeightsPrev_get, _htfe.Layer__feedBackWeightsPrev_set)
316 |     __swig_setmethods__["_spatialReconstruction"] = _htfe.Layer__spatialReconstruction_set
317 |     __swig_getmethods__["_spatialReconstruction"] = _htfe.Layer__spatialReconstruction_get
318 |     if _newclass:_spatialReconstruction = _swig_property(_htfe.Layer__spatialReconstruction_get, _htfe.Layer__spatialReconstruction_set)
319 |     __swig_setmethods__["_spatialReconstructionPrev"] = _htfe.Layer__spatialReconstructionPrev_set
320 |     __swig_getmethods__["_spatialReconstructionPrev"] = _htfe.Layer__spatialReconstructionPrev_get
321 |     if _newclass:_spatialReconstructionPrev = _swig_property(_htfe.Layer__spatialReconstructionPrev_get, _htfe.Layer__spatialReconstructionPrev_set)
322 |     __swig_setmethods__["_temporalReconstruction"] = _htfe.Layer__temporalReconstruction_set
323 |     __swig_getmethods__["_temporalReconstruction"] = _htfe.Layer__temporalReconstruction_get
324 |     if _newclass:_temporalReconstruction = _swig_property(_htfe.Layer__temporalReconstruction_get, _htfe.Layer__temporalReconstruction_set)
325 |     __swig_setmethods__["_temporalReconstructionPrev"] = _htfe.Layer__temporalReconstructionPrev_set
326 |     __swig_getmethods__["_temporalReconstructionPrev"] = _htfe.Layer__temporalReconstructionPrev_get
327 |     if _newclass:_temporalReconstructionPrev = _swig_property(_htfe.Layer__temporalReconstructionPrev_get, _htfe.Layer__temporalReconstructionPrev_set)
328 |     __swig_setmethods__["_nextTemporalReconstruction"] = _htfe.Layer__nextTemporalReconstruction_set
329 |     __swig_getmethods__["_nextTemporalReconstruction"] = _htfe.Layer__nextTemporalReconstruction_get
330 |     if _newclass:_nextTemporalReconstruction = _swig_property(_htfe.Layer__nextTemporalReconstruction_get, _htfe.Layer__nextTemporalReconstruction_set)
331 |     __swig_setmethods__["_nextTemporalReconstructionPrev"] = _htfe.Layer__nextTemporalReconstructionPrev_set
332 |     __swig_getmethods__["_nextTemporalReconstructionPrev"] = _htfe.Layer__nextTemporalReconstructionPrev_get
333 |     if _newclass:_nextTemporalReconstructionPrev = _swig_property(_htfe.Layer__nextTemporalReconstructionPrev_get, _htfe.Layer__nextTemporalReconstructionPrev_set)
334 |     __swig_setmethods__["_predictedSpatial"] = _htfe.Layer__predictedSpatial_set
335 |     __swig_getmethods__["_predictedSpatial"] = _htfe.Layer__predictedSpatial_get
336 |     if _newclass:_predictedSpatial = _swig_property(_htfe.Layer__predictedSpatial_get, _htfe.Layer__predictedSpatial_set)
337 |     __swig_setmethods__["_predictedSpatialPrev"] = _htfe.Layer__predictedSpatialPrev_set
338 |     __swig_getmethods__["_predictedSpatialPrev"] = _htfe.Layer__predictedSpatialPrev_get
339 |     if _newclass:_predictedSpatialPrev = _swig_property(_htfe.Layer__predictedSpatialPrev_get, _htfe.Layer__predictedSpatialPrev_set)
340 |     __swig_setmethods__["_inputReconstruction"] = _htfe.Layer__inputReconstruction_set
341 |     __swig_getmethods__["_inputReconstruction"] = _htfe.Layer__inputReconstruction_get
342 |     if _newclass:_inputReconstruction = _swig_property(_htfe.Layer__inputReconstruction_get, _htfe.Layer__inputReconstruction_set)
343 |     __swig_setmethods__["_predictedInputReconstruction"] = _htfe.Layer__predictedInputReconstruction_set
344 |     __swig_getmethods__["_predictedInputReconstruction"] = _htfe.Layer__predictedInputReconstruction_get
345 |     if _newclass:_predictedInputReconstruction = _swig_property(_htfe.Layer__predictedInputReconstruction_get, _htfe.Layer__predictedInputReconstruction_set)
346 |     def __init__(self): 
347 |         this = _htfe.new_Layer()
348 |         try: self.this.append(this)
349 |         except: self.this = this
350 |     __swig_destroy__ = _htfe.delete_Layer
351 |     __del__ = lambda self : None;
352 | Layer_swigregister = _htfe.Layer_swigregister
353 | Layer_swigregister(Layer)
354 | 
355 | class HTFE(_object):
356 |     __swig_setmethods__ = {}
357 |     __setattr__ = lambda self, name, value: _swig_setattr(self, HTFE, name, value)
358 |     __swig_getmethods__ = {}
359 |     __getattr__ = lambda self, name: _swig_getattr(self, HTFE, name)
360 |     __repr__ = _swig_repr
361 |     def createRandom(self, *args): return _htfe.HTFE_createRandom(self, *args)
362 |     def activate(self, *args): return _htfe.HTFE_activate(self, *args)
363 |     def learn(self, *args): return _htfe.HTFE_learn(self, *args)
364 |     def stepEnd(self): return _htfe.HTFE_stepEnd(self)
365 |     def getInputWidth(self): return _htfe.HTFE_getInputWidth(self)
366 |     def getInputHeight(self): return _htfe.HTFE_getInputHeight(self)
367 |     def getLayerDescs(self): return _htfe.HTFE_getLayerDescs(self)
368 |     def getLayers(self): return _htfe.HTFE_getLayers(self)
369 |     def getInputImage(self): return _htfe.HTFE_getInputImage(self)
370 |     def setInput(self, *args): return _htfe.HTFE_setInput(self, *args)
371 |     def getPrediction(self, *args): return _htfe.HTFE_getPrediction(self, *args)
372 |     def __init__(self): 
373 |         this = _htfe.new_HTFE()
374 |         try: self.this.append(this)
375 |         except: self.this = this
376 |     __swig_destroy__ = _htfe.delete_HTFE
377 |     __del__ = lambda self : None;
378 | HTFE_swigregister = _htfe.HTFE_swigregister
379 | HTFE_swigregister(HTFE)
380 | 
381 | SYS_ALLOW_CL_GL_CONTEXT = _htfe.SYS_ALLOW_CL_GL_CONTEXT
382 | _cpu = _htfe._cpu
383 | _gpu = _htfe._gpu
384 | _all = _htfe._all
385 | _none = _htfe._none
386 | class ComputeSystem(_object):
387 |     __swig_setmethods__ = {}
388 |     __setattr__ = lambda self, name, value: _swig_setattr(self, ComputeSystem, name, value)
389 |     __swig_getmethods__ = {}
390 |     __getattr__ = lambda self, name: _swig_getattr(self, ComputeSystem, name)
391 |     __repr__ = _swig_repr
392 |     def create(self, *args): return _htfe.ComputeSystem_create(self, *args)
393 |     def getPlatform(self): return _htfe.ComputeSystem_getPlatform(self)
394 |     def getDevice(self): return _htfe.ComputeSystem_getDevice(self)
395 |     def getContext(self): return _htfe.ComputeSystem_getContext(self)
396 |     def getQueue(self): return _htfe.ComputeSystem_getQueue(self)
397 |     def __init__(self): 
398 |         this = _htfe.new_ComputeSystem()
399 |         try: self.this.append(this)
400 |         except: self.this = this
401 |     __swig_destroy__ = _htfe.delete_ComputeSystem
402 |     __del__ = lambda self : None;
403 | ComputeSystem_swigregister = _htfe.ComputeSystem_swigregister
404 | ComputeSystem_swigregister(ComputeSystem)
405 | 
406 | class ComputeProgram(_object):
407 |     __swig_setmethods__ = {}
408 |     __setattr__ = lambda self, name, value: _swig_setattr(self, ComputeProgram, name, value)
409 |     __swig_getmethods__ = {}
410 |     __getattr__ = lambda self, name: _swig_getattr(self, ComputeProgram, name)
411 |     __repr__ = _swig_repr
412 |     def loadFromFile(self, *args): return _htfe.ComputeProgram_loadFromFile(self, *args)
413 |     def getProgram(self): return _htfe.ComputeProgram_getProgram(self)
414 |     def __init__(self): 
415 |         this = _htfe.new_ComputeProgram()
416 |         try: self.this.append(this)
417 |         except: self.this = this
418 |     __swig_destroy__ = _htfe.delete_ComputeProgram
419 |     __del__ = lambda self : None;
420 | ComputeProgram_swigregister = _htfe.ComputeProgram_swigregister
421 | ComputeProgram_swigregister(ComputeProgram)
422 | 
423 | # This file is compatible with both classic and new-style classes.
424 | 
425 | 
426 | 


--------------------------------------------------------------------------------
/source/htfe/HTFE.cpp:
--------------------------------------------------------------------------------
  1 | #include "HTFE.h"
  2 | 
  3 | #include <iostream>
  4 | #include <time.h>
  5 | 
  6 | using namespace htfe;
  7 | 
  8 | struct Uint2 {
  9 | 	unsigned int _x, _y;
 10 | };
 11 | 
 12 | struct Float2 {
 13 | 	float _x, _y;
 14 | };
 15 | 
 16 | struct Float4 {
 17 | 	float _x, _y, _z, _w;
 18 | };
 19 | 
 20 | struct Int2 {
 21 | 	int _x, _y;
 22 | };
 23 | 	
 24 | void HTFE::createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, int inputWidth, int inputHeight, const std::vector<LayerDesc> &layerDescs, float minInitWeight, float maxInitWeight) {
 25 | 	std::mt19937 generator(time(nullptr));
 26 | 
 27 | 	std::uniform_int_distribution<int> seedDist(0, 99999);
 28 | 
 29 | 	_inputWidth = inputWidth;
 30 | 	_inputHeight = inputHeight;
 31 | 
 32 | 	_layerDescs = layerDescs;
 33 | 
 34 | 	_layers.resize(_layerDescs.size());
 35 | 
 36 | 	cl::Kernel initializeLayerHiddenKernel = cl::Kernel(program.getProgram(), "initializeLayerHidden");
 37 | 	cl::Kernel initializeLayerVisibleKernel = cl::Kernel(program.getProgram(), "initializeLayerVisible");
 38 | 
 39 | 	_input.clear();
 40 | 	_input.resize(_inputWidth * _inputHeight, 0.0f);
 41 | 
 42 | 	_prediction.clear();
 43 | 	_prediction.resize(_inputWidth * _inputHeight, 0.0f);
 44 | 
 45 | 	_inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _inputWidth, _inputHeight);
 46 | 	_inputImagePrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _inputWidth, _inputHeight);
 47 | 
 48 | 	{
 49 | 		cl_uint4 clear = { 0, 0, 0, 0 };
 50 | 
 51 | 		cl::size_t<3> origin;
 52 | 		origin[0] = 0;
 53 | 		origin[1] = 0;
 54 | 		origin[2] = 0;
 55 | 
 56 | 		cl::size_t<3> region;
 57 | 		region[0] = _inputWidth;
 58 | 		region[1] = _inputHeight;
 59 | 		region[2] = 1;
 60 | 
 61 | 		cs.getQueue().enqueueFillImage(_inputImage, clear, origin, region);
 62 | 		cs.getQueue().enqueueFillImage(_inputImagePrev, clear, origin, region);
 63 | 	}
 64 | 
 65 | 	int prevWidth = _inputWidth;
 66 | 	int prevHeight = _inputHeight;
 67 | 
 68 | 	for (int l = 0; l < _layers.size(); l++) {
 69 | 		int numFeedForwardWeights = std::pow(_layerDescs[l]._receptiveFieldRadius * 2 + 1, 2);
 70 | 		int numReconstructionWeights = std::pow(_layerDescs[l]._reconstructionRadius * 2 + 1, 2);
 71 | 		int numLateralWeights = std::pow(_layerDescs[l]._lateralConnectionRadius * 2 + 1, 2);
 72 | 		int numFeedBackWeights = std::pow(_layerDescs[l]._feedBackConnectionRadius * 2 + 1, 2);
 73 | 
 74 | 		_layers[l]._hiddenFeedForwardActivations = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_RG, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 75 | 		
 76 | 		_layers[l]._hiddenFeedBackActivations = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_RG, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 77 | 		_layers[l]._hiddenFeedBackActivationsPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_RG, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 78 | 
 79 | 		_layers[l]._hiddenStatesFeedForward = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 80 | 		_layers[l]._hiddenStatesFeedForwardPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 81 | 
 82 | 		_layers[l]._hiddenStatesFeedBack = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 83 | 		_layers[l]._hiddenStatesFeedBackPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 84 | 		_layers[l]._hiddenStatesFeedBackPrevPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 85 | 
 86 | 		_layers[l]._feedForwardWeights = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numFeedForwardWeights);
 87 | 		_layers[l]._feedForwardWeightsPrev = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numFeedForwardWeights);
 88 | 
 89 | 		_layers[l]._reconstructionWeights = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight, numReconstructionWeights);
 90 | 		_layers[l]._reconstructionWeightsPrev = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight, numReconstructionWeights);
 91 | 
 92 | 		_layers[l]._visibleBiases = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight);
 93 | 		_layers[l]._visibleBiasesPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight);
 94 | 
 95 | 		_layers[l]._hiddenBiases = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 96 | 		_layers[l]._hiddenBiasesPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height);
 97 | 
 98 | 		_layers[l]._lateralWeights = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numLateralWeights);
 99 | 		_layers[l]._lateralWeightsPrev = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numLateralWeights);
100 | 
101 | 		_layers[l]._feedBackWeights = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numFeedBackWeights);
102 | 		_layers[l]._feedBackWeightsPrev = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), _layerDescs[l]._width, _layerDescs[l]._height, numFeedBackWeights);
103 | 
104 | 		_layers[l]._visibleReconstruction = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight);
105 | 		_layers[l]._visibleReconstructionPrev = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevWidth, prevHeight);
106 | 
107 | 		// Initialize
108 | 		Uint2 initSeedHidden;
109 | 		initSeedHidden._x = seedDist(generator);
110 | 		initSeedHidden._y = seedDist(generator);
111 | 
112 | 		int index = 0;
113 | 
114 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._hiddenFeedForwardActivations);
115 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._hiddenFeedBackActivations);
116 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._hiddenStatesFeedForward);
117 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._feedForwardWeights);
118 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._hiddenBiases);
119 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._lateralWeights);
120 | 		initializeLayerHiddenKernel.setArg(index++, _layers[l]._feedBackWeights);
121 | 		initializeLayerHiddenKernel.setArg(index++, numFeedForwardWeights);
122 | 		initializeLayerHiddenKernel.setArg(index++, numLateralWeights);
123 | 		initializeLayerHiddenKernel.setArg(index++, numFeedBackWeights);
124 | 		initializeLayerHiddenKernel.setArg(index++, initSeedHidden);
125 | 		initializeLayerHiddenKernel.setArg(index++, _layerDescs[l]._sparsity);
126 | 		initializeLayerHiddenKernel.setArg(index++, _layerDescs[l]._lateralScalar);
127 | 		initializeLayerHiddenKernel.setArg(index++, _layerDescs[l]._feedBackScalar);
128 | 		initializeLayerHiddenKernel.setArg(index++, minInitWeight);
129 | 		initializeLayerHiddenKernel.setArg(index++, maxInitWeight);
130 | 
131 | 		cs.getQueue().enqueueNDRangeKernel(initializeLayerHiddenKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
132 | 
133 | 		Uint2 initSeedVisible;
134 | 		initSeedVisible._x = seedDist(generator);
135 | 		initSeedVisible._y = seedDist(generator);
136 | 
137 | 		index = 0;
138 | 
139 | 		initializeLayerVisibleKernel.setArg(index++, _layers[l]._visibleBiases);
140 | 		initializeLayerVisibleKernel.setArg(index++, _layers[l]._visibleReconstruction);
141 | 		initializeLayerVisibleKernel.setArg(index++, _layers[l]._reconstructionWeights);
142 | 		initializeLayerVisibleKernel.setArg(index++, numReconstructionWeights);
143 | 		initializeLayerVisibleKernel.setArg(index++, initSeedVisible);
144 | 		initializeLayerVisibleKernel.setArg(index++, minInitWeight);
145 | 		initializeLayerVisibleKernel.setArg(index++, maxInitWeight);
146 | 
147 | 		cs.getQueue().enqueueNDRangeKernel(initializeLayerVisibleKernel, cl::NullRange, cl::NDRange(prevWidth, prevHeight));
148 | 
149 | 		{
150 | 			cl::size_t<3> origin;
151 | 			origin[0] = 0;
152 | 			origin[1] = 0;
153 | 			origin[2] = 0;
154 | 
155 | 			cl::size_t<3> region;
156 | 			region[0] = _layerDescs[l]._width;
157 | 			region[1] = _layerDescs[l]._height;
158 | 			region[2] = 1;
159 | 
160 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenFeedBackActivations, _layers[l]._hiddenFeedBackActivationsPrev, origin, origin, region);
161 | 		}
162 | 
163 | 		{
164 | 			cl::size_t<3> origin;
165 | 			origin[0] = 0;
166 | 			origin[1] = 0;
167 | 			origin[2] = 0;
168 | 
169 | 			cl::size_t<3> region;
170 | 			region[0] = prevWidth;
171 | 			region[1] = prevHeight;
172 | 			region[2] = 1;
173 | 
174 | 			cs.getQueue().enqueueCopyImage(_layers[l]._visibleReconstruction, _layers[l]._visibleReconstructionPrev, origin, origin, region);
175 | 		}
176 | 
177 | 		{
178 | 			cl::size_t<3> origin;
179 | 			origin[0] = 0;
180 | 			origin[1] = 0;
181 | 			origin[2] = 0;
182 | 
183 | 			cl::size_t<3> region;
184 | 			region[0] = _layerDescs[l]._width;
185 | 			region[1] = _layerDescs[l]._height;
186 | 			region[2] = 1;
187 | 
188 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenStatesFeedForward, _layers[l]._hiddenStatesFeedForwardPrev, origin, origin, region);
189 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenStatesFeedForward, _layers[l]._hiddenStatesFeedBack, origin, origin, region);
190 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenStatesFeedForward, _layers[l]._hiddenStatesFeedBackPrev, origin, origin, region);
191 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenStatesFeedForward, _layers[l]._hiddenStatesFeedBackPrevPrev, origin, origin, region);
192 | 		}
193 | 
194 | 		{
195 | 			cl::size_t<3> origin;
196 | 			origin[0] = 0;
197 | 			origin[1] = 0;
198 | 			origin[2] = 0;
199 | 
200 | 			cl::size_t<3> region;
201 | 			region[0] = _layerDescs[l]._width;
202 | 			region[1] = _layerDescs[l]._height;
203 | 			region[2] = numFeedForwardWeights;
204 | 
205 | 			cs.getQueue().enqueueCopyImage(_layers[l]._feedForwardWeights, _layers[l]._feedForwardWeightsPrev, origin, origin, region);
206 | 		}
207 | 
208 | 		{
209 | 			cl::size_t<3> origin;
210 | 			origin[0] = 0;
211 | 			origin[1] = 0;
212 | 			origin[2] = 0;
213 | 
214 | 			cl::size_t<3> region;
215 | 			region[0] = prevWidth;
216 | 			region[1] = prevHeight;
217 | 			region[2] = 1;
218 | 
219 | 			cs.getQueue().enqueueCopyImage(_layers[l]._visibleBiases, _layers[l]._visibleBiasesPrev, origin, origin, region);
220 | 		}
221 | 
222 | 		{
223 | 			cl::size_t<3> origin;
224 | 			origin[0] = 0;
225 | 			origin[1] = 0;
226 | 			origin[2] = 0;
227 | 
228 | 			cl::size_t<3> region;
229 | 			region[0] = _layerDescs[l]._width;
230 | 			region[1] = _layerDescs[l]._height;
231 | 			region[2] = 1;
232 | 
233 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenBiases, _layers[l]._hiddenBiasesPrev, origin, origin, region);
234 | 		}
235 | 
236 | 		{
237 | 			cl::size_t<3> origin;
238 | 			origin[0] = 0;
239 | 			origin[1] = 0;
240 | 			origin[2] = 0;
241 | 
242 | 			cl::size_t<3> region;
243 | 			region[0] = _layerDescs[l]._width;
244 | 			region[1] = _layerDescs[l]._height;
245 | 			region[2] = numLateralWeights;
246 | 
247 | 			cs.getQueue().enqueueCopyImage(_layers[l]._lateralWeights, _layers[l]._lateralWeightsPrev, origin, origin, region);
248 | 		}
249 | 
250 | 		{
251 | 			cl::size_t<3> origin;
252 | 			origin[0] = 0;
253 | 			origin[1] = 0;
254 | 			origin[2] = 0;
255 | 
256 | 			cl::size_t<3> region;
257 | 			region[0] = _layerDescs[l]._width;
258 | 			region[1] = _layerDescs[l]._height;
259 | 			region[2] = numFeedBackWeights;
260 | 
261 | 			cs.getQueue().enqueueCopyImage(_layers[l]._feedBackWeights, _layers[l]._feedBackWeightsPrev, origin, origin, region);
262 | 		}
263 | 
264 | 		{
265 | 			cl::size_t<3> origin;
266 | 			origin[0] = 0;
267 | 			origin[1] = 0;
268 | 			origin[2] = 0;
269 | 
270 | 			cl::size_t<3> region;
271 | 			region[0] = prevWidth;
272 | 			region[1] = prevHeight;
273 | 			region[2] = numReconstructionWeights;
274 | 
275 | 			cs.getQueue().enqueueCopyImage(_layers[l]._reconstructionWeights, _layers[l]._reconstructionWeightsPrev, origin, origin, region);
276 | 		}
277 | 
278 | 		prevWidth = _layerDescs[l]._width;
279 | 		prevHeight = _layerDescs[l]._height;
280 | 	}
281 | 
282 | 	_layerHiddenFeedForwardActivateKernel = cl::Kernel(program.getProgram(), "layerHiddenFeedForwardActivate");
283 | 	_layerHiddenFeedBackActivateKernel = cl::Kernel(program.getProgram(), "layerHiddenFeedBackActivate");
284 | 	_layerHiddenInhibitKernel = cl::Kernel(program.getProgram(), "layerHiddenInhibit");
285 | 	_layerVisibleReconstructKernel = cl::Kernel(program.getProgram(), "layerVisibleReconstruct");
286 | 	_layerHiddenWeightUpdateKernel = cl::Kernel(program.getProgram(), "layerHiddenWeightUpdate");
287 | 	_layerHiddenWeightUpdateLastKernel = cl::Kernel(program.getProgram(), "layerHiddenWeightUpdateLast");
288 | 	_layerVisibleWeightUpdateKernel = cl::Kernel(program.getProgram(), "layerVisibleWeightUpdate");
289 | }
290 | 
291 | void HTFE::activate(sys::ComputeSystem &cs) {	
292 | 	{
293 | 		cl::size_t<3> origin;
294 | 		origin[0] = 0;
295 | 		origin[1] = 0;
296 | 		origin[2] = 0;
297 | 
298 | 		cl::size_t<3> region;
299 | 		region[0] = _inputWidth;
300 | 		region[1] = _inputHeight;
301 | 		region[2] = 1;
302 | 
303 | 		cs.getQueue().enqueueWriteImage(_inputImage, CL_TRUE, origin, region, 0, 0, _input.data());
304 | 	}
305 | 	
306 | 	std::uniform_int_distribution<int> seedDist(0, 99999);
307 | 
308 | 	// ------------------------------------------------------------------------------
309 | 	// ------------------------------------ Go up -----------------------------------
310 | 	// ------------------------------------------------------------------------------
311 | 
312 | 	cl::Image2D* pPrevLayer = &_inputImage;
313 | 	int prevWidth = _inputWidth;
314 | 	int prevHeight = _inputHeight;
315 | 
316 | 	for (int l = 0; l < _layers.size(); l++) {
317 | 		float localActivity = std::round(_layerDescs[l]._sparsity * std::pow(2 * _layerDescs[l]._inhibitionRadius + 1, 2));
318 | 
319 | 		Int2 layerSize;
320 | 		layerSize._x = _layerDescs[l]._width;
321 | 		layerSize._y = _layerDescs[l]._height;
322 | 
323 | 		Int2 layerSizeMinusOne;
324 | 		layerSizeMinusOne._x = _layerDescs[l]._width - 1;
325 | 		layerSizeMinusOne._y = _layerDescs[l]._height - 1;
326 | 
327 | 		Float2 layerSizeMinusOneInv;
328 | 		layerSizeMinusOneInv._x = 1.0f / (_layerDescs[l]._width - 1);
329 | 		layerSizeMinusOneInv._y = 1.0f / (_layerDescs[l]._height - 1);
330 | 
331 | 		Int2 inputSize;
332 | 		inputSize._x = prevWidth;
333 | 		inputSize._y = prevHeight;
334 | 
335 | 		Int2 inputSizeMinusOne;
336 | 		inputSizeMinusOne._x = prevWidth - 1;
337 | 		inputSizeMinusOne._y = prevHeight - 1;
338 | 
339 | 		Float2 inputSizeMinusOneInv;
340 | 		inputSizeMinusOneInv._x = 1.0f / (prevWidth - 1);
341 | 		inputSizeMinusOneInv._y = 1.0f / (prevHeight - 1);
342 | 
343 | 		// -------------------------------- Activate --------------------------------
344 | 
345 | 		int index = 0;
346 | 
347 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, *pPrevLayer);
348 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrev);
349 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layers[l]._feedForwardWeightsPrev);
350 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layers[l]._lateralWeightsPrev);
351 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layers[l]._hiddenBiasesPrev);
352 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layers[l]._hiddenFeedForwardActivations);
353 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, layerSize);
354 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, layerSizeMinusOneInv);
355 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, inputSize);
356 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, inputSizeMinusOne);
357 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layerDescs[l]._receptiveFieldRadius);
358 | 		_layerHiddenFeedForwardActivateKernel.setArg(index++, _layerDescs[l]._lateralConnectionRadius);
359 | 
360 | 		cs.getQueue().enqueueNDRangeKernel(_layerHiddenFeedForwardActivateKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
361 | 
362 | 		// ---------------------------------- Inhibit ---------------------------------
363 | 
364 | 		index = 0;
365 | 
366 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenFeedForwardActivations);
367 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenStatesFeedForwardPrev);
368 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenStatesFeedForward);
369 | 		_layerHiddenInhibitKernel.setArg(index++, layerSize);
370 | 		_layerHiddenInhibitKernel.setArg(index++, _layerDescs[l]._inhibitionRadius);
371 | 		_layerHiddenInhibitKernel.setArg(index++, localActivity);
372 | 
373 | 		cs.getQueue().enqueueNDRangeKernel(_layerHiddenInhibitKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
374 | 
375 | 		pPrevLayer = &_layers[l]._hiddenStatesFeedForward;
376 | 		prevWidth = _layerDescs[l]._width;
377 | 		prevHeight = _layerDescs[l]._height;
378 | 	}
379 | 
380 | 	// ------------------------------------------------------------------------------
381 | 	// -------------------------------- Go back down --------------------------------
382 | 	// ------------------------------------------------------------------------------
383 | 
384 | 	for (int l = _layers.size() - 1; l >= 0; l--) {
385 | 		if (l > 0) {
386 | 			pPrevLayer = &_layers[l - 1]._hiddenStatesFeedForward;
387 | 			prevWidth = _layerDescs[l - 1]._width;
388 | 			prevHeight = _layerDescs[l - 1]._height;
389 | 		}
390 | 		else {
391 | 			pPrevLayer = &_inputImage;
392 | 			prevWidth = _inputWidth;
393 | 			prevHeight = _inputHeight;
394 | 		}
395 | 
396 | 		float localActivity = std::round(_layerDescs[l]._sparsity * std::pow(2 * _layerDescs[l]._inhibitionRadius + 1, 2));
397 | 
398 | 		Int2 layerSize;
399 | 		layerSize._x = _layerDescs[l]._width;
400 | 		layerSize._y = _layerDescs[l]._height;
401 | 
402 | 		Int2 layerSizeMinusOne;
403 | 		layerSizeMinusOne._x = _layerDescs[l]._width - 1;
404 | 		layerSizeMinusOne._y = _layerDescs[l]._height - 1;
405 | 
406 | 		Float2 layerSizeMinusOneInv;
407 | 		layerSizeMinusOneInv._x = 1.0f / (_layerDescs[l]._width - 1);
408 | 		layerSizeMinusOneInv._y = 1.0f / (_layerDescs[l]._height - 1);
409 | 
410 | 		Int2 inputSize;
411 | 		inputSize._x = prevWidth;
412 | 		inputSize._y = prevHeight;
413 | 
414 | 		Int2 inputSizeMinusOne;
415 | 		inputSizeMinusOne._x = prevWidth - 1;
416 | 		inputSizeMinusOne._y = prevHeight - 1;
417 | 
418 | 		Float2 inputSizeMinusOneInv;
419 | 		inputSizeMinusOneInv._x = 1.0f / (prevWidth - 1);
420 | 		inputSizeMinusOneInv._y = 1.0f / (prevHeight - 1);
421 | 
422 | 		Int2 nextSize;
423 | 		Int2 nextSizeMinusOne;
424 | 
425 | 		if (l == _layers.size() - 1) {
426 | 			nextSize._x = nextSize._y = 1;
427 | 			nextSizeMinusOne._x = nextSizeMinusOne._y = 0;
428 | 		}
429 | 		else {
430 | 			nextSize._x = _layerDescs[l + 1]._width;
431 | 			nextSize._y = _layerDescs[l + 1]._height;
432 | 			nextSizeMinusOne._x = _layerDescs[l + 1]._width - 1;
433 | 			nextSizeMinusOne._y = _layerDescs[l + 1]._height - 1;
434 | 		}
435 | 
436 | 		// -------------------------------- Activate --------------------------------
437 | 
438 | 		int index = 0;
439 | 
440 | 		if (l == _layers.size() - 1) {
441 | 			cl::size_t<3> origin;
442 | 			origin[0] = 0;
443 | 			origin[1] = 0;
444 | 			origin[2] = 0;
445 | 
446 | 			cl::size_t<3> region;
447 | 			region[0] = _layerDescs[l]._width;
448 | 			region[1] = _layerDescs[l]._height;
449 | 			region[2] = 1;
450 | 
451 | 			cs.getQueue().enqueueCopyImage(_layers[l]._hiddenFeedForwardActivations, _layers[l]._hiddenFeedBackActivations, origin, origin, region);
452 | 		}
453 | 		else {
454 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, _layers[l]._hiddenFeedForwardActivations);
455 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, _layers[l + 1]._hiddenFeedBackActivations);
456 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, _layers[l]._feedBackWeightsPrev);
457 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, _layers[l]._hiddenFeedBackActivations);
458 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, layerSize);
459 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, layerSizeMinusOneInv);
460 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, nextSize);
461 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, nextSizeMinusOne);
462 | 			_layerHiddenFeedBackActivateKernel.setArg(index++, _layerDescs[l]._feedBackConnectionRadius);
463 | 
464 | 			cs.getQueue().enqueueNDRangeKernel(_layerHiddenFeedBackActivateKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
465 | 		}
466 | 
467 | 		// ---------------------------------- Inhibit ---------------------------------
468 | 
469 | 		index = 0;
470 | 
471 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenFeedBackActivations);
472 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrev);
473 | 		_layerHiddenInhibitKernel.setArg(index++, _layers[l]._hiddenStatesFeedBack);
474 | 		_layerHiddenInhibitKernel.setArg(index++, layerSize);
475 | 		_layerHiddenInhibitKernel.setArg(index++, _layerDescs[l]._inhibitionRadius);
476 | 		_layerHiddenInhibitKernel.setArg(index++, localActivity);
477 | 
478 | 		cs.getQueue().enqueueNDRangeKernel(_layerHiddenInhibitKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
479 | 
480 | 		// --------------------- Make Predictions (Reconstruction) ---------------------
481 | 
482 | 		index = 0;
483 | 
484 | 		_layerVisibleReconstructKernel.setArg(index++, _layers[l]._hiddenStatesFeedBack);
485 | 		_layerVisibleReconstructKernel.setArg(index++, _layers[l]._reconstructionWeightsPrev);
486 | 		_layerVisibleReconstructKernel.setArg(index++, _layers[l]._visibleBiasesPrev);
487 | 		_layerVisibleReconstructKernel.setArg(index++, _layers[l]._visibleReconstruction);
488 | 		_layerVisibleReconstructKernel.setArg(index++, _layerDescs[l]._reconstructionRadius);
489 | 		_layerVisibleReconstructKernel.setArg(index++, inputSizeMinusOne);
490 | 		_layerVisibleReconstructKernel.setArg(index++, inputSizeMinusOneInv);
491 | 		_layerVisibleReconstructKernel.setArg(index++, layerSize);
492 | 		_layerVisibleReconstructKernel.setArg(index++, layerSizeMinusOne);
493 | 		_layerVisibleReconstructKernel.setArg(index++, layerSizeMinusOneInv);
494 | 
495 | 		cs.getQueue().enqueueNDRangeKernel(_layerVisibleReconstructKernel, cl::NullRange, cl::NDRange(prevWidth, prevHeight));
496 | 	}
497 | 
498 | 	{
499 | 		cl::size_t<3> origin;
500 | 		origin[0] = 0;
501 | 		origin[1] = 0;
502 | 		origin[2] = 0;
503 | 
504 | 		cl::size_t<3> region;
505 | 		region[0] = _inputWidth;
506 | 		region[1] = _inputHeight;
507 | 		region[2] = 1;
508 | 
509 | 		cs.getQueue().enqueueReadImage(_layers.front()._visibleReconstruction, CL_TRUE, origin, region, 0, 0, _prediction.data());
510 | 	}
511 | }
512 | 
513 | void HTFE::learn(sys::ComputeSystem &cs) {
514 | 	// ------------------------------------------------------------------------------
515 | 	// ---------------------- Weight Update and Predictions  ------------------------
516 | 	// ------------------------------------------------------------------------------
517 | 
518 | 	cl::Image2D* pPrevLayer = &_inputImage;
519 | 	int prevWidth = _inputWidth;
520 | 	int prevHeight = _inputHeight;
521 | 
522 | 	cl::Image2D* pPrevLayerFeedForwardPrev = &_inputImagePrev;
523 | 	cl::Image2D* pPrevLayerFeedBackPrev = &_inputImagePrev;
524 | 
525 | 	for (int l = 0; l < _layers.size(); l++) {
526 | 		float localActivity = std::round(_layerDescs[l]._sparsity * std::pow(2 * _layerDescs[l]._inhibitionRadius + 1, 2));
527 | 
528 | 		Int2 layerSize;
529 | 		layerSize._x = _layerDescs[l]._width;
530 | 		layerSize._y = _layerDescs[l]._height;
531 | 
532 | 		Int2 layerSizeMinusOne;
533 | 		layerSizeMinusOne._x = _layerDescs[l]._width - 1;
534 | 		layerSizeMinusOne._y = _layerDescs[l]._height - 1;
535 | 
536 | 		Float2 layerSizeMinusOneInv;
537 | 		layerSizeMinusOneInv._x = 1.0f / (_layerDescs[l]._width - 1);
538 | 		layerSizeMinusOneInv._y = 1.0f / (_layerDescs[l]._height - 1);
539 | 
540 | 		Int2 inputSize;
541 | 		inputSize._x = prevWidth;
542 | 		inputSize._y = prevHeight;
543 | 
544 | 		Int2 inputSizeMinusOne;
545 | 		inputSizeMinusOne._x = prevWidth - 1;
546 | 		inputSizeMinusOne._y = prevHeight - 1;
547 | 
548 | 		Float2 inputSizeMinusOneInv;
549 | 		inputSizeMinusOneInv._x = 1.0f / (prevWidth - 1);
550 | 		inputSizeMinusOneInv._y = 1.0f / (prevHeight - 1);
551 | 
552 | 		Int2 nextSize;
553 | 		Int2 nextSizeMinusOne;
554 | 
555 | 		if (l == _layers.size() - 1) {
556 | 			nextSize._x = nextSize._y = 1;
557 | 			nextSizeMinusOne._x = nextSizeMinusOne._y = 0;
558 | 		}
559 | 		else {
560 | 			nextSize._x = _layerDescs[l + 1]._width;
561 | 			nextSize._y = _layerDescs[l + 1]._height;
562 | 			nextSizeMinusOne._x = _layerDescs[l + 1]._width - 1;
563 | 			nextSizeMinusOne._y = _layerDescs[l + 1]._height - 1;
564 | 		}
565 | 
566 | 		// ------------------------------- Weight Updates -------------------------------
567 | 
568 | 		Float4 alphas;
569 | 		alphas._x = _layerDescs[l]._feedForwardAlpha;
570 | 		alphas._y = _layerDescs[l]._lateralAlpha;
571 | 		alphas._z = _layerDescs[l]._feedBackAlpha;
572 | 		alphas._w = _layerDescs[l]._hiddenBiasAlpha;
573 | 
574 | 		int index = 0;
575 | 
576 | 		if (l == _layers.size() - 1) {
577 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._visibleReconstructionPrev);
578 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, *pPrevLayer);
579 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, *pPrevLayerFeedForwardPrev);
580 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._hiddenFeedBackActivationsPrev);
581 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrev);
582 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrevPrev);
583 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._reconstructionWeightsPrev);
584 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._feedForwardWeightsPrev);
585 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._lateralWeightsPrev);
586 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._hiddenBiasesPrev);
587 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._feedForwardWeights);
588 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._lateralWeights);
589 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layers[l]._hiddenBiases);
590 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, layerSize);
591 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, layerSizeMinusOne);
592 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, layerSizeMinusOneInv);
593 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, inputSize);
594 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, inputSizeMinusOne);
595 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, inputSizeMinusOneInv);
596 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layerDescs[l]._receptiveFieldRadius);
597 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layerDescs[l]._lateralConnectionRadius);
598 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layerDescs[l]._reconstructionRadius);
599 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layerDescs[l]._sparsity);
600 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, alphas);
601 | 			_layerHiddenWeightUpdateLastKernel.setArg(index++, _layerDescs[l]._weightDecay);
602 | 
603 | 			cs.getQueue().enqueueNDRangeKernel(_layerHiddenWeightUpdateLastKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
604 | 		}
605 | 		else {
606 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._visibleReconstructionPrev);
607 | 			_layerHiddenWeightUpdateKernel.setArg(index++, *pPrevLayer);
608 | 			_layerHiddenWeightUpdateKernel.setArg(index++, *pPrevLayerFeedForwardPrev);
609 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._hiddenFeedBackActivationsPrev);
610 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrev);
611 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrevPrev);
612 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l + 1]._hiddenStatesFeedBackPrev);
613 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._reconstructionWeightsPrev);
614 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._feedForwardWeightsPrev);
615 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._lateralWeightsPrev);
616 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._hiddenBiasesPrev);
617 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._feedBackWeightsPrev);
618 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._feedForwardWeights);
619 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._lateralWeights);
620 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._hiddenBiases);
621 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layers[l]._feedBackWeights);
622 | 			_layerHiddenWeightUpdateKernel.setArg(index++, layerSize);
623 | 			_layerHiddenWeightUpdateKernel.setArg(index++, layerSizeMinusOne);
624 | 			_layerHiddenWeightUpdateKernel.setArg(index++, layerSizeMinusOneInv);
625 | 			_layerHiddenWeightUpdateKernel.setArg(index++, inputSize);
626 | 			_layerHiddenWeightUpdateKernel.setArg(index++, inputSizeMinusOne);
627 | 			_layerHiddenWeightUpdateKernel.setArg(index++, inputSizeMinusOneInv);
628 | 			_layerHiddenWeightUpdateKernel.setArg(index++, nextSize);
629 | 			_layerHiddenWeightUpdateKernel.setArg(index++, nextSizeMinusOne);
630 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._receptiveFieldRadius);
631 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._lateralConnectionRadius);
632 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._feedBackConnectionRadius);
633 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._reconstructionRadius);
634 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._sparsity);
635 | 			_layerHiddenWeightUpdateKernel.setArg(index++, alphas);
636 | 			_layerHiddenWeightUpdateKernel.setArg(index++, _layerDescs[l]._weightDecay);
637 | 
638 | 			cs.getQueue().enqueueNDRangeKernel(_layerHiddenWeightUpdateKernel, cl::NullRange, cl::NDRange(_layerDescs[l]._width, _layerDescs[l]._height));
639 | 		}
640 | 
641 | 		index = 0;
642 | 
643 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._visibleReconstructionPrev);
644 | 		_layerVisibleWeightUpdateKernel.setArg(index++, *pPrevLayer);
645 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._hiddenStatesFeedBackPrev);
646 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._reconstructionWeightsPrev);
647 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._visibleBiasesPrev);
648 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._reconstructionWeights);
649 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layers[l]._visibleBiases);
650 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layerDescs[l]._reconstructionRadius);
651 | 		_layerVisibleWeightUpdateKernel.setArg(index++, inputSizeMinusOne);
652 | 		_layerVisibleWeightUpdateKernel.setArg(index++, inputSizeMinusOneInv);
653 | 		_layerVisibleWeightUpdateKernel.setArg(index++, layerSize);
654 | 		_layerVisibleWeightUpdateKernel.setArg(index++, layerSizeMinusOne);
655 | 		_layerVisibleWeightUpdateKernel.setArg(index++, layerSizeMinusOneInv);
656 | 		_layerVisibleWeightUpdateKernel.setArg(index++, _layerDescs[l]._reconstructionAlpha);
657 | 
658 | 		cs.getQueue().enqueueNDRangeKernel(_layerVisibleWeightUpdateKernel, cl::NullRange, cl::NDRange(prevWidth, prevHeight));
659 | 
660 | 		pPrevLayer = &_layers[l]._hiddenStatesFeedForward; // Or _hiddenStatesFeedBack ?
661 | 		prevWidth = _layerDescs[l]._width;
662 | 		prevHeight = _layerDescs[l]._height;
663 | 
664 | 		pPrevLayerFeedForwardPrev = &_layers[l]._hiddenStatesFeedForwardPrev;
665 | 		pPrevLayerFeedBackPrev = &_layers[l]._hiddenStatesFeedBackPrev;
666 | 	}
667 | }
668 | 
669 | void HTFE::stepEnd() {
670 | 	// ------------------------------------------------------------------------------
671 | 	// ---------------------------------- Step End ----------------------------------
672 | 	// ------------------------------------------------------------------------------
673 | 
674 | 	for (int l = 0; l < _layers.size(); l++) {
675 | 		cl::Image2D temp2D;
676 | 
677 | 		std::swap(_layers[l]._visibleReconstruction, _layers[l]._visibleReconstructionPrev);
678 | 		std::swap(_layers[l]._hiddenFeedBackActivations, _layers[l]._hiddenFeedBackActivationsPrev);
679 | 		std::swap(_layers[l]._hiddenStatesFeedForward, _layers[l]._hiddenStatesFeedForwardPrev);
680 | 		
681 | 		temp2D = _layers[l]._hiddenStatesFeedBackPrevPrev;
682 | 		_layers[l]._hiddenStatesFeedBackPrevPrev = _layers[l]._hiddenStatesFeedBackPrev;
683 | 		_layers[l]._hiddenStatesFeedBackPrev = _layers[l]._hiddenStatesFeedBack;
684 | 		_layers[l]._hiddenStatesFeedBack = temp2D;
685 | 
686 | 		std::swap(_layers[l]._feedForwardWeights, _layers[l]._feedForwardWeightsPrev);
687 | 		std::swap(_layers[l]._reconstructionWeights, _layers[l]._reconstructionWeightsPrev);
688 | 		std::swap(_layers[l]._visibleBiases, _layers[l]._visibleBiasesPrev);
689 | 		std::swap(_layers[l]._hiddenBiases, _layers[l]._hiddenBiasesPrev);
690 | 		std::swap(_layers[l]._lateralWeights, _layers[l]._lateralWeightsPrev);
691 | 		std::swap(_layers[l]._feedBackWeights, _layers[l]._feedBackWeightsPrev);
692 | 	}
693 | 
694 | 	std::swap(_inputImage, _inputImagePrev);
695 | }
696 | 
697 | void HTFE::clearMemory(sys::ComputeSystem &cs) {
698 | 	// ------------------------------------------------------------------------------
699 | 	// -------------------------------- Clear Memory --------------------------------
700 | 	// ------------------------------------------------------------------------------
701 | 
702 | 	cl_uint4 clear = { 0, 0, 0, 0 };
703 | 
704 | 	for (int l = 0; l < _layers.size(); l++) {
705 | 		cl::size_t<3> origin;
706 | 		origin[0] = 0;
707 | 		origin[1] = 0;
708 | 		origin[2] = 0;
709 | 
710 | 		cl::size_t<3> region;
711 | 		region[0] = _layerDescs[l]._width;
712 | 		region[1] = _layerDescs[l]._height;
713 | 		region[2] = 1;
714 | 
715 | 		cs.getQueue().enqueueFillImage(_layers[l]._hiddenStatesFeedBackPrevPrev, clear, origin, region);
716 | 		cs.getQueue().enqueueFillImage(_layers[l]._hiddenStatesFeedBackPrev, clear, origin, region);
717 | 		cs.getQueue().enqueueFillImage(_layers[l]._hiddenStatesFeedBack, clear, origin, region);
718 | 	}
719 | }


--------------------------------------------------------------------------------