├── NeoRL ├── testImage.png ├── resources │ ├── arial.ttf │ ├── lineGradient.png │ ├── noreconstruction.png │ └── racing │ │ ├── racingCar.png │ │ ├── racingCollision.png │ │ ├── racingBackground.png │ │ ├── racingCheckpoints.png │ │ └── racingForeground.png ├── source │ ├── Audio_Generate.cpp │ ├── system │ │ ├── Uncopyable.h │ │ ├── ComputeProgram.h │ │ ├── ComputeProgram.cpp │ │ ├── ComputeSystem.h │ │ └── ComputeSystem.cpp │ ├── Settings.h │ ├── neo │ │ ├── ImageWhitener.cpp │ │ ├── ImageWhitener.h │ │ ├── Helpers.h │ │ ├── PredictiveHierarchy.h │ │ ├── PredictorSwarm.h │ │ ├── AgentPredQ.h │ │ ├── Predictor.h │ │ ├── Helpers.cpp │ │ ├── AgentSPG.h │ │ ├── SparseCoder.h │ │ ├── PredictiveHierarchy.cpp │ │ ├── Swarm.h │ │ ├── SparsePredictor.h │ │ ├── ComparisonSparseCoder.h │ │ ├── AgentSwarm.h │ │ ├── AgentER.h │ │ ├── AgentHA.h │ │ └── PredictorSwarm.cpp │ ├── vis │ │ ├── PrettySDR.h │ │ ├── Plot.h │ │ ├── PrettySDR.cpp │ │ └── Plot.cpp │ ├── COCO.cpp │ ├── SinusExample.cpp │ ├── runner │ │ └── Runner.h │ ├── deep │ │ ├── FERL.h │ │ ├── SDRRL.h │ │ └── SDRRL.cpp │ ├── SequenceRecall.cpp │ ├── RNNBenchmark.cpp │ ├── Binh_Test.cpp │ ├── TextPrediction.cpp │ ├── Balancer.cpp │ ├── TextTraining.cpp │ ├── dirent.h │ ├── NLevelGenerator.cpp │ └── MNIST_Video.cpp ├── CMakeLists.txt └── FindSFML.cmake ├── LICENSE.md ├── README.md └── .gitignore /NeoRL/testImage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/testImage.png -------------------------------------------------------------------------------- /NeoRL/resources/arial.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/arial.ttf -------------------------------------------------------------------------------- /NeoRL/resources/lineGradient.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/lineGradient.png -------------------------------------------------------------------------------- /NeoRL/source/Audio_Generate.cpp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/source/Audio_Generate.cpp -------------------------------------------------------------------------------- /NeoRL/resources/noreconstruction.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/noreconstruction.png -------------------------------------------------------------------------------- /NeoRL/resources/racing/racingCar.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/racing/racingCar.png -------------------------------------------------------------------------------- /NeoRL/resources/racing/racingCollision.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/racing/racingCollision.png -------------------------------------------------------------------------------- /NeoRL/resources/racing/racingBackground.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/racing/racingBackground.png -------------------------------------------------------------------------------- /NeoRL/resources/racing/racingCheckpoints.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/racing/racingCheckpoints.png -------------------------------------------------------------------------------- /NeoRL/resources/racing/racingForeground.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/222464/NeoRL/HEAD/NeoRL/resources/racing/racingForeground.png -------------------------------------------------------------------------------- /NeoRL/source/system/Uncopyable.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | namespace sys { 4 | /*! 5 | \brief Inherit from this class to make it Uncopyable 6 | */ 7 | class Uncopyable { 8 | protected: 9 | Uncopyable() {} 10 | virtual ~Uncopyable() {} 11 | private: 12 | Uncopyable(const Uncopyable &); 13 | Uncopyable &operator=(const Uncopyable &); 14 | }; 15 | } -------------------------------------------------------------------------------- /NeoRL/source/Settings.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #define EXPERIMENT_SPARSE_CODING 0 4 | #define EXPERIMENT_SINUS_EXAMPLE 1 5 | #define EXPERIMENT_BINH_TEST 2 6 | #define EXPERIMENT_MNIST_VIDEO 3 7 | #define EXPERIMENT_TEXT_PREDICTION 4 8 | #define EXPERIMENT_RNN_BENCHMARK 5 9 | #define EXPERIMENT_AUDIO_GENERATE 6 10 | #define EXPERIMENT_RUNNER 7 11 | #define EXPERIMENT_PONG 8 12 | #define EXPERIMENT_BALANCER 9 13 | #define EXPERIMENT_SEQUENCE_RECALL 10 14 | #define EXPERIMENT_COCO 11 15 | #define EXPERIMENT_TEXT_TRAINING 12 16 | #define EXPERIMENT_RACING 13 17 | #define EXPERIMENT_SLIME_VOLLEYBALL 14 18 | #define EXPERIMENT_N_LEVEL_GENERATOR 15 19 | 20 | #define EXPERIMENT_SELECTION EXPERIMENT_TEXT_PREDICTION 21 | -------------------------------------------------------------------------------- /NeoRL/source/system/ComputeProgram.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #include 6 | 7 | namespace sys { 8 | /*! 9 | \brief Compute program 10 | Holds OpenCL compute program with their associated kernels 11 | */ 12 | class ComputeProgram { 13 | private: 14 | /*! 15 | \brief OpenCL program 16 | */ 17 | cl::Program _program; 18 | 19 | public: 20 | /*! 21 | \brief Load from file 22 | Load program from a file 23 | */ 24 | bool loadFromFile(const std::string &name, ComputeSystem &cs); 25 | 26 | /*! 27 | \brief Get the underlying OpenCL program 28 | */ 29 | cl::Program &getProgram() { 30 | return _program; 31 | } 32 | }; 33 | } -------------------------------------------------------------------------------- /NeoRL/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | 3 | project(NeoRL) 4 | 5 | include_directories("${PROJECT_SOURCE_DIR}/source") 6 | 7 | # This is only required for the script to work in the version control 8 | set(CMAKE_MODULE_PATH "${PROJECT_SOURCE_DIR}") 9 | 10 | find_package(OpenCL REQUIRED) 11 | 12 | include_directories(${OpenCL_INCLUDE_DIRS}) 13 | 14 | find_package(SFML 2 COMPONENTS system window graphics audio network REQUIRED) 15 | 16 | include_directories(${SFML_INCLUDE_DIR}) 17 | 18 | file(GLOB_RECURSE LINK_SRC 19 | "source/*.h" 20 | "source/*.cpp" 21 | ) 22 | 23 | add_executable(NeoRL ${LINK_SRC}) 24 | 25 | target_link_libraries(NeoRL ${OpenCL_LIBRARIES}) 26 | target_link_libraries(NeoRL ${SFML_LIBRARIES}) -------------------------------------------------------------------------------- /NeoRL/source/neo/ImageWhitener.cpp: -------------------------------------------------------------------------------- 1 | #include "ImageWhitener.h" 2 | 3 | using namespace neo; 4 | 5 | void ImageWhitener::create(sys::ComputeSystem &cs, sys::ComputeProgram &program, cl_int2 imageSize, cl_int imageFormat, cl_int imageType) { 6 | _imageSize = imageSize; 7 | 8 | _result = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(imageFormat, imageType), imageSize.x, imageSize.y); 9 | 10 | _whitenKernel = cl::Kernel(program.getProgram(), "whiten"); 11 | } 12 | 13 | void ImageWhitener::filter(sys::ComputeSystem &cs, const cl::Image2D &input, cl_int kernelRadius, cl_float intensity) { 14 | int argIndex = 0; 15 | 16 | _whitenKernel.setArg(argIndex++, input); 17 | _whitenKernel.setArg(argIndex++, _result); 18 | _whitenKernel.setArg(argIndex++, _imageSize); 19 | _whitenKernel.setArg(argIndex++, kernelRadius); 20 | _whitenKernel.setArg(argIndex++, intensity); 21 | 22 | cs.getQueue().enqueueNDRangeKernel(_whitenKernel, cl::NullRange, cl::NDRange(_imageSize.x, _imageSize.y)); 23 | } -------------------------------------------------------------------------------- /NeoRL/source/system/ComputeProgram.cpp: -------------------------------------------------------------------------------- 1 | #include "ComputeProgram.h" 2 | 3 | #include 4 | #include 5 | 6 | using namespace sys; 7 | 8 | bool ComputeProgram::loadFromFile(const std::string &name, ComputeSystem &cs) { 9 | std::ifstream fromFile(name); 10 | 11 | if (!fromFile.is_open()) { 12 | #ifdef SYS_DEBUG 13 | std::cerr << "Could not open file " << name << "!" << std::endl; 14 | #endif 15 | return false; 16 | } 17 | 18 | std::string source = ""; 19 | 20 | while (!fromFile.eof() && fromFile.good()) { 21 | std::string line; 22 | 23 | std::getline(fromFile, line); 24 | 25 | source += line + "\n"; 26 | } 27 | 28 | _program = cl::Program(cs.getContext(), source); 29 | 30 | if (_program.build(std::vector(1, cs.getDevice())) != CL_SUCCESS) { 31 | #ifdef SYS_DEBUG 32 | std::cerr << "Error building: " << _program.getBuildInfo(cs.getDevice()) << std::endl; 33 | #endif 34 | return false; 35 | } 36 | 37 | return true; 38 | } -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | NeoRL 2 | Copyright (C) 2015 Eric Laukien 3 | 4 | This software is provided 'as-is', without any express or implied 5 | warranty. In no event will the authors be held liable for any damages 6 | arising from the use of this software. 7 | 8 | Permission is granted to anyone to use this software for any purpose, 9 | including commercial applications, and to alter it and redistribute it 10 | freely, subject to the following restrictions: 11 | 12 | 1. The origin of this software must not be misrepresented; you must not 13 | claim that you wrote the original software. If you use this software 14 | in a product, an acknowledgement in the product documentation would be 15 | appreciated but is not required. 16 | 2. Altered source versions must be plainly marked as such, and must not be 17 | misrepresented as being the original software. 18 | 3. This notice may not be removed or altered from any source distribution. 19 | 20 | ------------------------------------------------------------------------------ 21 | 22 | NeoRL uses the following external libraries: 23 | OpenCL -------------------------------------------------------------------------------- /NeoRL/source/vis/PrettySDR.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace vis { 6 | class PrettySDR { 7 | private: 8 | std::vector _nodes; 9 | 10 | int _width, _height; 11 | 12 | public: 13 | float _edgeRadius; 14 | float _nodeSpaceSize; 15 | float _nodeOuterRatio; 16 | float _nodeInnerRatio; 17 | 18 | int _edgeSegments; 19 | int _nodeOuterSegments; 20 | int _nodeInnerSegments; 21 | 22 | sf::Color _backgroundColor; 23 | sf::Color _nodeOuterColor; 24 | sf::Color _nodeInnerColor; 25 | 26 | PrettySDR() 27 | : _edgeRadius(4.0f), _nodeSpaceSize(16.0f), _nodeOuterRatio(0.85f), _nodeInnerRatio(0.75f), 28 | _edgeSegments(16), _nodeOuterSegments(16), _nodeInnerSegments(16), 29 | _backgroundColor(128, 128, 128), _nodeOuterColor(64, 64, 64), _nodeInnerColor(255, 0, 0) 30 | {} 31 | 32 | void create(int width, int height); 33 | 34 | float &operator[](int index) { 35 | return _nodes[index]; 36 | } 37 | 38 | float &at(int x, int y) { 39 | return _nodes[x + y * _width]; 40 | } 41 | 42 | void draw(sf::RenderTarget &rt, const sf::Vector2f &position); 43 | }; 44 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/ImageWhitener.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../system/ComputeSystem.h" 4 | #include "../system/ComputeProgram.h" 5 | 6 | namespace neo { 7 | /*! 8 | \brief Image whitener 9 | Applies local whitening transformation to input 10 | */ 11 | class ImageWhitener { 12 | private: 13 | /*! 14 | \brief Kernels 15 | */ 16 | cl::Kernel _whitenKernel; 17 | 18 | /*! 19 | \brief Resulting whitened image 20 | */ 21 | cl::Image2D _result; 22 | 23 | /*! 24 | \brief Size of the whitened image 25 | */ 26 | cl_int2 _imageSize; 27 | 28 | public: 29 | /*! 30 | \brief Create the image whitener 31 | Requires the image size and format. 32 | */ 33 | void create(sys::ComputeSystem &cs, sys::ComputeProgram &program, cl_int2 imageSize, cl_int imageFormat, cl_int imageType); 34 | 35 | /*! 36 | \brief Filter (whiten) an image with a kernel radius 37 | */ 38 | void filter(sys::ComputeSystem &cs, const cl::Image2D &input, cl_int kernelRadius, cl_float intensity = 1024.0f); 39 | 40 | /*! 41 | \brief Return filtered image result 42 | */ 43 | const cl::Image2D &getResult() const { 44 | return _result; 45 | } 46 | }; 47 | } -------------------------------------------------------------------------------- /NeoRL/source/vis/Plot.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | namespace vis { 6 | struct Point { 7 | sf::Vector2f _position; 8 | 9 | sf::Color _color; 10 | 11 | Point() 12 | : _color(sf::Color::Black) 13 | {} 14 | }; 15 | 16 | struct Curve { 17 | std::string _name; 18 | 19 | float _shadow; 20 | sf::Vector2f _shadowOffset; 21 | 22 | std::vector _points; 23 | 24 | Curve() 25 | : _shadow(0.5f), _shadowOffset(-4.0f, 4.0f) 26 | {} 27 | }; 28 | 29 | struct Plot { 30 | sf::Color _axesColor; 31 | sf::Color _backgroundColor; 32 | 33 | std::vector _curves; 34 | 35 | Plot() 36 | : _axesColor(sf::Color::Black), _backgroundColor(sf::Color::White) 37 | {} 38 | 39 | void draw(sf::RenderTarget &target, const sf::Texture &lineGradientTexture, const sf::Font &tickFont, float tickTextScale, 40 | const sf::Vector2f &domain, const sf::Vector2f &range, const sf::Vector2f &margins, const sf::Vector2f &tickIncrements, float axesSize, float lineSize, float tickSize, float tickLength, float textTickOffset, int precision); 41 | }; 42 | 43 | float vectorMagnitude(const sf::Vector2f &vector); 44 | sf::Vector2f vectorNormalize(const sf::Vector2f &vector); 45 | float vectorDot(const sf::Vector2f &left, const sf::Vector2f &right); 46 | } -------------------------------------------------------------------------------- /NeoRL/source/COCO.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_COCO 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | int main() { 16 | std::mt19937 generator(time(nullptr)); 17 | 18 | sys::ComputeSystem cs; 19 | 20 | cs.create(sys::ComputeSystem::_gpu); 21 | 22 | sys::ComputeProgram prog; 23 | 24 | prog.loadFromFile("resources/neoKernels.cl", cs); 25 | 26 | // --------------------------- Create the Sparse Coder --------------------------- 27 | 28 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 2, 2); 29 | 30 | std::vector layerDescs(3); 31 | 32 | layerDescs[0]._size = { 16, 16 }; 33 | layerDescs[1]._size = { 16, 16 }; 34 | layerDescs[2]._size = { 16, 16 }; 35 | 36 | neo::PredictiveHierarchy ph; 37 | 38 | ph.createRandom(cs, prog, { 2, 2 }, layerDescs, { -0.01f, 0.01f }, 0.0f, generator); 39 | //std::ifstream is("binh_save.neo"); 40 | 41 | //ph.readFromStream(cs, prog, is); 42 | 43 | sf::RenderWindow renderWindow; 44 | 45 | renderWindow.create(sf::VideoMode(1200, 600), "NeoRL - COCO", sf::Style::Default); 46 | 47 | renderWindow.setVerticalSyncEnabled(true); 48 | renderWindow.setFramerateLimit(60); 49 | 50 | return 0; 51 | } 52 | 53 | #endif -------------------------------------------------------------------------------- /NeoRL/source/system/ComputeSystem.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | 5 | #define CL_HPP_MINIMUM_OPENCL_VERSION 200 6 | #define CL_HPP_TARGET_OPENCL_VERSION 200 7 | 8 | #include 9 | 10 | #define SYS_DEBUG 11 | 12 | #define SYS_ALLOW_CL_GL_CONTEXT 0 13 | 14 | namespace sys { 15 | /*! 16 | \brief Compute system 17 | Holds OpenCL platform, device, context, and command queue 18 | */ 19 | class ComputeSystem : private Uncopyable { 20 | public: 21 | enum DeviceType { 22 | _cpu, _gpu, _all, _none 23 | }; 24 | 25 | private: 26 | //!@{ 27 | /*! 28 | \brief OpenCL handles 29 | */ 30 | cl::Platform _platform; 31 | cl::Device _device; 32 | cl::Context _context; 33 | cl::CommandQueue _queue; 34 | //!@} 35 | 36 | public: 37 | /*! 38 | \brief Create compute system with a given device type 39 | Optional: Create from an OpenGL context 40 | */ 41 | bool create(DeviceType type, bool createFromGLContext = false); 42 | 43 | /*! 44 | \brief Get underlying OpenCL platform 45 | */ 46 | cl::Platform &getPlatform() { 47 | return _platform; 48 | } 49 | 50 | /*! 51 | \brief Get underlying OpenCL device 52 | */ 53 | cl::Device &getDevice() { 54 | return _device; 55 | } 56 | 57 | /*! 58 | \brief Get underlying OpenCL context 59 | */ 60 | cl::Context &getContext() { 61 | return _context; 62 | } 63 | 64 | /*! 65 | \brief Get underlying OpenCL command queue 66 | */ 67 | cl::CommandQueue &getQueue() { 68 | return _queue; 69 | } 70 | }; 71 | } -------------------------------------------------------------------------------- /NeoRL/source/SinusExample.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_SINUS_EXAMPLE 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | float sig(float x) { 18 | return 1.0f / (1.0f + std::exp(-x)); 19 | } 20 | 21 | int main() { 22 | std::mt19937 generator(time(nullptr)); 23 | 24 | sys::ComputeSystem cs; 25 | 26 | cs.create(sys::ComputeSystem::_cpu); 27 | 28 | sys::ComputeProgram prog; 29 | 30 | prog.loadFromFile("resources/neoKernels.cl", cs); 31 | 32 | // --------------------------- Create the Sparse Coder --------------------------- 33 | 34 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 2, 2); 35 | 36 | std::vector layerDescs(3); 37 | 38 | neo::PredictiveHierarchy ph; 39 | 40 | ph.createRandom(cs, prog, { 2, 2 }, layerDescs, { -0.01f, 0.01f }, generator); 41 | 42 | for (int i = 0; i < 1000; i++) { 43 | float v = std::sin(i * 0.1f); 44 | 45 | std::vector vals(4); 46 | vals[0] = v; 47 | vals[1] = v - 1.0f; 48 | vals[2] = v + 1.0f; 49 | vals[3] = v * 2.0f; 50 | 51 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 2, 2, 1 }, 0, 0, vals.data()); 52 | 53 | ph.simStep(cs, inputImage); 54 | 55 | std::vector res(4); 56 | 57 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 2, 2, 1 }, 0, 0, res.data()); 58 | 59 | std::vector sdr(256); 60 | 61 | cs.getQueue().enqueueReadImage(ph.getLayer(0)._sc.getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { 8, 8, 1 }, 0, 0, sdr.data()); 62 | 63 | std::cout << res[0] << std::endl; 64 | } 65 | 66 | return 0; 67 | } 68 | 69 | #endif -------------------------------------------------------------------------------- /NeoRL/source/neo/Helpers.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../system/ComputeSystem.h" 4 | #include "../system/ComputeProgram.h" 5 | 6 | #include 7 | #include 8 | 9 | namespace neo { 10 | /*! 11 | \brief Buffer types (can be used as indices) 12 | */ 13 | enum BufferType { 14 | _front = 0, _back = 1 15 | }; 16 | 17 | //!@{ 18 | /*! 19 | \brief Double buffer types 20 | */ 21 | typedef std::array DoubleBuffer2D; 22 | typedef std::array DoubleBuffer3D; 23 | //!@} 24 | 25 | //!@{ 26 | /*! 27 | \brief Double buffer creation helpers 28 | */ 29 | DoubleBuffer2D createDoubleBuffer2D(sys::ComputeSystem &cs, cl_int2 size, cl_channel_order channelOrder, cl_channel_type channelType); 30 | DoubleBuffer3D createDoubleBuffer3D(sys::ComputeSystem &cs, cl_int3 size, cl_channel_order channelOrder, cl_channel_type channelType); 31 | //!@} 32 | 33 | //!@{ 34 | /*! 35 | \brief Double buffer initialization helpers 36 | */ 37 | void randomUniform(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng); 38 | void randomUniform(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng); 39 | void randomUniformXY(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXYKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng); 40 | void randomUniformXYZ(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXYZKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng); 41 | void randomUniformXY(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DXYKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng); 42 | void randomUniformXZ(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXZKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng); 43 | void randomUniformXZ(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DXZKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng); 44 | //!@} 45 | } -------------------------------------------------------------------------------- /NeoRL/source/runner/Runner.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "../Settings.h" 4 | 5 | #if EXPERIMENT_SELECTION == EXPERIMENT_RUNNER 6 | 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | class Runner { 13 | public: 14 | struct LimbSegmentDesc { 15 | float _relativeAngle; 16 | float _thickness, _length; 17 | float _minAngle, _maxAngle; 18 | float _maxTorque; 19 | float _maxSpeed; 20 | float _density; 21 | float _friction; 22 | float _restitution; 23 | bool _motorEnabled; 24 | 25 | LimbSegmentDesc() 26 | : _relativeAngle(0.0f), 27 | _thickness(0.03f), _length(0.125f), 28 | _minAngle(-0.4f), _maxAngle(0.4f), 29 | _maxTorque(8.0f), 30 | _maxSpeed(4.0f), 31 | _density(2.0f), 32 | _friction(2.0f), 33 | _restitution(0.01f), 34 | _motorEnabled(true) 35 | {} 36 | }; 37 | 38 | struct LimbSegment { 39 | b2Body* _pBody; 40 | b2PolygonShape _bodyShape; 41 | b2RevoluteJoint* _pJoint; 42 | 43 | float _maxSpeed; 44 | float _minAngle; 45 | float _maxAngle; 46 | }; 47 | 48 | struct Limb { 49 | std::vector _segments; 50 | 51 | void create(b2World* pWorld, const std::vector &descs, b2Body* pAttachBody, const b2Vec2 &localAttachPoint, uint16 categoryBits, uint16 maskBits); 52 | void remove(b2World* pWorld); 53 | }; 54 | private: 55 | std::shared_ptr _world; 56 | 57 | public: 58 | static sf::Color mulColors(const sf::Color &c1, const sf::Color &c2) { 59 | const float byteInv = 1.0f / 255.0f; 60 | 61 | return sf::Color(c1.r * c2.r * byteInv, 62 | c1.g * c2.g * byteInv, 63 | c1.b * c2.b * byteInv); 64 | } 65 | 66 | b2Body* _pBody; 67 | b2PolygonShape _bodyShape; 68 | 69 | Limb _leftBackLimb; 70 | Limb _leftFrontLimb; 71 | Limb _rightBackLimb; 72 | Limb _rightFrontLimb; 73 | 74 | Runner() 75 | : _world(nullptr) 76 | {} 77 | 78 | ~Runner(); 79 | 80 | void createDefault(const std::shared_ptr &world, const b2Vec2 &position, float angle, int layer); 81 | 82 | void renderDefault(sf::RenderTarget &rt, const sf::Color &color, float metersToPixels); 83 | 84 | void getStateVector(std::vector &state); 85 | void motorUpdate(const std::vector &action, float interpolateFactor); 86 | }; 87 | 88 | #endif -------------------------------------------------------------------------------- /NeoRL/source/vis/PrettySDR.cpp: -------------------------------------------------------------------------------- 1 | #include "PrettySDR.h" 2 | 3 | using namespace vis; 4 | 5 | void PrettySDR::create(int width, int height) { 6 | _width = width; 7 | _height = height; 8 | 9 | _nodes.clear(); 10 | _nodes.assign(width * height, 0.0f); 11 | } 12 | 13 | void PrettySDR::draw(sf::RenderTarget &rt, const sf::Vector2f &position) { 14 | float rWidth = _nodeSpaceSize * _width; 15 | float rHeight = _nodeSpaceSize * _height; 16 | 17 | sf::RectangleShape rsHorizontal; 18 | rsHorizontal.setPosition(position + sf::Vector2f(0.0f, _edgeRadius)); 19 | rsHorizontal.setSize(sf::Vector2f(rWidth, rHeight - _edgeRadius * 2.0f)); 20 | rsHorizontal.setFillColor(_backgroundColor); 21 | 22 | rt.draw(rsHorizontal); 23 | 24 | sf::RectangleShape rsVertical; 25 | rsVertical.setPosition(position + sf::Vector2f(_edgeRadius, 0.0f)); 26 | rsVertical.setSize(sf::Vector2f(rWidth - _edgeRadius * 2.0f, rHeight)); 27 | rsVertical.setFillColor(_backgroundColor); 28 | 29 | rt.draw(rsVertical); 30 | 31 | // Corners 32 | sf::CircleShape corner; 33 | corner.setRadius(_edgeRadius); 34 | corner.setPointCount(_edgeSegments); 35 | corner.setFillColor(_backgroundColor); 36 | corner.setOrigin(sf::Vector2f(_edgeRadius, _edgeRadius)); 37 | 38 | corner.setPosition(position + sf::Vector2f(_edgeRadius, _edgeRadius)); 39 | rt.draw(corner); 40 | 41 | corner.setPosition(position + sf::Vector2f(rWidth - _edgeRadius, _edgeRadius)); 42 | rt.draw(corner); 43 | 44 | corner.setPosition(position + sf::Vector2f(rWidth - _edgeRadius, rHeight - _edgeRadius)); 45 | rt.draw(corner); 46 | 47 | corner.setPosition(position + sf::Vector2f(_edgeRadius, rHeight - _edgeRadius)); 48 | rt.draw(corner); 49 | 50 | // Nodes 51 | sf::CircleShape outer; 52 | outer.setRadius(_nodeSpaceSize * _nodeOuterRatio * 0.5f); 53 | outer.setPointCount(_nodeOuterSegments); 54 | outer.setFillColor(_nodeOuterColor); 55 | outer.setOrigin(sf::Vector2f(outer.getRadius(), outer.getRadius())); 56 | 57 | sf::CircleShape inner; 58 | inner.setRadius(_nodeSpaceSize * _nodeOuterRatio * _nodeInnerRatio * 0.5f); 59 | inner.setPointCount(_nodeInnerSegments); 60 | inner.setFillColor(_nodeInnerColor); 61 | inner.setOrigin(sf::Vector2f(inner.getRadius(), inner.getRadius())); 62 | 63 | for (int x = 0; x < _width; x++) 64 | for (int y = 0; y < _height; y++) { 65 | outer.setPosition(position + sf::Vector2f(x * _nodeSpaceSize + _edgeRadius, y * _nodeSpaceSize + _edgeRadius)); 66 | 67 | rt.draw(outer); 68 | 69 | inner.setPosition(position + sf::Vector2f(x * _nodeSpaceSize + _edgeRadius, y * _nodeSpaceSize + _edgeRadius)); 70 | 71 | inner.setFillColor(sf::Color(_nodeInnerColor.r, _nodeInnerColor.g, _nodeInnerColor.b, 255 * at(x, y))); 72 | 73 | rt.draw(inner); 74 | } 75 | } -------------------------------------------------------------------------------- /NeoRL/source/system/ComputeSystem.cpp: -------------------------------------------------------------------------------- 1 | #include "ComputeSystem.h" 2 | 3 | #include 4 | 5 | using namespace sys; 6 | 7 | bool ComputeSystem::create(DeviceType type, bool createFromGLContext) { 8 | if (type == _none) { 9 | #ifdef SYS_DEBUG 10 | std::cout << "No OpenCL context created." << std::endl; 11 | #endif 12 | return true; 13 | } 14 | 15 | std::vector allPlatforms; 16 | cl::Platform::get(&allPlatforms); 17 | 18 | if (allPlatforms.empty()) { 19 | #ifdef SYS_DEBUG 20 | std::cout << "No platforms found. Check your OpenCL installation." << std::endl; 21 | #endif 22 | return false; 23 | } 24 | 25 | _platform = allPlatforms.front(); 26 | 27 | #ifdef SYS_DEBUG 28 | std::cout << "Using platform: " << _platform.getInfo() << std::endl; 29 | #endif 30 | 31 | std::vector allDevices; 32 | 33 | switch (type) { 34 | case _cpu: 35 | _platform.getDevices(CL_DEVICE_TYPE_CPU, &allDevices); 36 | break; 37 | case _gpu: 38 | _platform.getDevices(CL_DEVICE_TYPE_GPU, &allDevices); 39 | break; 40 | case _all: 41 | _platform.getDevices(CL_DEVICE_TYPE_ALL, &allDevices); 42 | break; 43 | } 44 | 45 | if (allDevices.empty()) { 46 | #ifdef SYS_DEBUG 47 | std::cout << "No devices found. Check your OpenCL installation." << std::endl; 48 | #endif 49 | return false; 50 | } 51 | 52 | _device = allDevices.front(); 53 | 54 | #ifdef SYS_DEBUG 55 | std::cout << "Using device: " << _device.getInfo() << std::endl; 56 | #endif 57 | 58 | #if(SYS_ALLOW_CL_GL_CONTEXT) 59 | if (createFromGLContext) { 60 | #if defined (__APPLE__) || defined(MACOSX) 61 | CGLContextObj kCGLContext = CGLGetCurrentContext(); 62 | CGLShareGroupObj kCGLShareGroup = CGLGetShareGroup(kCGLContext); 63 | cl_context_properties props[] = { 64 | CL_CONTEXT_PROPERTY_USE_CGL_SHAREGROUP_APPLE, (cl_context_properties)kCGLShareGroup, 65 | 0 66 | }; 67 | #else 68 | #if defined WIN32 69 | cl_context_properties props[] = { 70 | CL_GL_CONTEXT_KHR, (cl_context_properties)wglGetCurrentContext(), 71 | CL_WGL_HDC_KHR, (cl_context_properties)wglGetCurrentDC(), 72 | CL_CONTEXT_PLATFORM, (cl_context_properties)static_cast(_platform()), 73 | 0 74 | }; 75 | #else 76 | cl_context_properties props[] = { 77 | CL_GL_CONTEXT_KHR, (cl_context_properties)glXGetCurrentContext(), 78 | CL_GLX_DISPLAY_KHR, (cl_context_properties)glXGetCurrentDisplay(), 79 | CL_CONTEXT_PLATFORM, (cl_context_properties)static_cast(_platform()), 80 | 0 81 | }; 82 | #endif 83 | #endif 84 | 85 | _context = cl::Context(_device, props); 86 | } 87 | else 88 | #endif 89 | _context = _device; 90 | 91 | _queue = cl::CommandQueue(_context, _device); 92 | 93 | return true; 94 | } -------------------------------------------------------------------------------- /NeoRL/source/deep/FERL.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | 8 | namespace deep { 9 | class FERL { 10 | public: 11 | static float sigmoid(float x) { 12 | return 1.0f / (1.0f + std::exp(-x)); 13 | } 14 | 15 | struct ReplaySample { 16 | std::vector _visible; 17 | 18 | float _originalQ; 19 | float _q; 20 | }; 21 | 22 | private: 23 | struct Connection { 24 | float _weight; 25 | }; 26 | 27 | struct Hidden { 28 | Connection _bias; 29 | std::vector _connections; 30 | float _state; 31 | 32 | Hidden() 33 | : _state(0.0f) 34 | {} 35 | }; 36 | 37 | struct Visible { 38 | Connection _bias; 39 | float _state; 40 | 41 | Visible() 42 | : _state(0.0f) 43 | {} 44 | }; 45 | 46 | std::vector _hidden; 47 | std::vector _visible; 48 | std::vector _actions; 49 | 50 | int _numState; 51 | int _numAction; 52 | 53 | float _zInv; 54 | 55 | float _prevValue; 56 | 57 | std::vector _prevVisible; 58 | std::vector _prevHidden; 59 | 60 | std::list _replaySamples; 61 | 62 | public: 63 | FERL(); 64 | 65 | void createRandom(int numState, int numAction, int numHidden, float weightStdDev, std::mt19937 &generator); 66 | 67 | void createFromParents(const FERL &parent1, const FERL &parent2, float averageChance, std::mt19937 &generator); 68 | 69 | void mutate(float perturbationStdDev, std::mt19937 &generator); 70 | 71 | // Returns action index 72 | void step(const std::vector &state, std::vector &action, 73 | float reward, float qAlpha, float gamma, float lambdaGamma, 74 | float actionAlpha, int actionSearchIterations, int actionSearchSamples, float actionSearchAlpha, 75 | float breakChance, float perturbationStdDev, 76 | int maxNumReplaySamples, int replayIterations, float gradientAlpha, 77 | std::mt19937 &generator); 78 | 79 | void activate(); 80 | void updateOnError(float error); 81 | 82 | float freeEnergy() const; 83 | 84 | void saveToFile(std::ostream &os, bool saveReplayInformation = false); 85 | void loadFromFile(std::istream &is, bool loadReplayInformation = false); 86 | 87 | float value() const { 88 | return -freeEnergy() * _zInv; 89 | } 90 | 91 | int getNumState() const { 92 | return _numState; 93 | } 94 | 95 | int getNumAction() const { 96 | return _numAction; 97 | } 98 | 99 | int getNumVisible() const { 100 | return _visible.size(); 101 | } 102 | 103 | int getNumHidden() const { 104 | return _hidden.size(); 105 | } 106 | 107 | float getZInv() const { 108 | return _zInv; 109 | } 110 | 111 | const std::list &getSamples() const { 112 | return _replaySamples; 113 | } 114 | }; 115 | } -------------------------------------------------------------------------------- /NeoRL/source/deep/SDRRL.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include 5 | 6 | namespace deep { 7 | // Unit part of the self-optimizing hierarchy. 8 | class SDRRL { 9 | private: 10 | struct Connection { 11 | float _weight; 12 | float _trace; 13 | 14 | Connection() 15 | : _trace(0.0f) 16 | {} 17 | }; 18 | 19 | struct Cell { 20 | std::vector _feedForwardConnections; 21 | std::vector _lateralConnections; 22 | std::vector _actionConnections; 23 | 24 | float _threshold; 25 | 26 | float _activation; 27 | 28 | float _state; 29 | 30 | float _actionState; 31 | float _actionError; 32 | 33 | Cell() 34 | : _actionState(0.0f), _actionError(0.0f) 35 | {} 36 | }; 37 | 38 | struct Action { 39 | float _state; 40 | float _statePrev; 41 | float _exploratoryState; 42 | float _error; 43 | 44 | //std::vector _connections; 45 | 46 | Action() 47 | : _state(0.0f), _statePrev(0.0f), _exploratoryState(0.0f) 48 | {} 49 | }; 50 | 51 | std::vector _inputs; 52 | std::vector _reconstructionError; 53 | std::vector _cells; 54 | std::vector _qConnections; 55 | std::vector _actions; 56 | 57 | int _numStates; 58 | 59 | float _prevValue; 60 | float _averageSurprise; 61 | 62 | public: 63 | static float relu(float x, float leak) { 64 | return x > 0.0f ? x : x * leak; 65 | } 66 | 67 | static float relud(float x, float leak) { 68 | return x > 0.0f ? 1.0f : leak; 69 | } 70 | 71 | static float sigmoid(float x) { 72 | return 1.0f / (1.0f + std::exp(-x)); 73 | } 74 | 75 | SDRRL() 76 | : _prevValue(0.0f), _averageSurprise(0.0f) 77 | {} 78 | 79 | void createRandom(int numStates, int numActions, int numCells, float initMinWeight, float initMaxWeight, float initMinInhibition, float initMaxInhibition, float initThreshold, std::mt19937 &generator); 80 | 81 | void simStep(float reward, float sparsity, float gamma, 82 | float gateFeedForwardAlpha, float gateLateralAlpha, float gateThresholdAlpha, 83 | float qAlpha, float actionAlpha, int actionDeriveIterations, float actionDeriveAlpha, float gammaLambda, 84 | float explorationStdDev, float explorationBreak, 85 | float averageSurpiseDecay, float surpriseLearnFactor, std::mt19937 &generator); 86 | 87 | void setState(int index, float value) { 88 | _inputs[index] = value; 89 | } 90 | 91 | float getAction(int index) const { 92 | return _actions[index]._exploratoryState; 93 | } 94 | 95 | int getNumStates() const { 96 | return _inputs.size(); 97 | } 98 | 99 | int getNumActions() const { 100 | return _actions.size(); 101 | } 102 | 103 | int getNumCells() const { 104 | return _cells.size(); 105 | } 106 | 107 | float getCellState(int index) const { 108 | return _cells[index]._state; 109 | } 110 | }; 111 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![DOI](https://zenodo.org/badge/19602/222464/NeoRL.svg)](https://zenodo.org/badge/latestdoi/19602/222464/NeoRL) 2 | 3 | ![NeoRL Logo](http://i1218.photobucket.com/albums/dd401/222464/NeoRL_logo_med.png) 4 | 5 | Welcome to NeoRL, an algorithmic GPU neocortex simulation library. 6 | 7 | # Installation 8 | 9 | NeoRL requires OpenCL 2.0 or greater to run. Unfortunately this excludes Nvidia hardware, but it will work for AMD and Intel processors. 10 | If you have an AMD card, I recommend getting the AMD APP SDK here: http://developer.amd.com/tools-and-sdks/opencl-zone/amd-accelerated-parallel-processing-app-sdk/ 11 | Otherwise, you can try the Intel OpenCL SDK: https://software.intel.com/en-us/intel-opencl 12 | It should also be feasible to run it on a Xeon Phi. 13 | 14 | In order to run the demos, you will also need to install SFML. Additionally, one demo (Runner) requires Box2D in order to work. 15 | 16 | To select a demo, change the macros in Settings.h. 17 | 18 | You can generate documentation using Doxygen and the provided doxygen_config file. 19 | 20 | # Overview 21 | 22 | See the accompanying blog posts at to discover how NeoRL works internally at http://www.twistedkeyboardsoftware.com/ 23 | You don't need to know this in order to use it, it can be treated as a black box sequence predictor as well. 24 | 25 | NeoRL contains both reinforcement learning agents as well as a predictive hierarchy. These are fully online learning algorithms. 26 | The simplest usage of both the reinforcement learning agents and the predictive hierarchy involves calling: 27 | 28 | ```cpp 29 | std::mt19937 generator(time(nullptr)); 30 | 31 | sys::ComputeSystem cs; 32 | 33 | cs.create(sys::ComputeSystem::_gpu); 34 | 35 | sys::ComputeProgram prog; 36 | 37 | prog.loadFromFile("resources/neoKernels.cl", cs); 38 | 39 | // --------------------------- Create the Predictive Hierarchy --------------------------- 40 | 41 | // Temporary input buffer 42 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 2, 2); 43 | 44 | // Layer descriptors for hierarchy 45 | std::vector layerDescs(3); 46 | 47 | layerDescs[0]._size = { 16, 16 }; 48 | layerDescs[1]._size = { 16, 16 }; 49 | layerDescs[2]._size = { 16, 16 }; 50 | 51 | // Hierarchy itself 52 | neo::PredictiveHierarchy ph; 53 | 54 | // 2x2 input field 55 | ph.createRandom(cs, prog, { 2, 2 }, layerDescs, { -0.01f, 0.01f }, 0.0f, generator); 56 | ``` 57 | 58 | You can then step the simulation with: 59 | 60 | ```cpp 61 | // Copy vals into temporary OpenCL image buffer 62 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 2, 2, 1 }, 0, 0, vals.data()); 63 | 64 | // Step the simulation 65 | ph.simStep(cs, inputImage); 66 | 67 | // Retrieve the prediction (same dimensions as input field) 68 | std::vector pred(4); 69 | 70 | cs.getQueue().enqueueReadImage(ph.getFirstLayerPred().getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { 2, 2, 1 }, 0, 0, pred.data()); 71 | ``` 72 | 73 | See the demos for more complicated usage. 74 | 75 | # License 76 | 77 | ZLib license. See LICENSE.md -------------------------------------------------------------------------------- /NeoRL/source/SequenceRecall.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #include "neo/PredictiveHierarchy.h" 4 | 5 | #include 6 | #include 7 | 8 | #if EXPERIMENT_SELECTION == EXPERIMENT_SEQUENCE_RECALL 9 | 10 | int main() { 11 | std::mt19937 generator(std::time(nullptr)); 12 | 13 | sys::ComputeSystem cs; 14 | 15 | cs.create(sys::ComputeSystem::_gpu); 16 | 17 | sys::ComputeProgram prog; 18 | 19 | prog.loadFromFile("resources/neoKernels2.cl", cs); 20 | 21 | // --------------------------- Create the Sparse Coder --------------------------- 22 | 23 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 4, 4); 24 | 25 | std::vector layerDescs(3); 26 | 27 | layerDescs[0]._size = { 16, 16 }; 28 | layerDescs[1]._size = { 12, 12 }; 29 | layerDescs[2]._size = { 8, 8 }; 30 | 31 | neo::PredictiveHierarchy ph; 32 | 33 | ph.createRandom(cs, prog, { 4, 4 }, layerDescs, { -0.1f, 0.1f }, generator); 34 | 35 | std::uniform_int_distribution item_dist(0, 9); 36 | 37 | std::vector inputVec(16, 0.0f); 38 | 39 | float avg_error = 0.0f; 40 | 41 | for (int train_iter = 0; train_iter < 1000; train_iter++) { 42 | std::vector items(10); 43 | 44 | for (int show_iter = 0; show_iter < 10; show_iter++) { 45 | items[show_iter] = item_dist(generator); 46 | 47 | for (int i = 0; i < 16; i++) 48 | inputVec[i] = 0.0f; 49 | 50 | inputVec[items[show_iter]] = 1.0f; 51 | 52 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data()); 53 | 54 | ph.simStep(cs, inputImage); 55 | } 56 | 57 | for (int i = 0; i < 16; i++) 58 | inputVec[i] = 0.0f; 59 | 60 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data()); 61 | 62 | for (int wait_iter = 0; wait_iter < 10; wait_iter++) { 63 | ph.simStep(cs, inputImage); 64 | } 65 | 66 | // Show delimiter (item = 10) 67 | for (int i = 0; i < 16; i++) 68 | inputVec[i] = 0.0f; 69 | 70 | inputVec[10] = 1.0f; 71 | 72 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data()); 73 | 74 | ph.simStep(cs, inputImage); 75 | 76 | float error = 0.0f; 77 | 78 | std::vector pred(16, 0.0f); 79 | 80 | for (int recall_iter = 0; recall_iter < 10; recall_iter++) { 81 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, pred.data()); 82 | 83 | for (int i = 0; i < 16; i++) { 84 | if (i == items[recall_iter]) 85 | error += std::pow(1.0f - pred[i], 2); 86 | else 87 | error += std::pow(0.0f - pred[i], 2); 88 | } 89 | 90 | for (int i = 0; i < 16; i++) 91 | inputVec[i] = 0.0f; 92 | 93 | inputVec[items[recall_iter]] = 1.0f; 94 | 95 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 4, 4, 1 }, 0, 0, inputVec.data()); 96 | 97 | ph.simStep(cs, inputImage); 98 | } 99 | 100 | std::cout << error << std::endl; 101 | } 102 | 103 | return 0; 104 | } 105 | 106 | #endif -------------------------------------------------------------------------------- /NeoRL/source/neo/PredictiveHierarchy.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "SparsePredictor.h" 4 | #include "ImageWhitener.h" 5 | 6 | namespace neo { 7 | /*! 8 | \brief Predictive hierarchy (no RL) 9 | */ 10 | class PredictiveHierarchy { 11 | public: 12 | /*! 13 | \brief Layer desc 14 | */ 15 | struct LayerDesc { 16 | /*! 17 | \brief Size of layer 18 | */ 19 | cl_int2 _size; 20 | 21 | /*! 22 | \brief Radii 23 | */ 24 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 25 | 26 | //!@{ 27 | /*! 28 | \brief Sparse predictor parameters 29 | */ 30 | cl_float _spWeightEncodeAlpha; 31 | cl_float _spWeightDecodeAlpha; 32 | cl_float _spWeightLambda; 33 | cl_float _spActiveRatio; 34 | cl_float _spBiasAlpha; 35 | //!@} 36 | 37 | /*! 38 | \brief Initialize defaults 39 | */ 40 | LayerDesc() 41 | : _size({ 8, 8 }), 42 | _feedForwardRadius(5), _recurrentRadius(5), _lateralRadius(5), _feedBackRadius(6), _predictiveRadius(6), 43 | _spWeightEncodeAlpha(0.001f), _spWeightDecodeAlpha(0.02f), _spWeightLambda(0.9f), 44 | _spActiveRatio(0.08f), _spBiasAlpha(0.1f) 45 | {} 46 | }; 47 | 48 | /*! 49 | \brief Layer 50 | */ 51 | struct Layer { 52 | /*! 53 | \brief Sparse predictor 54 | */ 55 | SparsePredictor _sp; 56 | 57 | /*! 58 | \brief Layer for additional error signals 59 | */ 60 | cl::Image2D _additionalErrors; 61 | }; 62 | 63 | private: 64 | /*! 65 | \brief Store input size 66 | */ 67 | cl_int2 _inputSize; 68 | 69 | //!@{ 70 | /*! 71 | \brief Layers and descs 72 | */ 73 | std::vector _layers; 74 | std::vector _layerDescs; 75 | //!@} 76 | 77 | /*! 78 | \brief Input whitener 79 | */ 80 | ImageWhitener _inputWhitener; 81 | 82 | /*! 83 | \brief Zero layer for capping of the network 84 | */ 85 | cl::Image2D _zeroLayer; 86 | 87 | public: 88 | //!@{ 89 | /*! 90 | \brief Whitening parameters 91 | */ 92 | cl_int _whiteningKernelRadius; 93 | cl_float _whiteningIntensity; 94 | //!@} 95 | 96 | /*! 97 | \brief Initialize defaults 98 | */ 99 | PredictiveHierarchy() 100 | : _whiteningKernelRadius(1), 101 | _whiteningIntensity(1024.0f) 102 | {} 103 | 104 | /*! 105 | \brief Create a predictive hierarchy with random initialization 106 | Requires the compute system, program with the NeoRL kernels, and initialization information 107 | */ 108 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 109 | cl_int2 inputSize, const std::vector &layerDescs, 110 | cl_float2 initWeightRange, 111 | std::mt19937 &rng); 112 | 113 | /*! 114 | \brief Simulation step of hierarchy 115 | */ 116 | void simStep(sys::ComputeSystem &cs, const cl::Image2D &input, bool learn = true, bool whiten = false); 117 | 118 | /*! 119 | \brief Get number of layers 120 | */ 121 | size_t getNumLayers() const { 122 | return _layers.size(); 123 | } 124 | 125 | /*! 126 | \brief Get access to a layer 127 | */ 128 | const Layer &getLayer(int index) const { 129 | return _layers[index]; 130 | } 131 | 132 | /*! 133 | \brief Get access to a layer desc 134 | */ 135 | const LayerDesc &getLayerDescs(int index) const { 136 | return _layerDescs[index]; 137 | } 138 | 139 | /*! 140 | \brief Get the prediction 141 | */ 142 | const cl::Image2D &getPrediction() const { 143 | return _layers.front()._sp.getVisibleLayer(0)._predictions[_back]; 144 | } 145 | 146 | /*! 147 | \brief Get input whitener 148 | */ 149 | const ImageWhitener &getInputWhitener() const { 150 | return _inputWhitener; 151 | } 152 | }; 153 | } -------------------------------------------------------------------------------- /NeoRL/source/RNNBenchmark.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_RNN_BENCHMARK 4 | 5 | #include 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | int main() { 18 | std::mt19937 generator(time(nullptr)); 19 | 20 | std::vector> timeSeries; 21 | 22 | std::ifstream fromFile("resources/data.txt"); 23 | 24 | if (!fromFile.is_open()) { 25 | std::cerr << "Could not open data.txt!" << std::endl; 26 | 27 | return 1; 28 | } 29 | 30 | // Skip first line 31 | std::string line; 32 | 33 | std::getline(fromFile, line); 34 | 35 | int numEntries = 1; 36 | 37 | for (int i = 0; i < line.size(); i++) 38 | if (line[i] == ',') 39 | numEntries++; 40 | 41 | int numSkipEntries = 1; 42 | 43 | int numEntriesUse = numEntries - numSkipEntries; 44 | 45 | std::vector minimums(numEntriesUse, 999999999.0); 46 | std::vector maximums(numEntriesUse, -999999999.0); 47 | 48 | while (fromFile.good() && !fromFile.eof()) { 49 | std::vector entries(numEntriesUse); 50 | 51 | std::string line; 52 | 53 | std::getline(fromFile, line); 54 | 55 | std::istringstream fromLine(line); 56 | 57 | std::string param; 58 | 59 | // Skip entries 60 | for (int i = 0; i < numSkipEntries; i++) { 61 | std::string entry; 62 | 63 | std::getline(fromLine, entry, ','); 64 | } 65 | 66 | for (int i = 0; i < numEntriesUse; i++) { 67 | std::string entry; 68 | 69 | std::getline(fromLine, entry, ','); 70 | 71 | if (entry == "") 72 | entries[i] = 0.0; 73 | else { 74 | double value = std::stod(entry); 75 | 76 | maximums[i] = std::max(maximums[i], value); 77 | minimums[i] = std::min(minimums[i], value); 78 | 79 | entries[i] = value; 80 | } 81 | } 82 | 83 | timeSeries.push_back(entries); 84 | } 85 | 86 | // Rescale 87 | for (int i = 0; i < timeSeries.size(); i++) { 88 | for (int j = 0; j < timeSeries[i].size(); j++) { 89 | timeSeries[i][j] = (timeSeries[i][j] - minimums[j]) / std::max(0.0001, (maximums[j] - minimums[j])); 90 | } 91 | } 92 | 93 | /*timeSeries.clear(); 94 | 95 | timeSeries.resize(10); 96 | timeSeries[0] = { 0.0f, 1.0f, 0.0f }; 97 | timeSeries[1] = { 0.0f, 0.0f, 0.0f }; 98 | timeSeries[2] = { 1.0f, 1.0f, 0.0f }; 99 | timeSeries[3] = { 0.0f, 0.0f, 1.0f }; 100 | timeSeries[4] = { 0.0f, 1.0f, 0.0f }; 101 | timeSeries[5] = { 0.0f, 0.0f, 1.0f }; 102 | timeSeries[6] = { 0.0f, 0.0f, 0.0f }; 103 | timeSeries[7] = { 0.0f, 0.0f, 0.0f }; 104 | timeSeries[8] = { 0.0f, 1.0f, 0.0f }; 105 | timeSeries[9] = { 0.0f, 1.0f, 1.0f };*/ 106 | 107 | std::vector layerDescs(3); 108 | 109 | layerDescs[0]._width = 8; 110 | layerDescs[0]._height = 8; 111 | 112 | layerDescs[1]._width = 6; 113 | layerDescs[1]._height = 6; 114 | 115 | layerDescs[2]._width = 4; 116 | layerDescs[2]._height = 4; 117 | 118 | sdr::IPredictiveRSDR prsdr; 119 | 120 | prsdr.createRandom(4, 5, 8, layerDescs, -0.01f, 0.01f, 0.0f, generator); 121 | 122 | float avgError = 1.0f; 123 | 124 | float avgErrorDecay = 0.01f; 125 | 126 | for (int iter = 0; iter < 1000; iter++) { 127 | for (int i = 0; i < timeSeries.size(); i++) { 128 | float error = 0.0f; 129 | 130 | for (int j = 0; j < timeSeries[i].size(); j++) { 131 | error += std::pow(prsdr.getPrediction(j) - timeSeries[i][j], 2); 132 | 133 | prsdr.setInput(j, timeSeries[i][j]); 134 | } 135 | 136 | avgError = (1.0f - avgErrorDecay) * avgError + avgErrorDecay * error; 137 | 138 | prsdr.simStep(generator); 139 | 140 | if (i % 10 == 0) { 141 | std::cout << "Iteration " << i << ": " << avgError << std::endl; 142 | } 143 | } 144 | } 145 | 146 | return 0; 147 | } 148 | 149 | #endif -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ## Ignore Visual Studio temporary files, build results, and 2 | ## files generated by popular Visual Studio add-ons. 3 | 4 | # User-specific files 5 | *.suo 6 | *.user 7 | *.userosscache 8 | *.sln.docstates 9 | 10 | # User-specific files (MonoDevelop/Xamarin Studio) 11 | *.userprefs 12 | 13 | # Build results 14 | [Dd]ebug/ 15 | [Dd]ebugPublic/ 16 | [Rr]elease/ 17 | [Rr]eleases/ 18 | x64/ 19 | x86/ 20 | build/ 21 | bld/ 22 | [Bb]in/ 23 | [Oo]bj/ 24 | 25 | # Visual Studo 2015 cache/options directory 26 | .vs/ 27 | 28 | # MSTest test Results 29 | [Tt]est[Rr]esult*/ 30 | [Bb]uild[Ll]og.* 31 | 32 | # NUNIT 33 | *.VisualState.xml 34 | TestResult.xml 35 | 36 | # Build Results of an ATL Project 37 | [Dd]ebugPS/ 38 | [Rr]eleasePS/ 39 | dlldata.c 40 | 41 | *_i.c 42 | *_p.c 43 | *_i.h 44 | *.ilk 45 | *.meta 46 | *.obj 47 | *.pch 48 | *.pdb 49 | *.pgc 50 | *.pgd 51 | *.rsp 52 | *.sbr 53 | *.tlb 54 | *.tli 55 | *.tlh 56 | *.tmp 57 | *.tmp_proj 58 | *.log 59 | *.vspscc 60 | *.vssscc 61 | .builds 62 | *.pidb 63 | *.svclog 64 | *.scc 65 | 66 | # Chutzpah Test files 67 | _Chutzpah* 68 | 69 | # Visual C++ cache files 70 | ipch/ 71 | *.aps 72 | *.ncb 73 | *.opensdf 74 | *.sdf 75 | *.cachefile 76 | 77 | # Visual Studio profiler 78 | *.psess 79 | *.vsp 80 | *.vspx 81 | 82 | # TFS 2012 Local Workspace 83 | $tf/ 84 | 85 | # Guidance Automation Toolkit 86 | *.gpState 87 | 88 | # ReSharper is a .NET coding add-in 89 | _ReSharper*/ 90 | *.[Rr]e[Ss]harper 91 | *.DotSettings.user 92 | 93 | # JustCode is a .NET coding addin-in 94 | .JustCode 95 | 96 | # TeamCity is a build add-in 97 | _TeamCity* 98 | 99 | # DotCover is a Code Coverage Tool 100 | *.dotCover 101 | 102 | # NCrunch 103 | _NCrunch_* 104 | .*crunch*.local.xml 105 | 106 | # MightyMoose 107 | *.mm.* 108 | AutoTest.Net/ 109 | 110 | # Web workbench (sass) 111 | .sass-cache/ 112 | 113 | # Installshield output folder 114 | [Ee]xpress/ 115 | 116 | # DocProject is a documentation generator add-in 117 | DocProject/buildhelp/ 118 | DocProject/Help/*.HxT 119 | DocProject/Help/*.HxC 120 | DocProject/Help/*.hhc 121 | DocProject/Help/*.hhk 122 | DocProject/Help/*.hhp 123 | DocProject/Help/Html2 124 | DocProject/Help/html 125 | 126 | # Click-Once directory 127 | publish/ 128 | 129 | # Publish Web Output 130 | *.[Pp]ublish.xml 131 | *.azurePubxml 132 | # TODO: Comment the next line if you want to checkin your web deploy settings 133 | # but database connection strings (with potential passwords) will be unencrypted 134 | *.pubxml 135 | *.publishproj 136 | 137 | # NuGet Packages 138 | *.nupkg 139 | # The packages folder can be ignored because of Package Restore 140 | **/packages/* 141 | # except build/, which is used as an MSBuild target. 142 | !**/packages/build/ 143 | # Uncomment if necessary however generally it will be regenerated when needed 144 | #!**/packages/repositories.config 145 | 146 | # Windows Azure Build Output 147 | csx/ 148 | *.build.csdef 149 | 150 | # Windows Store app package directory 151 | AppPackages/ 152 | 153 | # Others 154 | *.[Cc]ache 155 | ClientBin/ 156 | [Ss]tyle[Cc]op.* 157 | ~$* 158 | *~ 159 | *.dbmdl 160 | *.dbproj.schemaview 161 | *.pfx 162 | *.publishsettings 163 | node_modules/ 164 | bower_components/ 165 | 166 | # RIA/Silverlight projects 167 | Generated_Code/ 168 | 169 | # Backup & report files from converting an old project file 170 | # to a newer Visual Studio version. Backup files are not needed, 171 | # because we have git ;-) 172 | _UpgradeReport_Files/ 173 | Backup*/ 174 | UpgradeLog*.XML 175 | UpgradeLog*.htm 176 | 177 | # SQL Server files 178 | *.mdf 179 | *.ldf 180 | 181 | # Business Intelligence projects 182 | *.rdl.data 183 | *.bim.layout 184 | *.bim_*.settings 185 | 186 | # Microsoft Fakes 187 | FakesAssemblies/ 188 | 189 | # Node.js Tools for Visual Studio 190 | .ntvs_analysis.dat 191 | 192 | # Visual Studio 6 build log 193 | *.plg 194 | 195 | # Visual Studio 6 workspace options file 196 | *.opt 197 | 198 | #more visual studio stuff 199 | *.sln 200 | 201 | #Project specific 202 | *.dll 203 | *.lib 204 | *.exp 205 | *.pdb 206 | *.ilk 207 | *.junkdgfdgfdgf 208 | *.vcxproj 209 | *.vcxproj.filters 210 | CMakeFiles/ 211 | CMakeCache.txt 212 | cmake_install.cmake -------------------------------------------------------------------------------- /NeoRL/source/neo/PredictorSwarm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Swarm of predictors 8 | Can learn with OLPOMDP to perturb predictions 9 | */ 10 | class PredictorSwarm { 11 | public: 12 | /*! 13 | \brief Visible layer desc 14 | */ 15 | struct VisibleLayerDesc { 16 | /*! 17 | \brief Size of input layer 18 | */ 19 | cl_int2 _size; 20 | 21 | /*! 22 | \brief Radius onto input 23 | */ 24 | cl_int _radius; 25 | 26 | /*! 27 | \brief Initialize defaults 28 | */ 29 | VisibleLayerDesc() 30 | : _size({ 8, 8 }), _radius(4) 31 | {} 32 | }; 33 | 34 | /*! 35 | \brief Visible layer 36 | */ 37 | struct VisibleLayer { 38 | /*! 39 | \brief Weights 40 | */ 41 | DoubleBuffer3D _weights; 42 | DoubleBuffer3D _qTraces; 43 | 44 | /*! 45 | \brief Transformations 46 | */ 47 | cl_float2 _hiddenToVisible; 48 | cl_float2 _visibleToHidden; 49 | //!@} 50 | 51 | /*! 52 | \brief Radius onto hidden (reverse from visible layer desc) 53 | */ 54 | cl_int2 _reverseRadii; 55 | }; 56 | 57 | private: 58 | //!@{ 59 | /*! 60 | \brief Hidden states and activations 61 | */ 62 | DoubleBuffer2D _hiddenStates; 63 | DoubleBuffer2D _hiddenActivations; 64 | //!@} 65 | 66 | /*! 67 | \brief Hidden size 68 | */ 69 | cl_int2 _hiddenSize; 70 | 71 | /*! 72 | \brief Summation temporary buffer 73 | */ 74 | DoubleBuffer2D _hiddenSummationTemp; 75 | 76 | //!@{ 77 | /*! 78 | \brief Visible layers and descs 79 | */ 80 | std::vector _visibleLayerDescs; 81 | std::vector _visibleLayers; 82 | //!@} 83 | 84 | //!@{ 85 | /*! 86 | \brief Kernels 87 | */ 88 | cl::Kernel _activateKernel; 89 | cl::Kernel _solveHiddenKernel; 90 | cl::Kernel _solveHiddenNoInhibitionKernel; 91 | cl::Kernel _learnWeightsTracesInhibitedKernel; 92 | cl::Kernel _reconstructionErrorKernel; 93 | //!@} 94 | 95 | public: 96 | /*! 97 | \brief Create a comparison sparse coder with random initialization 98 | Requires the compute system, program with the NeoRL kernels, and initialization information 99 | */ 100 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 101 | const std::vector &visibleLayerDescs, cl_int2 hiddenSize, cl_float2 initWeightRange, 102 | std::mt19937 &rng); 103 | 104 | //!@{ 105 | /*! 106 | \brief Activate predictor 107 | */ 108 | void activate(sys::ComputeSystem &cs, const cl::Image2D &targets, const std::vector &visibleStates, const std::vector &visibleStatesPrev, float activeRatio, int inhibitionRadius, float noise, std::mt19937 &rng); 109 | void activateNoInhibition(sys::ComputeSystem &cs, const cl::Image2D &targets, const std::vector &visibleStates, const std::vector &visibleStatesPrev, float activeRatio, int inhibitionRadius, float noise, std::mt19937 &rng); 110 | //!@} 111 | 112 | //!@{ 113 | /*! 114 | \brief Learn with RL 115 | */ 116 | void learn(sys::ComputeSystem &cs, float reward, float gamma, const cl::Image2D &targets, std::vector &visibleStatesPrev, cl_float2 weightAlpha, cl_float2 weightLambda, cl_float biasAlpha, cl_float activeRatio, float noise); 117 | //!@} 118 | 119 | /*! 120 | \brief Get number of visible layers 121 | */ 122 | size_t getNumVisibleLayers() const { 123 | return _visibleLayers.size(); 124 | } 125 | 126 | /*! 127 | \brief Get access to a visible layer 128 | */ 129 | const VisibleLayer &getVisibleLayer(int index) const { 130 | return _visibleLayers[index]; 131 | } 132 | 133 | /*! 134 | \brief Get access to a visible layer desc 135 | */ 136 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 137 | return _visibleLayerDescs[index]; 138 | } 139 | 140 | /*! 141 | \brief Get hidden size 142 | */ 143 | cl_int2 getHiddenSize() const { 144 | return _hiddenSize; 145 | } 146 | 147 | /*! 148 | \brief Get hidden states 149 | */ 150 | const DoubleBuffer2D &getHiddenStates() const { 151 | return _hiddenStates; 152 | } 153 | }; 154 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/AgentPredQ.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "SparsePredictor.h" 4 | #include "ImageWhitener.h" 5 | 6 | namespace neo { 7 | /*! 8 | \brief Predictive hierarchy (no RL) 9 | */ 10 | class AgentPredQ { 11 | public: 12 | /*! 13 | \brief Layer desc 14 | */ 15 | struct LayerDesc { 16 | /*! 17 | \brief Size of layer 18 | */ 19 | cl_int2 _size; 20 | 21 | /*! 22 | \brief Radii 23 | */ 24 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 25 | 26 | //!@{ 27 | /*! 28 | \brief Sparse predictor parameters 29 | */ 30 | cl_float _spWeightEncodeAlpha; 31 | cl_float _spWeightDecodeAlpha; 32 | cl_float _spWeightLambda; 33 | cl_float _spActiveRatio; 34 | cl_float _spBiasAlpha; 35 | //!@} 36 | 37 | /*! 38 | \brief Initialize defaults 39 | */ 40 | LayerDesc() 41 | : _size({ 8, 8 }), 42 | _feedForwardRadius(5), _recurrentRadius(5), _lateralRadius(5), _feedBackRadius(6), _predictiveRadius(6), 43 | _spWeightEncodeAlpha(0.01f), _spWeightDecodeAlpha(0.01f), _spWeightLambda(0.95f), 44 | _spActiveRatio(0.02f), _spBiasAlpha(0.001f) 45 | {} 46 | }; 47 | 48 | /*! 49 | \brief Layer 50 | */ 51 | struct Layer { 52 | /*! 53 | \brief Sparse predictor 54 | */ 55 | SparsePredictor _sp; 56 | 57 | /*! 58 | \brief Layer for additional error signals 59 | */ 60 | cl::Image2D _additionalErrors; 61 | }; 62 | 63 | private: 64 | //!@{ 65 | /*! 66 | \brief Store sizes 67 | */ 68 | cl_int2 _inputSize; 69 | cl_int2 _actionSize; 70 | cl_int2 _qSize; 71 | //!@} 72 | 73 | //!@{ 74 | /*! 75 | \brief Layers and descs 76 | */ 77 | std::vector _layers; 78 | std::vector _layerDescs; 79 | 80 | cl::Image2D _qInputLayer; 81 | cl::Image2D _qRetrievalLayer; 82 | cl::Image2D _qTransforms; 83 | //!@} 84 | 85 | /*! 86 | \brief Input whitener 87 | */ 88 | ImageWhitener _inputWhitener; 89 | 90 | /*! 91 | \brief Zero layer for capping of the network 92 | */ 93 | cl::Image2D _zeroLayer; 94 | 95 | /*! 96 | \brief For RL 97 | */ 98 | float _prevValue; 99 | 100 | //!@{ 101 | /*! 102 | \brief Additional kernels 103 | */ 104 | cl::Kernel _setQKernel; 105 | cl::Kernel _getQKernel; 106 | //!@} 107 | 108 | public: 109 | //!@{ 110 | /*! 111 | \brief Whitening parameters 112 | */ 113 | cl_int _whiteningKernelRadius; 114 | cl_float _whiteningIntensity; 115 | //!@} 116 | 117 | //!@{ 118 | /*! 119 | \brief For RL 120 | */ 121 | float _qAlpha; 122 | float _qGamma; 123 | //!@} 124 | 125 | /*! 126 | \brief Initialize defaults 127 | */ 128 | AgentPredQ() 129 | : _whiteningKernelRadius(1), 130 | _whiteningIntensity(1024.0f), 131 | // RL 132 | _prevValue(0.0f), 133 | _qAlpha(0.5f), 134 | _qGamma(0.98f) 135 | {} 136 | 137 | /*! 138 | \brief Create a predictive hierarchy with random initialization 139 | Requires the compute system, program with the NeoRL kernels, and initialization information 140 | */ 141 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 142 | cl_int2 inputSize, cl_int2 actionSize, cl_int2 qSize, const std::vector &layerDescs, 143 | cl_float2 initWeightRange, 144 | std::mt19937 &rng); 145 | 146 | /*! 147 | \brief Simulation step of hierarchy 148 | */ 149 | void simStep(sys::ComputeSystem &cs, float reward, const cl::Image2D &input, const cl::Image2D &actionTaken, bool learn = true, bool whiten = false); 150 | 151 | /*! 152 | \brief Get number of layers 153 | */ 154 | size_t getNumLayers() const { 155 | return _layers.size(); 156 | } 157 | 158 | /*! 159 | \brief Get access to a layer 160 | */ 161 | const Layer &getLayer(int index) const { 162 | return _layers[index]; 163 | } 164 | 165 | /*! 166 | \brief Get access to a layer desc 167 | */ 168 | const LayerDesc &getLayerDescs(int index) const { 169 | return _layerDescs[index]; 170 | } 171 | 172 | /*! 173 | \brief Get the prediction 174 | */ 175 | const cl::Image2D &getAction() const { 176 | return _layers.front()._sp.getVisibleLayer(2)._predictions[_back]; 177 | } 178 | 179 | /*! 180 | \brief Get input whitener 181 | */ 182 | const ImageWhitener &getInputWhitener() const { 183 | return _inputWhitener; 184 | } 185 | }; 186 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/Predictor.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Predictor 8 | Maps from 2D SDRs to a 2D prediction 9 | */ 10 | class Predictor { 11 | public: 12 | /*! 13 | \brief Type of the nonlinearity 14 | */ 15 | enum NonlinearityType { 16 | _identity, _binary, _tanH 17 | }; 18 | 19 | /*! 20 | \brief Visible layer desc 21 | */ 22 | struct VisibleLayerDesc { 23 | /*! 24 | \brief Size of layer 25 | */ 26 | cl_int2 _size; 27 | 28 | /*! 29 | \brief Radius onto input 30 | */ 31 | cl_int _radius; 32 | 33 | /*! 34 | \brief Initialize defaults 35 | */ 36 | VisibleLayerDesc() 37 | : _size({ 8, 8 }), _radius(4) 38 | {} 39 | }; 40 | 41 | /*! 42 | \brief Visible layer 43 | */ 44 | struct VisibleLayer { 45 | /*! 46 | \brief Weights 47 | */ 48 | DoubleBuffer3D _weights; 49 | 50 | //!@{ 51 | /*! 52 | \brief Transformations 53 | */ 54 | cl_float2 _hiddenToVisible; 55 | cl_float2 _visibleToHidden; 56 | //!@} 57 | 58 | /*! 59 | \brief Radius onto hidden (reverse from visible layer desc) 60 | */ 61 | cl_int2 _reverseRadii; 62 | }; 63 | 64 | private: 65 | //!@{ 66 | /*! 67 | \brief Hidden states and activations 68 | */ 69 | DoubleBuffer2D _hiddenStates; 70 | //!@} 71 | 72 | /*! 73 | \brief Hidden size 74 | */ 75 | cl_int2 _hiddenSize; 76 | 77 | /*! 78 | \brief Hidden summation temprorary buffer 79 | */ 80 | DoubleBuffer2D _hiddenSummationTemp; 81 | 82 | //!@{ 83 | /*! 84 | \brief Layers and descs 85 | */ 86 | std::vector _visibleLayerDescs; 87 | std::vector _visibleLayers; 88 | //!@} 89 | 90 | //!@{ 91 | /*! 92 | \brief Kernels 93 | */ 94 | cl::Kernel _activateKernel; 95 | cl::Kernel _solveHiddenBinaryKernel; 96 | cl::Kernel _solveHiddenTanHKernel; 97 | cl::Kernel _learnWeightsKernel; 98 | cl::Kernel _learnWeightsTracesKernel; 99 | cl::Kernel _learnQWeightsTracesKernel; 100 | //!@} 101 | 102 | /*! 103 | \brief Remember if is using traces 104 | */ 105 | bool _useTraces; 106 | 107 | public: 108 | /*! 109 | \brief Create a comparison sparse coder with random initialization 110 | Requires the compute system, program with the NeoRL kernels, and initialization information 111 | */ 112 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 113 | const std::vector &visibleLayerDescs, cl_int2 hiddenSize, cl_float2 initWeightRange, 114 | bool useTraces, 115 | std::mt19937 &rng); 116 | 117 | /*! 118 | \brief Activate predictor 119 | */ 120 | void activate(sys::ComputeSystem &cs, const std::vector &visibleStates, NonlinearityType nonlinearityType, bool bufferSwap = true); 121 | 122 | //!@{ 123 | /*! 124 | \brief Learning functions 125 | */ 126 | void learn(sys::ComputeSystem &cs, const cl::Image2D &targets, std::vector &visibleStatesPrev, float weightAlpha); 127 | void learn(sys::ComputeSystem &cs, float tdError, const cl::Image2D &targets, std::vector &visibleStatesPrev, float weightAlpha, float weightLambda); 128 | void learnQ(sys::ComputeSystem &cs, float tdError, std::vector &visibleStatesPrev, float weightAlpha, float weightLambda); 129 | void learnCurrent(sys::ComputeSystem &cs, const cl::Image2D &targets, std::vector &visibleStates, float weightAlpha); 130 | //!@} 131 | 132 | /*! 133 | \brief Write to stream 134 | */ 135 | void writeToStream(sys::ComputeSystem &cs, std::ostream &os) const; 136 | 137 | /*! 138 | \brief Read from stream 139 | */ 140 | void readFromStream(sys::ComputeSystem &cs, sys::ComputeProgram &program, std::istream &is); 141 | 142 | /*! 143 | \brief Get number of visible layers 144 | */ 145 | size_t getNumVisibleLayers() const { 146 | return _visibleLayers.size(); 147 | } 148 | 149 | /*! 150 | \brief Get access to visible layer 151 | */ 152 | const VisibleLayer &getVisibleLayer(int index) const { 153 | return _visibleLayers[index]; 154 | } 155 | 156 | /*! 157 | \brief Get access to visible layer 158 | */ 159 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 160 | return _visibleLayerDescs[index]; 161 | } 162 | 163 | /*! 164 | \brief Get hidden size 165 | */ 166 | cl_int2 getHiddenSize() const { 167 | return _hiddenSize; 168 | } 169 | 170 | /*! 171 | \brief Get hidden states 172 | */ 173 | const DoubleBuffer2D &getHiddenStates() const { 174 | return _hiddenStates; 175 | } 176 | }; 177 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/Helpers.cpp: -------------------------------------------------------------------------------- 1 | #include "Helpers.h" 2 | 3 | using namespace neo; 4 | 5 | DoubleBuffer2D neo::createDoubleBuffer2D(sys::ComputeSystem &cs, cl_int2 size, cl_channel_order channelOrder, cl_channel_type channelType) { 6 | DoubleBuffer2D db; 7 | 8 | db[_front] = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(channelOrder, channelType), size.x, size.y); 9 | db[_back] = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(channelOrder, channelType), size.x, size.y); 10 | 11 | return db; 12 | } 13 | 14 | DoubleBuffer3D neo::createDoubleBuffer3D(sys::ComputeSystem &cs, cl_int3 size, cl_channel_order channelOrder, cl_channel_type channelType) { 15 | DoubleBuffer3D db; 16 | 17 | db[_front] = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(channelOrder, channelType), size.x, size.y, size.z); 18 | db[_back] = cl::Image3D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(channelOrder, channelType), size.x, size.y, size.z); 19 | 20 | return db; 21 | } 22 | 23 | void neo::randomUniform(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng) { 24 | int argIndex = 0; 25 | 26 | std::uniform_int_distribution seedDist(0, 999); 27 | 28 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 29 | 30 | randomUniform2DKernel.setArg(argIndex++, image2D); 31 | randomUniform2DKernel.setArg(argIndex++, seed); 32 | randomUniform2DKernel.setArg(argIndex++, range); 33 | 34 | cs.getQueue().enqueueNDRangeKernel(randomUniform2DKernel, cl::NullRange, cl::NDRange(size.x, size.y)); 35 | } 36 | 37 | void neo::randomUniform(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng) { 38 | int argIndex = 0; 39 | 40 | std::uniform_int_distribution seedDist(0, 999); 41 | 42 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 43 | 44 | randomUniform3DKernel.setArg(argIndex++, image3D); 45 | randomUniform3DKernel.setArg(argIndex++, seed); 46 | randomUniform3DKernel.setArg(argIndex++, range); 47 | 48 | cs.getQueue().enqueueNDRangeKernel(randomUniform3DKernel, cl::NullRange, cl::NDRange(size.x, size.y, size.z)); 49 | } 50 | 51 | void neo::randomUniformXY(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXYKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng) { 52 | int argIndex = 0; 53 | 54 | std::uniform_int_distribution seedDist(0, 999); 55 | 56 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 57 | 58 | randomUniform2DXYKernel.setArg(argIndex++, image2D); 59 | randomUniform2DXYKernel.setArg(argIndex++, seed); 60 | randomUniform2DXYKernel.setArg(argIndex++, range); 61 | 62 | cs.getQueue().enqueueNDRangeKernel(randomUniform2DXYKernel, cl::NullRange, cl::NDRange(size.x, size.y)); 63 | } 64 | 65 | void neo::randomUniformXYZ(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXYZKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng) { 66 | int argIndex = 0; 67 | 68 | std::uniform_int_distribution seedDist(0, 999); 69 | 70 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 71 | 72 | randomUniform2DXYZKernel.setArg(argIndex++, image2D); 73 | randomUniform2DXYZKernel.setArg(argIndex++, seed); 74 | randomUniform2DXYZKernel.setArg(argIndex++, range); 75 | 76 | cs.getQueue().enqueueNDRangeKernel(randomUniform2DXYZKernel, cl::NullRange, cl::NDRange(size.x, size.y)); 77 | } 78 | 79 | void neo::randomUniformXY(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DXYKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng) { 80 | int argIndex = 0; 81 | 82 | std::uniform_int_distribution seedDist(0, 999); 83 | 84 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 85 | 86 | randomUniform3DXYKernel.setArg(argIndex++, image3D); 87 | randomUniform3DXYKernel.setArg(argIndex++, seed); 88 | randomUniform3DXYKernel.setArg(argIndex++, range); 89 | 90 | cs.getQueue().enqueueNDRangeKernel(randomUniform3DXYKernel, cl::NullRange, cl::NDRange(size.x, size.y, size.z)); 91 | } 92 | 93 | void neo::randomUniformXZ(cl::Image2D &image2D, sys::ComputeSystem &cs, cl::Kernel &randomUniform2DXZKernel, cl_int2 size, cl_float2 range, std::mt19937 &rng) { 94 | int argIndex = 0; 95 | 96 | std::uniform_int_distribution seedDist(0, 999); 97 | 98 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 99 | 100 | randomUniform2DXZKernel.setArg(argIndex++, image2D); 101 | randomUniform2DXZKernel.setArg(argIndex++, seed); 102 | randomUniform2DXZKernel.setArg(argIndex++, range); 103 | 104 | cs.getQueue().enqueueNDRangeKernel(randomUniform2DXZKernel, cl::NullRange, cl::NDRange(size.x, size.y)); 105 | } 106 | 107 | void neo::randomUniformXZ(cl::Image3D &image3D, sys::ComputeSystem &cs, cl::Kernel &randomUniform3DXZKernel, cl_int3 size, cl_float2 range, std::mt19937 &rng) { 108 | int argIndex = 0; 109 | 110 | std::uniform_int_distribution seedDist(0, 999); 111 | 112 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 113 | 114 | randomUniform3DXZKernel.setArg(argIndex++, image3D); 115 | randomUniform3DXZKernel.setArg(argIndex++, seed); 116 | randomUniform3DXZKernel.setArg(argIndex++, range); 117 | 118 | cs.getQueue().enqueueNDRangeKernel(randomUniform3DXZKernel, cl::NullRange, cl::NDRange(size.x, size.y, size.z)); 119 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/AgentSPG.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ComparisonSparseCoder.h" 4 | #include "Predictor.h" 5 | #include "PredictorSwarm.h" 6 | #include "ImageWhitener.h" 7 | 8 | namespace neo { 9 | /*! 10 | \brief Policy gradient agent 11 | */ 12 | class AgentSPG { 13 | public: 14 | /*! 15 | \brief Layer desc 16 | */ 17 | struct LayerDesc { 18 | /*! 19 | \brief Size of layer 20 | */ 21 | cl_int2 _size; 22 | 23 | /*! 24 | \brief Radii 25 | */ 26 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 27 | 28 | //!@{ 29 | /*! 30 | \brief Sparse coder parameters 31 | */ 32 | cl_float _scWeightAlpha; 33 | cl_float _scWeightRecurrentAlpha; 34 | cl_float _scWeightLambda; 35 | cl_float _scActiveRatio; 36 | cl_float _scBoostAlpha; 37 | //!@} 38 | 39 | //!@{ 40 | /*! 41 | \brief RL 42 | */ 43 | cl_float2 _alpha; 44 | cl_float _gamma; 45 | cl_float2 _lambda; 46 | cl_float _noise; 47 | //!@} 48 | 49 | /*! 50 | \brief Initialize defaults 51 | */ 52 | LayerDesc() 53 | : _size({ 8, 8 }), 54 | _feedForwardRadius(5), _recurrentRadius(5), _lateralRadius(5), _feedBackRadius(6), _predictiveRadius(6), 55 | _scWeightAlpha(0.001f), _scWeightRecurrentAlpha(0.001f), _scWeightLambda(0.96f), 56 | _scActiveRatio(0.02f), _scBoostAlpha(0.01f), 57 | _alpha({ 0.01f, 0.01f }), _gamma(0.98f), _lambda({ 0.96f, 0.96f }), _noise(0.0f) 58 | {} 59 | }; 60 | 61 | /*! 62 | \brief Layer 63 | */ 64 | struct Layer { 65 | //!@{ 66 | /*! 67 | \brief Sparse coder and predictor 68 | */ 69 | ComparisonSparseCoder _sc; 70 | PredictorSwarm _pred; 71 | //!@} 72 | 73 | //!@{ 74 | /*! 75 | \brief For prediction reward determination 76 | */ 77 | cl::Image2D _predReward; 78 | cl::Image2D _propagatedPredReward; 79 | //!@} 80 | }; 81 | 82 | private: 83 | /*! 84 | \brief Store input size 85 | */ 86 | cl_int2 _inputSize; 87 | 88 | /*! 89 | \brief Store action size 90 | */ 91 | cl_int2 _actionSize; 92 | 93 | //!@{ 94 | /*! 95 | \brief Layers and descs 96 | */ 97 | std::vector _layers; 98 | std::vector _layerDescs; 99 | //!@} 100 | 101 | //!@{ 102 | /*! 103 | \brief Kernels for hierarchy 104 | */ 105 | cl::Kernel _predictionRewardKernel; 106 | cl::Kernel _predictionRewardPropagationKernel; 107 | //!@} 108 | 109 | //!@{ 110 | /*! 111 | \brief Input whiteners 112 | */ 113 | ImageWhitener _inputWhitener; 114 | ImageWhitener _actionWhitener; 115 | //!@} 116 | 117 | public: 118 | //!@{ 119 | /*! 120 | \brief Whitening parameters 121 | */ 122 | cl_int _whiteningKernelRadius; 123 | cl_float _whiteningIntensity; 124 | //!@} 125 | 126 | /*! 127 | \brief Action prediction parameters 128 | */ 129 | cl_float _actionPredAlpha; 130 | 131 | /*! 132 | \brief Initialize defaults 133 | */ 134 | AgentSPG() 135 | : _whiteningKernelRadius(2), 136 | _whiteningIntensity(1024.0f), 137 | _actionPredAlpha(0.1f) 138 | {} 139 | 140 | /*! 141 | \brief Create a comparison sparse coder with random initialization 142 | Requires the compute system, program with the NeoRL kernels, and initialization information. 143 | */ 144 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 145 | cl_int2 inputSize, cl_int2 actionSize, cl_int actionPredRadius, const std::vector &layerDescs, 146 | cl_float2 initWeightRange, 147 | std::mt19937 &rng); 148 | 149 | //!@{ 150 | /*! 151 | \brief Simulation step of hierarchy 152 | */ 153 | void simStep(sys::ComputeSystem &cs, float reward, const cl::Image2D &input, const cl::Image2D &actionTaken, std::mt19937 &rng, bool learn = true, bool useInputWhitener = true, bool binaryOutput = false); 154 | //!@} 155 | 156 | /*! 157 | \brief Clear working memory 158 | */ 159 | void clearMemory(sys::ComputeSystem &cs); 160 | 161 | /*! 162 | \brief Write to stream 163 | */ 164 | void writeToStream(sys::ComputeSystem &cs, std::ostream &os) const; 165 | 166 | /*! 167 | \brief Read from stream 168 | */ 169 | void readFromStream(sys::ComputeSystem &cs, sys::ComputeProgram &program, std::istream &is); 170 | 171 | /*! 172 | \brief Get number of layers 173 | */ 174 | size_t getNumLayers() const { 175 | return _layers.size(); 176 | } 177 | 178 | /*! 179 | \brief Get access to a layer 180 | */ 181 | const Layer &getLayer(int index) const { 182 | return _layers[index]; 183 | } 184 | 185 | /*! 186 | \brief Get access to a layer desc 187 | */ 188 | const LayerDesc &getLayerDescs(int index) const { 189 | return _layerDescs[index]; 190 | } 191 | 192 | /*! 193 | \brief Get exploratory action 194 | */ 195 | const cl::Image2D &getAction() const { 196 | return _layers.front()._pred.getHiddenStates()[_back]; 197 | } 198 | 199 | /*! 200 | \brief Get input whitener 201 | */ 202 | const ImageWhitener &getInputWhitener() const { 203 | return _inputWhitener; 204 | } 205 | 206 | /*! 207 | \brief Get action whitener 208 | */ 209 | const ImageWhitener &getActionWhitener() const { 210 | return _actionWhitener; 211 | } 212 | }; 213 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/SparseCoder.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Sparse coder 8 | Performs iterative sparse coding with explaining-away 9 | */ 10 | class SparseCoder { 11 | public: 12 | /*! 13 | \brief Visible layer desc 14 | */ 15 | struct VisibleLayerDesc { 16 | /*! 17 | \brief Size of layer 18 | */ 19 | cl_int2 _size; 20 | 21 | /*! 22 | \brief Radius 23 | */ 24 | cl_int _radius; 25 | 26 | //!@{ 27 | /*! 28 | \brief Learning parameters 29 | */ 30 | cl_float _weightAlpha; 31 | cl_float _weightLambda; 32 | //!@} 33 | 34 | /*! 35 | \brief Whether or not to ignore middle node 36 | When recurrent, ignore middle node (self) if desired 37 | */ 38 | bool _ignoreMiddle; 39 | 40 | /*! 41 | \brief Whether or not to use eligibility traces 42 | */ 43 | bool _useTraces; 44 | 45 | /*! 46 | \brief Initialize defaults 47 | */ 48 | VisibleLayerDesc() 49 | : _size({ 8, 8 }), _radius(4), _weightAlpha(0.01f), _weightLambda(0.95f), 50 | _ignoreMiddle(false), _useTraces(false) 51 | {} 52 | }; 53 | 54 | /*! 55 | \brief Visible layer 56 | */ 57 | struct VisibleLayer { 58 | /*! 59 | \brief Weights 60 | */ 61 | DoubleBuffer3D _weights; 62 | 63 | //!@{ 64 | /*! 65 | \brief Transformations 66 | */ 67 | cl_float2 _hiddenToVisible; 68 | cl_float2 _visibleToHidden; 69 | //!@} 70 | 71 | /*! 72 | \brief Radius onto hidden (reverse from visible layer desc) 73 | */ 74 | cl_int2 _reverseRadii; 75 | }; 76 | 77 | private: 78 | //!@{ 79 | /*! 80 | \brief Spiking, resulting state, activations, and neuron threshold buffers 81 | */ 82 | DoubleBuffer2D _hiddenSpikes; 83 | DoubleBuffer2D _hiddenStates; 84 | DoubleBuffer2D _hiddenActivations; 85 | DoubleBuffer2D _hiddenThresholds; 86 | //!@} 87 | 88 | /*! 89 | \brief Lateral (inhibition) weights 90 | */ 91 | DoubleBuffer3D _lateralWeights; 92 | 93 | /*! 94 | \brief Lateral (inhibition) radius 95 | */ 96 | cl_int _lateralRadius; 97 | 98 | /*! 99 | \brief Hidden size 100 | */ 101 | cl_int2 _hiddenSize; 102 | 103 | /*! 104 | \brief Summation temporary buffer 105 | */ 106 | DoubleBuffer2D _hiddenSummationTemp; 107 | 108 | //!@{ 109 | /*! 110 | \brief Visible layers and descs 111 | */ 112 | std::vector _visibleLayerDescs; 113 | std::vector _visibleLayers; 114 | //!@} 115 | 116 | //!@{ 117 | /*! 118 | \brief Kernels 119 | */ 120 | cl::Kernel _reconstructVisibleKernel; 121 | cl::Kernel _activateKernel; 122 | cl::Kernel _solveHiddenKernel; 123 | cl::Kernel _learnThresholdsKernel; 124 | cl::Kernel _learnWeightsKernel; 125 | cl::Kernel _learnWeightsTracesKernel; 126 | cl::Kernel _learnWeightsLateralKernel; 127 | //!@} 128 | 129 | public: 130 | /*! 131 | \brief Create a comparison sparse coder with random initialization 132 | Requires the compute system, program with the NeoRL kernels, and initialization information 133 | */ 134 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 135 | const std::vector &visibleLayerDescs, cl_int2 hiddenSize, cl_int lateralRadius, cl_float2 initWeightRange, cl_float2 initLateralWeightRange, cl_float initThreshold, 136 | std::mt19937 &rng); 137 | 138 | /*! 139 | \brief Create a comparison sparse coder with random initialization 140 | Requires the compute system, program with the NeoRL kernels, and initialization information 141 | */ 142 | void activate(sys::ComputeSystem &cs, const std::vector &visibleStates, cl_int iterations, cl_float leak); 143 | 144 | //!@{ 145 | /*! 146 | \brief Learn functions, with and without eligibility traces/rewards 147 | */ 148 | void learn(sys::ComputeSystem &cs, const std::vector &visibleStates, float weightLateralAlpha, float thresholdAlpha, float activeRatio); 149 | void learn(sys::ComputeSystem &cs, const cl::Image2D &rewards, const std::vector &visibleStates, float weightLateralAlpha, float thresholdAlpha, float activeRatio); 150 | //!@} 151 | 152 | /*! 153 | \brief Reconstruct (find input from sparse codes) 154 | */ 155 | void reconstruct(sys::ComputeSystem &cs, const cl::Image2D &hiddenStates, int visibleLayerIndex, cl::Image2D &visibleStates); 156 | 157 | /*! 158 | \brief Get number of visible layers 159 | */ 160 | size_t getNumVisibleLayers() const { 161 | return _visibleLayers.size(); 162 | } 163 | 164 | /*! 165 | \brief Get access to a visible layer 166 | */ 167 | const VisibleLayer &getVisibleLayer(int index) const { 168 | return _visibleLayers[index]; 169 | } 170 | 171 | /*! 172 | \brief Get visible layer descs 173 | */ 174 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 175 | return _visibleLayerDescs[index]; 176 | } 177 | 178 | /*! 179 | \brief Get hidden size 180 | */ 181 | cl_int2 getHiddenSize() const { 182 | return _hiddenSize; 183 | } 184 | 185 | /*! 186 | \brief Get hidden states 187 | */ 188 | const DoubleBuffer2D &getHiddenStates() const { 189 | return _hiddenStates; 190 | } 191 | 192 | /*! 193 | \brief Get hidden thresholds 194 | */ 195 | const DoubleBuffer2D &getHiddenThresholds() const { 196 | return _hiddenThresholds; 197 | } 198 | }; 199 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/PredictiveHierarchy.cpp: -------------------------------------------------------------------------------- 1 | #include "PredictiveHierarchy.h" 2 | 3 | using namespace neo; 4 | 5 | void PredictiveHierarchy::createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 6 | cl_int2 inputSize, const std::vector &layerDescs, 7 | cl_float2 initWeightRange, 8 | std::mt19937 &rng) 9 | { 10 | _inputSize = inputSize; 11 | 12 | _layerDescs = layerDescs; 13 | _layers.resize(_layerDescs.size()); 14 | 15 | cl_int2 prevLayerSize = inputSize; 16 | 17 | for (int l = 0; l < _layers.size(); l++) { 18 | std::vector spDescs; 19 | 20 | if (l == 0) { 21 | spDescs.resize(2); 22 | 23 | spDescs[0]._size = prevLayerSize; 24 | spDescs[0]._encodeRadius = _layerDescs[l]._feedForwardRadius; 25 | spDescs[0]._predDecodeRadius = _layerDescs[l]._predictiveRadius; 26 | spDescs[0]._feedBackDecodeRadius = _layerDescs[l]._feedBackRadius; 27 | spDescs[0]._predictThresholded = false; 28 | spDescs[0]._predict = true; 29 | spDescs[0]._ignoreMiddle = false; 30 | spDescs[0]._useForInput = true; 31 | 32 | spDescs[1]._size = _layerDescs[l]._size; 33 | spDescs[1]._encodeRadius = _layerDescs[l]._recurrentRadius; 34 | spDescs[1]._predDecodeRadius = _layerDescs[l]._predictiveRadius; 35 | spDescs[1]._feedBackDecodeRadius = _layerDescs[l]._feedBackRadius; 36 | spDescs[1]._predictThresholded = true; 37 | spDescs[1]._predict = false; 38 | spDescs[1]._ignoreMiddle = true; 39 | spDescs[1]._useForInput = false; 40 | } 41 | else { 42 | spDescs.resize(2); 43 | 44 | spDescs[0]._size = prevLayerSize; 45 | spDescs[0]._encodeRadius = _layerDescs[l]._feedForwardRadius; 46 | spDescs[0]._predDecodeRadius = _layerDescs[l]._predictiveRadius; 47 | spDescs[0]._feedBackDecodeRadius = _layerDescs[l]._feedBackRadius; 48 | spDescs[0]._predictThresholded = true; 49 | spDescs[0]._predict = true; 50 | spDescs[0]._ignoreMiddle = false; 51 | spDescs[0]._useForInput = false; 52 | 53 | spDescs[1]._size = _layerDescs[l]._size; 54 | spDescs[1]._encodeRadius = _layerDescs[l]._recurrentRadius; 55 | spDescs[1]._predDecodeRadius = _layerDescs[l]._predictiveRadius; 56 | spDescs[1]._feedBackDecodeRadius = _layerDescs[l]._feedBackRadius; 57 | spDescs[1]._predictThresholded = true; 58 | spDescs[1]._predict = false; 59 | spDescs[1]._ignoreMiddle = true; 60 | spDescs[1]._useForInput = false; 61 | } 62 | 63 | std::vector feedBackSizes(2); 64 | 65 | if (l < _layers.size() - 1) 66 | feedBackSizes[0] = feedBackSizes[1] = _layerDescs[l]._size; 67 | else 68 | feedBackSizes[0] = feedBackSizes[1] = { 1, 1 }; 69 | 70 | _layers[l]._sp.createRandom(cs, program, spDescs, _layerDescs[l]._size, feedBackSizes, _layerDescs[l]._lateralRadius, initWeightRange, rng); 71 | 72 | _layers[l]._additionalErrors = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), prevLayerSize.x, prevLayerSize.y); 73 | 74 | cs.getQueue().enqueueFillImage(_layers[l]._additionalErrors, cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, { 0, 0, 0 }, { static_cast(prevLayerSize.x), static_cast(prevLayerSize.y), 1 }); 75 | 76 | prevLayerSize = _layerDescs[l]._size; 77 | } 78 | 79 | _inputWhitener.create(cs, program, _inputSize, CL_R, CL_FLOAT); 80 | 81 | _zeroLayer = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 1, 1); 82 | 83 | cs.getQueue().enqueueFillImage(_zeroLayer, cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, { 0, 0, 0 }, { 1, 1, 1 }); 84 | } 85 | 86 | void PredictiveHierarchy::simStep(sys::ComputeSystem &cs, const cl::Image2D &input, bool learn, bool whiten) { 87 | // Whiten input 88 | if (whiten) 89 | _inputWhitener.filter(cs, input, _whiteningKernelRadius, _whiteningIntensity); 90 | 91 | // Feed forward 92 | cl::Image2D prevLayerState = whiten ? _inputWhitener.getResult() : input; 93 | 94 | for (int l = 0; l < _layers.size(); l++) { 95 | std::vector visibleStates(2); 96 | 97 | visibleStates[0] = prevLayerState; 98 | visibleStates[1] = _layers[l]._sp.getHiddenStates()[_back]; 99 | 100 | _layers[l]._sp.activateEncoder(cs, visibleStates, _layerDescs[l]._spActiveRatio); 101 | 102 | prevLayerState = _layers[l]._sp.getHiddenStates()[_front]; 103 | } 104 | 105 | // Feed back 106 | for (int l = _layers.size() - 1; l >= 0; l--) { 107 | std::vector feedBackStates(2); 108 | 109 | if (l < _layers.size() - 1) 110 | feedBackStates[0] = feedBackStates[1] = _layers[l + 1]._sp.getVisibleLayer(0)._predictions[_back]; 111 | else 112 | feedBackStates[0] = feedBackStates[1] = _zeroLayer; 113 | 114 | _layers[l]._sp.activateDecoder(cs, feedBackStates); 115 | } 116 | 117 | if (learn) { 118 | // Feed forward 119 | prevLayerState = input; 120 | 121 | for (int l = 0; l < _layers.size(); l++) { 122 | // Encoder 123 | std::vector visibleStates(2); 124 | 125 | visibleStates[0] = prevLayerState; 126 | visibleStates[1] = _layers[l]._sp.getHiddenStates()[_front]; 127 | 128 | std::vector feedBackStatesPrev(2); 129 | 130 | if (l < _layers.size() - 1) 131 | feedBackStatesPrev[0] = feedBackStatesPrev[1] = _layers[l + 1]._sp.getVisibleLayer(0)._predictions[_front]; 132 | else 133 | feedBackStatesPrev[0] = feedBackStatesPrev[1] = _zeroLayer; 134 | 135 | _layers[l]._sp.learn(cs, visibleStates, feedBackStatesPrev, { _layers[l]._additionalErrors, _layers[l]._additionalErrors }, 136 | _layerDescs[l]._spWeightEncodeAlpha, _layerDescs[l]._spWeightDecodeAlpha, _layerDescs[l]._spWeightLambda, _layerDescs[l]._spBiasAlpha, _layerDescs[l]._spActiveRatio); 137 | 138 | prevLayerState = _layers[l]._sp.getHiddenStates()[_back]; 139 | } 140 | } 141 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/Swarm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Swarm of overlapping SDRRL units 8 | Selects actions using deterministic policy gradients 9 | */ 10 | class Swarm { 11 | public: 12 | /*! 13 | \brief Visible layer desc 14 | */ 15 | struct VisibleLayerDesc { 16 | /*! 17 | \brief Size of layer 18 | */ 19 | cl_int2 _size; 20 | 21 | //!@{ 22 | /*! 23 | \brief Radii 24 | */ 25 | cl_int _qRadius; 26 | cl_int _hiddenRadius; 27 | cl_int _startRadius; 28 | //!@} 29 | 30 | /*! 31 | \brief Initialize defaults 32 | */ 33 | VisibleLayerDesc() 34 | : _size({ 8, 8 }), _qRadius(4), _hiddenRadius(4), _startRadius(4) 35 | {} 36 | }; 37 | 38 | /*! 39 | \brief Visibile layer 40 | */ 41 | struct VisibleLayer { 42 | /*! 43 | \brief Starting predicted action (non-exploratory) 44 | */ 45 | cl::Image2D _predictedAction; 46 | 47 | //!@{ 48 | /*! 49 | \brief Actions, exporatory and non-exploratory 50 | */ 51 | cl::Image2D _actions; 52 | cl::Image2D _actionsExploratory; 53 | //!@} 54 | 55 | //!@{ 56 | /*! 57 | \brief Weights for Q values and starting action prediction 58 | */ 59 | DoubleBuffer3D _qWeights; 60 | DoubleBuffer3D _startWeights; 61 | //!@} 62 | 63 | //!@{ 64 | /*! 65 | \brief Transformations 66 | */ 67 | cl_float2 _hiddenToVisible; 68 | cl_float2 _visibleToHidden; 69 | //!@} 70 | 71 | /*! 72 | \brief Radius onto hidden (reverse from visible layer desc) 73 | */ 74 | cl_int2 _reverseQRadii; 75 | }; 76 | 77 | private: 78 | //!@{ 79 | /*! 80 | \brief Q states, hidden states, and biases 81 | */ 82 | DoubleBuffer2D _qStates; 83 | DoubleBuffer2D _hiddenStates; 84 | DoubleBuffer2D _hiddenBiases; 85 | //!@} 86 | 87 | /*! 88 | \brief Q Weights 89 | */ 90 | DoubleBuffer3D _qWeights; 91 | 92 | //!@{ 93 | /*! 94 | \brief Hidden errors and hidden temporal differences 95 | */ 96 | cl::Image2D _hiddenErrors; 97 | cl::Image2D _hiddenTD; 98 | //!@} 99 | 100 | //!@{ 101 | /*! 102 | \brief Q and hidden sizes 103 | */ 104 | cl_int2 _qSize; 105 | cl_int2 _hiddenSize; 106 | //!@} 107 | 108 | /*! 109 | \brief Q radius 110 | */ 111 | int _qRadius; 112 | 113 | //!@{ 114 | /*! 115 | \brief Q transformations 116 | */ 117 | cl_float2 _qToHidden; 118 | cl_float2 _hiddenToQ; 119 | //!@} 120 | 121 | /*! 122 | \brief Q radius onto Q layer (reverse from hidden layer) 123 | */ 124 | cl_int2 _reverseQRadii; 125 | 126 | /*! 127 | \brief Hidden summation temporary buffer 128 | */ 129 | DoubleBuffer2D _hiddenSummationTemp; 130 | 131 | //!@{ 132 | /*! 133 | \brief Visible layers and descs 134 | */ 135 | std::vector _visibleLayerDescs; 136 | std::vector _visibleLayers; 137 | //!@} 138 | 139 | //!@{ 140 | /*! 141 | \brief Kernels 142 | */ 143 | cl::Kernel _predictAction; 144 | cl::Kernel _qInitSummationKernel; 145 | cl::Kernel _qActivateToHiddenKernel; 146 | cl::Kernel _qActivateToQKernel; 147 | cl::Kernel _qSolveHiddenKernel; 148 | cl::Kernel _explorationKernel; 149 | cl::Kernel _qPropagateToHiddenErrorKernel; 150 | cl::Kernel _qPropagateToHiddenTDKernel; 151 | cl::Kernel _hiddenPropagateToVisibleActionKernel; 152 | cl::Kernel _startLearnWeightsKernel; 153 | cl::Kernel _qLearnVisibleWeightsTracesKernel; 154 | cl::Kernel _qLearnHiddenWeightsTracesKernel; 155 | cl::Kernel _qLearnHiddenBiasesTracesKernel; 156 | //!@} 157 | 158 | public: 159 | /*! 160 | \brief Create a comparison sparse coder with random initialization 161 | Requires the compute system, program with the NeoRL kernels, and initialization information 162 | */ 163 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 164 | const std::vector &visibleLayerDescs, cl_int2 qSize, cl_int2 hiddenSize, int qRadius, cl_float2 initWeightRange, 165 | std::mt19937 &rng); 166 | 167 | /*! 168 | \brief Simulation stemp 169 | Requires the compute system, gating states, and RL parameters 170 | */ 171 | void simStep(sys::ComputeSystem &cs, float reward, 172 | const cl::Image2D &hiddenStatesFeedForward, const cl::Image2D &actionsFeedBack, 173 | float expPert, float expBreak, int annealIterations, float actionAlpha, 174 | float alphaHiddenQ, float alphaQ, float alphaPred, float lambda, float gamma, std::mt19937 &rng); 175 | 176 | /*! 177 | \brief Get number of visible layers 178 | */ 179 | size_t getNumVisibleLayers() const { 180 | return _visibleLayers.size(); 181 | } 182 | 183 | /*! 184 | \brief Get access to a visible layer 185 | */ 186 | const VisibleLayer &getVisibleLayer(int index) const { 187 | return _visibleLayers[index]; 188 | } 189 | 190 | /*! 191 | \brief Get access to a visible layer desc 192 | */ 193 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 194 | return _visibleLayerDescs[index]; 195 | } 196 | 197 | /*! 198 | \brief Get hidden layer size 199 | */ 200 | cl_int2 getHiddenSize() const { 201 | return _hiddenSize; 202 | } 203 | 204 | /*! 205 | \brief Get hidden states 206 | */ 207 | const DoubleBuffer2D &getHiddenStates() const { 208 | return _hiddenStates; 209 | } 210 | 211 | /*! 212 | \brief Get hidden errors 213 | */ 214 | const cl::Image2D &getHiddenErrors() const { 215 | return _hiddenErrors; 216 | } 217 | 218 | /*! 219 | \brief Get hidden temporal differences 220 | */ 221 | const cl::Image2D &getHiddenTD() const { 222 | return _hiddenTD; 223 | } 224 | }; 225 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/SparsePredictor.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Sparse predictor 8 | Learns a sparse code that is then used to predict the next input. Can be used with multiple layers 9 | */ 10 | class SparsePredictor { 11 | public: 12 | /*! 13 | \brief Visible layer desc 14 | */ 15 | struct VisibleLayerDesc { 16 | /*! 17 | \brief Size of layer 18 | */ 19 | cl_int2 _size; 20 | 21 | /*! 22 | \brief Radius onto input 23 | */ 24 | cl_int _encodeRadius; 25 | 26 | /*! 27 | \brief Radius onto hidden 28 | */ 29 | cl_int _predDecodeRadius; 30 | 31 | /*! 32 | \brief Radius onto higher prediction 33 | */ 34 | cl_int _feedBackDecodeRadius; 35 | 36 | /*! 37 | \brief Whether or not the predictions should be binary 38 | */ 39 | unsigned char _predictThresholded; 40 | 41 | /*! 42 | \brief Whether or not the middle (center) input should be ignored (self in recurrent schemes) 43 | */ 44 | unsigned char _ignoreMiddle; 45 | 46 | /*! 47 | \brief Whether this layer should be predicted 48 | */ 49 | bool _predict; 50 | 51 | /*! 52 | \brief Whether this layer should be used as input (encoded) 53 | */ 54 | bool _useForInput; 55 | 56 | /*! 57 | \brief Initialize defaults 58 | */ 59 | VisibleLayerDesc() 60 | : _size({ 8, 8 }), _encodeRadius(4), _predDecodeRadius(4), _feedBackDecodeRadius(4), 61 | _predictThresholded(true), _ignoreMiddle(false), _predict(true), _useForInput(true) 62 | {} 63 | }; 64 | 65 | /*! 66 | \brief Visible layer 67 | */ 68 | struct VisibleLayer { 69 | /*! 70 | \brief Predictions 71 | */ 72 | DoubleBuffer2D _predictions; 73 | 74 | //!@{ 75 | /*! 76 | \brief Temporary error buffers 77 | */ 78 | cl::Image2D _predError; 79 | //!@} 80 | 81 | //!@{ 82 | /*! 83 | \brief Weights 84 | */ 85 | DoubleBuffer3D _encoderWeights; // Encoding weights (creates spatio-temporal sparse code) 86 | DoubleBuffer3D _predDecoderWeights; // Predictive decoding weights (points to t + 1) 87 | DoubleBuffer3D _feedBackDecoderWeights; // Feed back decoding weights (points to t + 1) 88 | //!@} 89 | 90 | //!@{ 91 | /*! 92 | \brief Transformations 93 | */ 94 | cl_float2 _hiddenToVisible; 95 | cl_float2 _visibleToHidden; 96 | cl_float2 _visibleToFeedBack; 97 | //!@} 98 | }; 99 | 100 | private: 101 | //!@{ 102 | /*! 103 | \brief Hidden states, biases 104 | */ 105 | DoubleBuffer2D _hiddenStates; 106 | DoubleBuffer2D _hiddenBiases; 107 | //!@} 108 | 109 | /*! 110 | \brief Hidden size 111 | */ 112 | cl_int2 _hiddenSize; 113 | 114 | /*! 115 | \brief Feed back size 116 | */ 117 | std::vector _feedBackSizes; 118 | 119 | /*! 120 | \brief Lateral (inhibitory) radius 121 | */ 122 | cl_int _lateralRadius; 123 | 124 | /*! 125 | \brief Hidden activation summation temporary buffer 126 | */ 127 | DoubleBuffer2D _hiddenActivationSummationTemp; 128 | 129 | /*! 130 | \brief Hidden error summation temporary buffer 131 | */ 132 | DoubleBuffer2D _hiddenErrorSummationTemp; 133 | 134 | //!@{ 135 | /*! 136 | \brief Layers and descs 137 | */ 138 | std::vector _visibleLayerDescs; 139 | std::vector _visibleLayers; 140 | //!@} 141 | 142 | //!@{ 143 | /*! 144 | \brief Kernels 145 | */ 146 | cl::Kernel _encodeKernel; 147 | cl::Kernel _decodeKernel; 148 | cl::Kernel _solveHiddenKernel; 149 | cl::Kernel _predictionErrorKernel; 150 | cl::Kernel _errorPropagationKernel; 151 | cl::Kernel _learnEncoderWeightsKernel; 152 | cl::Kernel _learnDecoderWeightsKernel; 153 | cl::Kernel _learnBiasesKernel; 154 | //!@} 155 | 156 | public: 157 | /*! 158 | \brief Create a comparison sparse coder with random initialization 159 | Requires the compute system, program with the NeoRL kernels, and initialization information 160 | */ 161 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 162 | const std::vector &visibleLayerDescs, cl_int2 hiddenSize, const std::vector &feedBackSizes, cl_int lateralRadius, cl_float2 initWeightRange, 163 | std::mt19937 &rng); 164 | 165 | /*! 166 | \brief Activate predictor 167 | */ 168 | void activateEncoder(sys::ComputeSystem &cs, const std::vector &visibleStates, float activeRatio); 169 | void activateDecoder(sys::ComputeSystem &cs, const std::vector &feedBackStates); 170 | 171 | //!@{ 172 | /*! 173 | \brief Learning functions 174 | */ 175 | void learn(sys::ComputeSystem &cs, const std::vector &visibleStates, 176 | const std::vector &feedBackStatesPrev, const std::vector &addidionalErrors, 177 | float weightEncodeAlpha, float weightDecodeAlpha, float weightLambda, float biasAlpha, float activeRatio); 178 | //!@} 179 | 180 | /*! 181 | \brief Get number of visible layers 182 | */ 183 | size_t getNumVisibleLayers() const { 184 | return _visibleLayers.size(); 185 | } 186 | 187 | /*! 188 | \brief Get access to visible layer 189 | */ 190 | const VisibleLayer &getVisibleLayer(int index) const { 191 | return _visibleLayers[index]; 192 | } 193 | 194 | /*! 195 | \brief Get access to visible layer 196 | */ 197 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 198 | return _visibleLayerDescs[index]; 199 | } 200 | 201 | /*! 202 | \brief Get hidden size 203 | */ 204 | cl_int2 getHiddenSize() const { 205 | return _hiddenSize; 206 | } 207 | 208 | /*! 209 | \brief Get hidden states 210 | */ 211 | const DoubleBuffer2D &getHiddenStates() const { 212 | return _hiddenStates; 213 | } 214 | }; 215 | } -------------------------------------------------------------------------------- /NeoRL/source/Binh_Test.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_BINH_TEST 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | #include 10 | 11 | #include 12 | #include 13 | #include 14 | 15 | int main() 16 | { 17 | std::mt19937 generator(time(nullptr)); 18 | 19 | sys::ComputeSystem cs; 20 | 21 | cs.create(sys::ComputeSystem::_gpu); 22 | 23 | sys::ComputeProgram prog; 24 | 25 | prog.loadFromFile("resources/neoKernels2.cl", cs); 26 | 27 | // --------------------------- Create the Sparse Coder --------------------------- 28 | 29 | std::vector inputBuffer(5 * 5, 0.0f); 30 | std::vector inputTransform(2 * (5 * 5 - 1), 0.0f); 31 | 32 | std::uniform_real_distribution dist01(0.0f, 1.0f); 33 | 34 | for (int i = 0; i < inputTransform.size(); i++) 35 | inputTransform[i] = dist01(generator) * 2.0f - 1.0f; 36 | 37 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 5, 5); 38 | 39 | std::vector layerDescs(3); 40 | 41 | layerDescs[0]._size = { 8, 8 }; 42 | layerDescs[0]._predictiveRadius = 8; 43 | layerDescs[0]._feedBackRadius = 8; 44 | layerDescs[1]._size = { 8, 8 }; 45 | layerDescs[2]._size = { 8, 8 }; 46 | 47 | neo::PredictiveHierarchy ph; 48 | 49 | ph.createRandom(cs, prog, { 5, 5 }, layerDescs, { -0.2f, 0.2f }, generator); 50 | 51 | sf::RenderWindow renderWindow; 52 | 53 | renderWindow.create(sf::VideoMode(1200, 600), "Binh's Test", sf::Style::Default); 54 | 55 | renderWindow.setVerticalSyncEnabled(true); 56 | renderWindow.setFramerateLimit(60); 57 | 58 | vis::Plot plot; 59 | plot._curves.resize(2); 60 | plot._curves[0]._shadow = 0.1f; // input 61 | plot._curves[1]._shadow = 0.1f; // predict 62 | 63 | sf::RenderTexture plotRT; 64 | plotRT.create(1200, 600, false); 65 | 66 | sf::Texture lineGradient; 67 | lineGradient.loadFromFile("resources/lineGradient.png"); 68 | 69 | sf::Font tickFont; 70 | tickFont.loadFromFile("resources/arial.ttf"); 71 | 72 | float minCurve = -5.0f; 73 | float maxCurve = 5.0f; 74 | 75 | plotRT.setActive(); 76 | plotRT.clear(sf::Color::White); 77 | 78 | const int maxBufferSize = 50; 79 | 80 | bool quit = false; 81 | bool autoplay = false; 82 | 83 | bool sPrev = false; 84 | 85 | int index = -1; 86 | do { 87 | sf::Event event; 88 | 89 | while (renderWindow.pollEvent(event)) { 90 | switch (event.type) { 91 | case sf::Event::Closed: 92 | quit = true; 93 | break; 94 | } 95 | } 96 | 97 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape)) 98 | quit = true; 99 | 100 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Space)) 101 | autoplay = false; 102 | 103 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::C)) 104 | autoplay = true; 105 | 106 | sPrev = sf::Keyboard::isKeyPressed(sf::Keyboard::S); 107 | 108 | if (autoplay || sf::Keyboard::isKeyPressed(sf::Keyboard::Right)) { 109 | index++; 110 | 111 | float value = std::sin(0.164f * 3.141596f * index + 0.25f) + 0.7f * std::sin(0.12352f * 3.141596f * index * 1.5f + 0.2154f) + 0.5f * std::sin(0.0612f * 3.141596f * index * 3.0f - 0.2112f); 112 | 113 | inputBuffer[0] = value; 114 | 115 | for (int i = 1; i < inputBuffer.size(); i++) { 116 | inputBuffer[i] = value * inputTransform[2 * (i - 1) + 0] + inputTransform[2 * (i - 1) + 1]; 117 | } 118 | 119 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 5, 5, 1 }, 0, 0, inputBuffer.data()); 120 | 121 | ph.simStep(cs, inputImage, true, true); 122 | 123 | std::vector res(inputBuffer.size()); 124 | 125 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 5, 5, 1 }, 0, 0, res.data()); 126 | 127 | float v = res[0]; 128 | 129 | // Plot target data 130 | vis::Point p; 131 | p._position.x = index; 132 | p._position.y = value; 133 | p._color = sf::Color::Red; 134 | plot._curves[0]._points.push_back(p); 135 | 136 | // Plot predicted data 137 | vis::Point p1; 138 | p1._position.x = index; 139 | p1._position.y = v; 140 | p1._color = sf::Color::Blue; 141 | plot._curves[1]._points.push_back(p1); 142 | 143 | if (plot._curves[0]._points.size() > maxBufferSize) { 144 | plot._curves[0]._points.erase(plot._curves[0]._points.begin()); 145 | 146 | int firstIndex = 0; 147 | 148 | for (std::vector::iterator it = plot._curves[0]._points.begin(); it != plot._curves[0]._points.end(); ++it, ++firstIndex) 149 | (*it)._position.x = firstIndex; 150 | 151 | plot._curves[1]._points.erase(plot._curves[1]._points.begin()); 152 | 153 | firstIndex = 0; 154 | 155 | for (std::vector::iterator it = plot._curves[1]._points.begin(); it != plot._curves[1]._points.end(); ++it, ++firstIndex) 156 | (*it)._position.x = firstIndex; 157 | } 158 | 159 | renderWindow.clear(); 160 | 161 | plot.draw(plotRT, lineGradient, tickFont, 0.5f, sf::Vector2f(0.0f, plot._curves[0]._points.size()), sf::Vector2f(minCurve, maxCurve), sf::Vector2f(64.0f, 64.0f), sf::Vector2f(plot._curves[0]._points.size() / 10.0f, (maxCurve - minCurve) / 10.0f), 2.0f, 4.0f, 2.0f, 6.0f, 2.0f, 4); 162 | 163 | plotRT.display(); 164 | 165 | sf::Sprite plotSprite; 166 | plotSprite.setTexture(plotRT.getTexture()); 167 | 168 | renderWindow.draw(plotSprite); 169 | renderWindow.display(); 170 | 171 | std::vector data(64); 172 | 173 | cs.getQueue().enqueueReadImage(ph.getLayer(0)._sp.getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { 8, 8, 1 }, 0, 0, data.data()); 174 | 175 | for (int x = 0; x < 8; x++) { 176 | 177 | for (int y = 0; y < 8; y++) 178 | std::cout << data[x + y * 8] << " "; 179 | 180 | std::cout << std::endl; 181 | } 182 | } 183 | } while (!quit); 184 | 185 | return 0; 186 | 187 | } 188 | 189 | 190 | #endif -------------------------------------------------------------------------------- /NeoRL/source/neo/ComparisonSparseCoder.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "Helpers.h" 4 | 5 | namespace neo { 6 | /*! 7 | \brief Comparison sparse coder 8 | Creates 2D sparse codes for a set of input layers 9 | */ 10 | class ComparisonSparseCoder { 11 | public: 12 | /*! 13 | \brief Desc for a visible layer 14 | */ 15 | struct VisibleLayerDesc { 16 | /*! 17 | \brief Whether this layer is predictively coding 18 | */ 19 | bool _isPredictiveCoding; 20 | 21 | /*! 22 | \brief Size of visible layer 23 | */ 24 | cl_int2 _size; 25 | 26 | /*! 27 | \brief Radius onto the visible layer 28 | */ 29 | cl_int _radius; 30 | 31 | //!@{ 32 | /*! 33 | \brief Learning parameters 34 | */ 35 | cl_float _weightAlpha; 36 | cl_float _weightLambda; 37 | //!@} 38 | 39 | /*! 40 | \brief Whether or not the center neuron (self in recurrent schemes) should be ignored 41 | */ 42 | bool _ignoreMiddle; 43 | 44 | /*! 45 | \brief Whether or not to use eligibility traces 46 | */ 47 | bool _useTraces; 48 | 49 | /*! 50 | \brief Initialize defaults 51 | */ 52 | VisibleLayerDesc() 53 | : _isPredictiveCoding(false), 54 | _size({ 8, 8 }), _radius(4), _weightAlpha(0.0001f), _weightLambda(0.95f), 55 | _ignoreMiddle(false), _useTraces(false) 56 | {} 57 | }; 58 | 59 | /*! 60 | \brief Visible layer 61 | */ 62 | struct VisibleLayer { 63 | //!@{ 64 | /*! 65 | \brief Weights 66 | */ 67 | DoubleBuffer3D _weights; 68 | //!@} 69 | 70 | //!@{ 71 | /*! 72 | \brief Transformations 73 | */ 74 | cl_float2 _hiddenToVisible; 75 | cl_float2 _visibleToHidden; 76 | //!@} 77 | 78 | /*! 79 | \brief Radius onto hidden (reverse from visible layer desc) 80 | */ 81 | cl_int2 _reverseRadii; 82 | }; 83 | 84 | private: 85 | //!@{ 86 | /*! 87 | \brief Hidden states and biases 88 | */ 89 | DoubleBuffer2D _hiddenStates; 90 | DoubleBuffer2D _hiddenBiases; 91 | //!@} 92 | 93 | /*! 94 | \brief Lateral (inhibition) radius 95 | */ 96 | cl_int _lateralRadius; 97 | 98 | /*! 99 | \brief Hidden size 100 | */ 101 | cl_int2 _hiddenSize; 102 | 103 | //!@{ 104 | /*! 105 | \brief Temporary summation buffers 106 | */ 107 | DoubleBuffer2D _hiddenActivationSummationTemp; 108 | DoubleBuffer2D _hiddenPredictionSummationTemp; 109 | //!@} 110 | 111 | //!@{ 112 | /*! 113 | \brief Descs and layers 114 | */ 115 | std::vector _visibleLayerDescs; 116 | std::vector _visibleLayers; 117 | //!@} 118 | 119 | //!@{ 120 | /*! 121 | \brief Kernels 122 | */ 123 | cl::Kernel _activateKernel; 124 | cl::Kernel _activateIgnoreMiddleKernel; 125 | cl::Kernel _solveHiddenKernel; 126 | cl::Kernel _learnHiddenBiasesKernel; 127 | cl::Kernel _learnHiddenWeightsActivationKernel; 128 | cl::Kernel _learnHiddenWeightsTracesActivationKernel; 129 | cl::Kernel _learnHiddenWeightsPredictionKernel; 130 | cl::Kernel _learnHiddenWeightsTracesPredictionKernel; 131 | cl::Kernel _forwardKernel; 132 | //!@} 133 | 134 | public: 135 | /*! 136 | \brief Create a comparison sparse coder with random initialization 137 | Requires the compute system, program with the NeoRL kernels, and initialization information 138 | */ 139 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 140 | const std::vector &visibleLayerDescs, 141 | cl_int2 hiddenSize, cl_int lateralRadius, cl_float2 initWeightRange, 142 | std::mt19937 &rng); 143 | 144 | /*! 145 | \brief Activate (find sparse codes) 146 | */ 147 | void activate(sys::ComputeSystem &cs, const std::vector &visibleStates, float activeRatio, bool bufferSwap = true); 148 | 149 | /*! 150 | \brief Reconstruct (find input from sparse codes) 151 | */ 152 | void reconstruct(sys::ComputeSystem &cs, const cl::Image2D &hiddenStates, int visibleLayerIndex, cl::Image2D &visibleStates); 153 | 154 | //!@{ 155 | /*! 156 | \brief Learn, with and without use of rewards + eligibility traces 157 | */ 158 | void learn(sys::ComputeSystem &cs, const std::vector &visibleStates, float boostAlpha, float activeRatio); 159 | void learn(sys::ComputeSystem &cs, const cl::Image2D &rewards, std::vector &visibleStates, float boostAlpha, float activeRatio); 160 | //!@} 161 | 162 | /*! 163 | \brief Clear working memory 164 | */ 165 | void clearMemory(sys::ComputeSystem &cs); 166 | 167 | /*! 168 | \brief Write to stream 169 | */ 170 | void writeToStream(sys::ComputeSystem &cs, std::ostream &os) const; 171 | 172 | /*! 173 | \brief Read from stream 174 | */ 175 | void readFromStream(sys::ComputeSystem &cs, sys::ComputeProgram &program, std::istream &is); 176 | 177 | /*! 178 | \brief Get number of visible layers 179 | */ 180 | size_t getNumVisibleLayers() const { 181 | return _visibleLayers.size(); 182 | } 183 | 184 | /*! 185 | \brief Get access to a visible layer 186 | */ 187 | const VisibleLayer &getVisibleLayer(int index) const { 188 | return _visibleLayers[index]; 189 | } 190 | 191 | /*! 192 | \brief Get access to a visible layer desc 193 | */ 194 | const VisibleLayerDesc &getVisibleLayerDesc(int index) const { 195 | return _visibleLayerDescs[index]; 196 | } 197 | 198 | /*! 199 | \brief Get hidden size 200 | */ 201 | cl_int2 getHiddenSize() const { 202 | return _hiddenSize; 203 | } 204 | 205 | /*! 206 | \brief Get hidden states 207 | */ 208 | const DoubleBuffer2D &getHiddenStates() const { 209 | return _hiddenStates; 210 | } 211 | 212 | /*! 213 | \brief Get hidden activations 214 | */ 215 | const DoubleBuffer2D &getHiddenActivations() const { 216 | return _hiddenActivationSummationTemp; 217 | } 218 | 219 | /*! 220 | \brief Get hidden biases 221 | */ 222 | const DoubleBuffer2D &getHiddenBiases() const { 223 | return _hiddenBiases; 224 | } 225 | }; 226 | } -------------------------------------------------------------------------------- /NeoRL/source/neo/AgentSwarm.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ComparisonSparseCoder.h" 4 | #include "Predictor.h" 5 | #include "Swarm.h" 6 | 7 | namespace neo { 8 | /*! 9 | \brief SDRRL swarm gradient agent 10 | */ 11 | class AgentSwarm { 12 | public: 13 | struct LayerDesc { 14 | /*! 15 | \brief Size of hidden layer in hierarchy 16 | */ 17 | cl_int2 _hiddenSize; 18 | 19 | /*! 20 | \brief Size of Q layer in hierarchy 21 | */ 22 | cl_int2 _qSize; 23 | 24 | //!@{ 25 | /*! 26 | \brief Radii 27 | */ 28 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 29 | cl_int _qRadiusHiddenFeedForwardAttention, _qRadiusHiddenRecurrentAttention, _qRadiusHiddenAction, _qRadius; 30 | cl_int _startRadiusHiddenFeedForwardAttention, _startRadiusHiddenRecurrentAttention, _startRadiusHiddenAction; 31 | //!@} 32 | 33 | //!@{ 34 | /*! 35 | \brief Sparse coder parameters 36 | */ 37 | cl_float _scWeightAlpha; 38 | cl_float _scWeightRecurrentAlpha; 39 | cl_float _scWeightLambda; 40 | cl_float _scActiveRatio; 41 | cl_float _scBoostAlpha; 42 | //!@} 43 | 44 | //!@{ 45 | /*! 46 | \brief Baseline parameters 47 | */ 48 | cl_float _baseLineDecay; 49 | cl_float _baseLineSensitivity; 50 | //!@} 51 | 52 | /*! 53 | \brief Predictor parameters 54 | */ 55 | cl_float _predWeightAlpha; 56 | 57 | //!@{ 58 | /*! 59 | \brief Swarm parameters 60 | */ 61 | cl_int _swarmAnnealingIterations; 62 | cl_float _swarmActionDeriveAlpha; 63 | cl_float _swarmQAlpha; 64 | cl_float _swarmQHiddenAlpha; 65 | cl_float _swarmPredAlpha; 66 | cl_float _swarmLambda; 67 | cl_float _swarmGamma; 68 | cl_float _swarmExpPert; 69 | cl_float _swarmExpBreak; 70 | //!@} 71 | 72 | /*! 73 | \brief Attention parameters 74 | */ 75 | cl_float _minAttention; 76 | 77 | /*! 78 | \brief Initialize defaults 79 | */ 80 | LayerDesc() 81 | : _hiddenSize({ 8, 8 }), _qSize({ 4, 4 }), 82 | _feedForwardRadius(4), _recurrentRadius(4), _lateralRadius(4), _feedBackRadius(4), _predictiveRadius(4), 83 | _qRadiusHiddenFeedForwardAttention(4), _qRadiusHiddenRecurrentAttention(4), _qRadiusHiddenAction(4), _qRadius(4), 84 | _startRadiusHiddenFeedForwardAttention(4), _startRadiusHiddenRecurrentAttention(4), _startRadiusHiddenAction(4), 85 | _scWeightAlpha(0.01f), _scWeightRecurrentAlpha(0.001f), _scWeightLambda(0.95f), 86 | _scActiveRatio(0.06f), _scBoostAlpha(0.01f), 87 | _baseLineDecay(0.01f), _baseLineSensitivity(0.01f), 88 | _predWeightAlpha(0.01f), 89 | _swarmAnnealingIterations(1), _swarmActionDeriveAlpha(0.08f), 90 | _swarmQAlpha(0.01f), _swarmQHiddenAlpha(0.1f), 91 | _swarmPredAlpha(0.01f), _swarmLambda(0.95f), _swarmGamma(0.99f), 92 | _swarmExpPert(0.1f), _swarmExpBreak(0.03f), 93 | _minAttention(0.05f) 94 | {} 95 | }; 96 | 97 | /*! 98 | \brief Layer (Hidden + Q) in hierarchy 99 | */ 100 | struct Layer { 101 | /*! 102 | \brief Sparse coder 103 | */ 104 | ComparisonSparseCoder _sc; 105 | 106 | /*! 107 | \brief Predictor 108 | */ 109 | Predictor _pred; 110 | 111 | /*! 112 | \brief Swarm 113 | */ 114 | Swarm _swarm; 115 | 116 | //!@{ 117 | /*! 118 | \brief Temporary buffers 119 | */ 120 | cl::Image2D _modulatedFeedForwardInput; 121 | cl::Image2D _modulatedRecurrentInput; 122 | cl::Image2D _inhibitedAction; 123 | //!@} 124 | 125 | /*! 126 | \brief Baselines 127 | */ 128 | DoubleBuffer2D _baseLines; 129 | 130 | /*! 131 | \brief Reward for sparse coder 132 | */ 133 | cl::Image2D _reward; 134 | 135 | /*! 136 | \brief Previous hidden states 137 | */ 138 | cl::Image2D _scHiddenStatesPrev; 139 | }; 140 | 141 | private: 142 | //!@{ 143 | /*! 144 | \brief Layers and layer descs 145 | */ 146 | std::vector _layers; 147 | std::vector _layerDescs; 148 | //!@} 149 | 150 | /*! 151 | \brief Action for last layer (all zeros, dummy buffer) 152 | */ 153 | cl::Image2D _lastLayerAction; 154 | 155 | //!@{ 156 | /*! 157 | \brief Kernels 158 | */ 159 | cl::Kernel _baseLineUpdateKernel; 160 | cl::Kernel _baseLineUpdateSumErrorKernel; 161 | cl::Kernel _inhibitKernel; 162 | cl::Kernel _modulateKernel; 163 | //!@} 164 | 165 | public: 166 | /*! 167 | \brief Create an agent with random initialization 168 | Requires the compute system, program with the NeoRL kernels, input/action sizes, layer descs, and initialization information 169 | */ 170 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 171 | cl_int2 inputSize, cl_int2 actionSize, cl_int firstLayerPredictorRadius, const std::vector &layerDescs, 172 | cl_float2 initWeightRange, 173 | std::mt19937 &rng); 174 | 175 | /*! 176 | \brief Simulation step of agent 177 | */ 178 | void simStep(sys::ComputeSystem &cs, float reward, const cl::Image2D &input, std::mt19937 &rng); 179 | 180 | /*! 181 | \brief Clear working memory 182 | */ 183 | void clearMemory(sys::ComputeSystem &cs); 184 | 185 | /*! 186 | \brief Number of layers in hierarchy 187 | */ 188 | size_t getNumLayers() const { 189 | return _layers.size(); 190 | } 191 | 192 | /*! 193 | \brief Get access to a layer 194 | */ 195 | const Layer &getLayer(int index) const { 196 | return _layers[index]; 197 | } 198 | 199 | /*! 200 | \brief Get access to a layer desc 201 | */ 202 | const LayerDesc &getLayerDescs(int index) const { 203 | return _layerDescs[index]; 204 | } 205 | 206 | /*! 207 | \brief Get predictor from first layer 208 | */ 209 | const Predictor &getFirstLayerPred() const { 210 | return _layers.front()._pred; 211 | } 212 | 213 | /*! 214 | \brief Get exploratory actions image 215 | */ 216 | const cl::Image2D &getExploratoryActions() const { 217 | return _layers.front()._swarm.getVisibleLayer(2)._actionsExploratory; 218 | } 219 | }; 220 | } -------------------------------------------------------------------------------- /NeoRL/source/TextPrediction.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_TEXT_PREDICTION 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | int main() { 21 | sf::RenderWindow window; 22 | 23 | sf::ContextSettings glContextSettings; 24 | glContextSettings.antialiasingLevel = 4; 25 | 26 | window.create(sf::VideoMode(800, 600), "Link", sf::Style::Default, glContextSettings); 27 | 28 | //window.setFramerateLimit(60); 29 | //window.setVerticalSyncEnabled(true); 30 | 31 | std::mt19937 generator(time(nullptr)); 32 | 33 | sys::ComputeSystem cs; 34 | 35 | cs.create(sys::ComputeSystem::_gpu); 36 | 37 | sys::ComputeProgram prog; 38 | 39 | prog.loadFromFile("resources/neoKernels2.cl", cs); 40 | 41 | // --------------------------- Create the Sparse Coder --------------------------- 42 | 43 | std::uniform_real_distribution dist01(0.0f, 1.0f); 44 | 45 | std::ifstream fromFile("corpus.txt"); 46 | 47 | fromFile.seekg(0, std::ios::end); 48 | size_t size = fromFile.tellg(); 49 | std::string test(size, ' '); 50 | fromFile.seekg(0); 51 | fromFile.read(&test[0], size); 52 | 53 | int minimum = 255; 54 | int maximum = 0; 55 | 56 | for (int i = 0; i < test.length(); i++) { 57 | minimum = std::min(static_cast(test[i]), minimum); 58 | maximum = std::max(static_cast(test[i]), maximum); 59 | } 60 | 61 | int numInputs = maximum - minimum + 1; 62 | 63 | int inputsRoot = std::ceil(std::sqrt(static_cast(numInputs))); 64 | 65 | std::vector layerDescs(1); 66 | 67 | layerDescs[0]._size = { 16, 16 }; 68 | //layerDescs[1]._size = { 8, 8 }; 69 | //layerDescs[2]._size = { 8, 8 }; 70 | 71 | neo::PredictiveHierarchy ph; 72 | 73 | ph.createRandom(cs, prog, { inputsRoot, inputsRoot }, layerDescs, { -0.01f, 0.01f }, generator); 74 | 75 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), inputsRoot, inputsRoot); 76 | 77 | std::vector input(inputsRoot * inputsRoot, 0.0f); 78 | std::vector pred(inputsRoot * inputsRoot, 0.0f); 79 | char predChar = 0; 80 | 81 | // ---------------------------- Game Loop ----------------------------- 82 | 83 | sf::View view = window.getDefaultView(); 84 | 85 | bool quit = false; 86 | 87 | sf::Clock clock; 88 | 89 | float dt = 0.017f; 90 | 91 | int current = 0; 92 | 93 | float averageError = 0.0f; 94 | 95 | sf::Font font; 96 | if (!font.loadFromFile("C:/Windows/Fonts/Arial.ttf")) 97 | return 1; 98 | 99 | sf::Text avgText; 100 | avgText.setColor(sf::Color::Red); 101 | avgText.setFont(font); 102 | avgText.setPosition(sf::Vector2f(100.0f, 100.0f)); 103 | 104 | bool modeGenerate = false; 105 | 106 | bool tildePressed = false; 107 | 108 | float noiseAmount = 0.05f; 109 | 110 | std::normal_distribution noiseDist(0.0f, 1.0f); 111 | 112 | do { 113 | clock.restart(); 114 | 115 | // ----------------------------- Input ----------------------------- 116 | 117 | sf::Event windowEvent; 118 | 119 | while (window.pollEvent(windowEvent)) 120 | { 121 | switch (windowEvent.type) 122 | { 123 | case sf::Event::Closed: 124 | quit = true; 125 | break; 126 | } 127 | } 128 | 129 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape)) 130 | quit = true; 131 | 132 | if (!tildePressed && sf::Keyboard::isKeyPressed(sf::Keyboard::Tilde)) { 133 | modeGenerate = !modeGenerate; 134 | } 135 | 136 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Up)) { 137 | noiseAmount += 0.001f; 138 | 139 | std::cout << noiseAmount << std::endl; 140 | } 141 | else if (sf::Keyboard::isKeyPressed(sf::Keyboard::Down)) { 142 | noiseAmount = std::max(0.0f, noiseAmount - 0.001f); 143 | 144 | std::cout << noiseAmount << std::endl; 145 | } 146 | 147 | tildePressed = sf::Keyboard::isKeyPressed(sf::Keyboard::Tilde); 148 | 149 | { 150 | window.clear(); 151 | 152 | window.draw(avgText); 153 | 154 | const float scale = 4.0f; 155 | 156 | window.display(); 157 | } 158 | 159 | if (modeGenerate) { 160 | for (int i = 0; i < inputsRoot * inputsRoot; i++) 161 | input[i] = noiseDist(generator) * noiseAmount; 162 | 163 | int index = predChar - minimum; 164 | 165 | input[index] = 1.0f + noiseDist(generator) * noiseAmount; 166 | } 167 | else { 168 | for (int i = 0; i < inputsRoot * inputsRoot; i++) 169 | input[i] = 0.0f; 170 | 171 | int index = test[current] - minimum; 172 | 173 | input[index] = 1.0f; 174 | } 175 | 176 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { static_cast(inputsRoot), static_cast(inputsRoot), 1 }, 0, 0, input.data()); 177 | 178 | ph.simStep(cs, inputImage, !modeGenerate); 179 | 180 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { static_cast(inputsRoot), static_cast(inputsRoot), 1 }, 0, 0, pred.data()); 181 | 182 | int predIndex = 0; 183 | 184 | for (int i = 1; i < numInputs; i++) 185 | if (pred[i] > pred[predIndex]) 186 | predIndex = i; 187 | 188 | predChar = predIndex + minimum; 189 | 190 | std::cout << predChar; 191 | 192 | current = (current + 1) % test.length(); 193 | 194 | float error = 1.0f; 195 | 196 | if (predChar == test[current]) 197 | error = 0.0f; 198 | 199 | //for (int i = 0; i < pred.size(); i++) 200 | // std::cout << pred[i] << " "; 201 | 202 | //std::cout << predChar << " " << test[current] << std::endl; 203 | 204 | /*std::vector data(64); 205 | 206 | cs.getQueue().enqueueReadImage(ph.getLayer(0)._sp.getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { 8, 8, 1 }, 0, 0, data.data()); 207 | 208 | for (int x = 0; x < 8; x++) { 209 | 210 | for (int y = 0; y < 8; y++) 211 | std::cout << data[x + y * 8] << " "; 212 | 213 | std::cout << std::endl; 214 | }*/ 215 | 216 | averageError = 0.99f * averageError + 0.01f * error; 217 | avgText.setString("Avg Err: " + std::to_string(averageError)); 218 | 219 | if (current == 0) 220 | std::cout << "\n"; 221 | } while (!quit); 222 | 223 | return 0; 224 | } 225 | 226 | #endif -------------------------------------------------------------------------------- /NeoRL/source/Balancer.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_BALANCER 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | #include 15 | #include 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | int main() { 22 | std::mt19937 generator(time(nullptr)); 23 | 24 | sys::ComputeSystem cs; 25 | 26 | cs.create(sys::ComputeSystem::_cpu); 27 | 28 | sys::ComputeProgram prog; 29 | 30 | prog.loadFromFile("resources/neoKernels.cl", cs); 31 | 32 | std::uniform_real_distribution dist01(0.0f, 1.0f); 33 | 34 | sf::RenderWindow window; 35 | 36 | window.create(sf::VideoMode(800, 800), "BIDInet", sf::Style::Default); 37 | 38 | window.setFramerateLimit(60); 39 | window.setVerticalSyncEnabled(true); 40 | 41 | int inWidth = 2; 42 | int inHeight = 2; 43 | 44 | int aWidth = 2; 45 | int aHeight = 2; 46 | 47 | std::vector layerDescs(1); 48 | 49 | layerDescs[0]._size = { 8, 8 }; 50 | //layerDescs[1]._size = { 8, 8 }; 51 | 52 | neo::AgentSwarm agent; 53 | 54 | agent.createRandom(cs, prog, { inWidth, inHeight }, { aWidth, aHeight }, 6, 8, 4, layerDescs, { -0.01f, 0.01f }, 0.0f, generator); 55 | 56 | // ---------------------------- Game Loop ----------------------------- 57 | 58 | std::vector layerTextures(layerDescs.size()); 59 | 60 | bool quit = false; 61 | 62 | sf::Clock clock; 63 | 64 | float dt = 0.017f; 65 | 66 | float averageReward = 0.0f; 67 | const float averageRewardDecay = 0.003f; 68 | 69 | int steps = 0; 70 | 71 | float action = 0.0f; 72 | float ballPosition = 0.5f; 73 | float ballVelocity = 0.0f; 74 | 75 | do { 76 | clock.restart(); 77 | 78 | // ----------------------------- Input ----------------------------- 79 | 80 | sf::Event windowEvent; 81 | 82 | while (window.pollEvent(windowEvent)) 83 | { 84 | switch (windowEvent.type) 85 | { 86 | case sf::Event::Closed: 87 | quit = true; 88 | break; 89 | } 90 | } 91 | 92 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape)) 93 | quit = true; 94 | 95 | ballVelocity += 0.001f * ballPosition + 0.03f * action; 96 | 97 | ballPosition += ballVelocity; 98 | 99 | if (ballPosition > 1.0f || ballPosition < -1.0f) 100 | ballVelocity = 0.0f; 101 | 102 | ballPosition = std::min(1.0f, std::max(-1.0f, ballPosition)); 103 | 104 | float reward = -ballPosition * ballPosition; 105 | 106 | averageReward = (1.0f - averageRewardDecay) * averageReward + averageRewardDecay * reward; 107 | 108 | agent.setState(0, ballPosition); 109 | agent.setState(1, ballPosition + 1.0f); 110 | agent.setState(2, 1.0f - ballPosition); 111 | agent.setState(3, 2.0f * ballPosition); 112 | 113 | agent.simStep(reward, cs, generator); 114 | 115 | float act = 0.0f; 116 | 117 | for (int i = 0; i < 4; i++) { 118 | act += agent.getAction(i); 119 | } 120 | 121 | action += 0.1f * (std::min(1.0f, std::max(-1.0f, act)) - action); 122 | 123 | //std::cout << averageReward << std::endl; 124 | 125 | if (!sf::Keyboard::isKeyPressed(sf::Keyboard::T)) { 126 | window.clear(); 127 | 128 | sf::ConvexShape fulcrum; 129 | 130 | fulcrum.setPointCount(3); 131 | 132 | fulcrum.setPoint(0, sf::Vector2f(-16.0f, 16.0f)); 133 | fulcrum.setPoint(1, sf::Vector2f(16.0f, 16.0f)); 134 | fulcrum.setPoint(2, sf::Vector2f(0.0f, 0.0f)); 135 | 136 | fulcrum.setPosition(window.getSize().x * 0.5f, window.getSize().y * 0.5f + 128.0f); 137 | 138 | window.draw(fulcrum); 139 | 140 | sf::RectangleShape plank; 141 | plank.setSize(sf::Vector2f(256.0f, 4.0f)); 142 | plank.setOrigin(sf::Vector2f(256.0f, 4.0f) * 0.5f); 143 | 144 | plank.setPosition(fulcrum.getPosition()); 145 | 146 | plank.setRotation(30.0f * action); 147 | 148 | window.draw(plank); 149 | 150 | sf::CircleShape ball; 151 | ball.setRadius(8.0f); 152 | ball.setOrigin(8.0f, 8.0f); 153 | ball.setPosition(fulcrum.getPosition() + sf::Vector2f(std::cos(plank.getRotation() * 3.141596f / 180.0f) * ballPosition * plank.getSize().x * 0.5f, -ball.getRadius() + std::sin(plank.getRotation() * 3.141596f / 180.0f) * ballPosition * plank.getSize().x * 0.5f)); 154 | 155 | window.draw(ball); 156 | 157 | sf::Image predImg; 158 | 159 | predImg.create(inWidth, inHeight); 160 | 161 | for (int x = 0; x < inWidth; x++) 162 | for (int y = 0; y < inHeight; y++) { 163 | sf::Color c = sf::Color::White; 164 | 165 | c.r = c.g = c.b = 255.0f * std::min(1.0f, std::max(0.0f, agent.getPrediction(x, y))); 166 | 167 | predImg.setPixel(x, y, c); 168 | } 169 | 170 | sf::Texture t; 171 | t.loadFromImage(predImg); 172 | 173 | sf::Sprite s; 174 | 175 | s.setTexture(t); 176 | 177 | s.setScale(4.0f, 4.0f); 178 | 179 | s.setPosition(4.0f * 16.0f, 0.0f); 180 | 181 | window.draw(s); 182 | 183 | float xOffset = 0.0f; 184 | float scale = 4.0f; 185 | 186 | for (int l = 0; l < layerDescs.size(); l++) { 187 | std::vector data(layerDescs[l]._size.x * layerDescs[l]._size.y); 188 | 189 | cs.getQueue().enqueueReadImage(agent.getLayer(l)._scHiddenStatesPrev, CL_TRUE, { 0, 0, 0 }, { static_cast(layerDescs[l]._size.x), static_cast(layerDescs[l]._size.y), 1 }, 0, 0, data.data()); 190 | 191 | sf::Image img; 192 | 193 | img.create(layerDescs[l]._size.x, layerDescs[l]._size.y); 194 | 195 | for (int x = 0; x < img.getSize().x; x++) 196 | for (int y = 0; y < img.getSize().y; y++) { 197 | sf::Color c = sf::Color::White; 198 | 199 | c.r = c.b = c.g = 255.0f * data[x + y * img.getSize().x]; 200 | 201 | img.setPixel(x, y, c); 202 | } 203 | 204 | layerTextures[l].loadFromImage(img); 205 | 206 | sf::Sprite s; 207 | 208 | s.setTexture(layerTextures[l]); 209 | 210 | s.setPosition(xOffset, window.getSize().y - img.getSize().y * scale); 211 | 212 | s.setScale(scale, scale); 213 | 214 | window.draw(s); 215 | 216 | xOffset += img.getSize().x * scale; 217 | } 218 | 219 | window.display(); 220 | } 221 | 222 | if (steps % 100 == 0) 223 | std::cout << "Steps: " << steps << " Average Reward: " << averageReward << std::endl; 224 | 225 | //dt = clock.getElapsedTime().asSeconds(); 226 | 227 | steps++; 228 | 229 | } while (!quit); 230 | 231 | return 0; 232 | } 233 | 234 | #endif -------------------------------------------------------------------------------- /NeoRL/source/neo/AgentER.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ComparisonSparseCoder.h" 4 | #include "Predictor.h" 5 | #include "ImageWhitener.h" 6 | 7 | #include 8 | 9 | namespace neo { 10 | /*! 11 | \brief Predictive hierarchy (no RL) 12 | */ 13 | class AgentER { 14 | public: 15 | /*! 16 | \brief Layer desc 17 | */ 18 | struct LayerDesc { 19 | /*! 20 | \brief Size of layer 21 | */ 22 | cl_int2 _size; 23 | 24 | /*! 25 | \brief Radii 26 | */ 27 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 28 | 29 | //!@{ 30 | /*! 31 | \brief Sparse coder parameters 32 | */ 33 | cl_float _scWeightAlpha; 34 | cl_float _scWeightRecurrentAlpha; 35 | cl_float _scActiveRatio; 36 | cl_float _scBoostAlpha; 37 | //!@} 38 | 39 | //!@{ 40 | /*! 41 | \brief Predictor parameters 42 | */ 43 | cl_float _predWeightAlpha; 44 | //!@} 45 | 46 | /*! 47 | \brief Initialize defaults 48 | */ 49 | LayerDesc() 50 | : _size({ 8, 8 }), 51 | _feedForwardRadius(5), _recurrentRadius(0), _lateralRadius(5), _feedBackRadius(6), _predictiveRadius(6), 52 | _scWeightAlpha(0.001f), _scWeightRecurrentAlpha(0.001f), 53 | _scActiveRatio(0.05f), _scBoostAlpha(0.01f), 54 | _predWeightAlpha(0.01f) 55 | {} 56 | }; 57 | 58 | /*! 59 | \brief Layer 60 | */ 61 | struct Layer { 62 | //!@{ 63 | /*! 64 | \brief Sparse coder and predictor 65 | */ 66 | ComparisonSparseCoder _sc; 67 | Predictor _pred; 68 | //!@} 69 | 70 | //!@{ 71 | /*! 72 | \brief For prediction reward determination 73 | */ 74 | cl::Image2D _predReward; 75 | cl::Image2D _propagatedPredReward; 76 | //!@} 77 | 78 | //!@{ 79 | /*! 80 | \brief For replay 81 | */ 82 | DoubleBuffer2D _scStatesTemp; 83 | DoubleBuffer2D _predStatesTemp; 84 | //!@} 85 | }; 86 | 87 | /*! 88 | \brief Replay buffer frame 89 | */ 90 | struct ReplayFrame { 91 | std::vector> _layerStateBitIndices; 92 | std::vector> _layerPredBitIndices; 93 | 94 | std::vector _prevExploratoryAction; 95 | std::vector _prevBestAction; 96 | 97 | float _q; 98 | float _originalQ; 99 | }; 100 | 101 | private: 102 | /*! 103 | \brief Store input size 104 | */ 105 | cl_int2 _inputSize; 106 | 107 | /*! 108 | \brief Store action size 109 | */ 110 | cl_int2 _actionSize; 111 | 112 | /*! 113 | \brief Store Q size 114 | */ 115 | cl_int2 _qSize; 116 | 117 | /*! 118 | \brief Q Input layer 119 | */ 120 | cl::Image2D _qInput; 121 | 122 | /*! 123 | \brief Q Target layer 124 | */ 125 | cl::Image2D _qTarget; 126 | 127 | /*! 128 | \brief Action target layer 129 | */ 130 | cl::Image2D _actionTarget; 131 | 132 | /*! 133 | \brief Transformation on Q values 134 | */ 135 | cl::Image2D _qTransform; 136 | 137 | /*! 138 | \brief Additional predictor for Q 139 | */ 140 | Predictor _qPred; 141 | 142 | //!@{ 143 | /*! 144 | \brief Layers and descs 145 | */ 146 | std::vector _layers; 147 | std::vector _layerDescs; 148 | //!@} 149 | 150 | //!@{ 151 | /*! 152 | \brief Kernels for hierarchy 153 | */ 154 | cl::Kernel _predictionRewardKernel; 155 | cl::Kernel _predictionRewardPropagationKernel; 156 | cl::Kernel _setQKernel; 157 | //!@} 158 | 159 | /*! 160 | \brief Input whiteners 161 | */ 162 | ImageWhitener _inputWhitener; 163 | ImageWhitener _actionWhitener; 164 | ImageWhitener _qWhitener; 165 | 166 | //!@{ 167 | /*! 168 | \brief Remember previous Q 169 | */ 170 | float _prevValue; 171 | float _prevQ; 172 | float _prevTDError; 173 | //!@} 174 | 175 | /*! 176 | \brief Experience replay buffer 177 | */ 178 | std::list _frames; 179 | 180 | public: 181 | //!@{ 182 | /*! 183 | \brief Whitening parameters 184 | */ 185 | cl_int _whiteningKernelRadius; 186 | cl_float _whiteningIntensity; 187 | //!@} 188 | 189 | //!@{ 190 | /*! 191 | \brief RL parameters 192 | */ 193 | cl_float _qGamma; 194 | cl_float _qAlpha; 195 | cl_float _qWeightAlpha; 196 | 197 | int _maxReplayFrames; 198 | int _replayIterations; 199 | //!@} 200 | 201 | /*! 202 | \brief Initialize defaults 203 | */ 204 | AgentER() 205 | : _whiteningKernelRadius(1), 206 | _whiteningIntensity(1024.0f), 207 | _qGamma(0.98f), _qAlpha(0.5f), 208 | _qWeightAlpha(0.01f), 209 | _maxReplayFrames(600), _replayIterations(10), 210 | _prevValue(0.0f), _prevQ(0.0f), _prevTDError(0.0f) 211 | {} 212 | 213 | /*! 214 | \brief Create a comparison sparse coder with random initialization 215 | Requires the compute system, program with the NeoRL kernels, and initialization information. 216 | */ 217 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 218 | cl_int2 inputSize, cl_int2 actionSize, cl_int2 qSize, 219 | const std::vector &layerDescs, 220 | cl_float2 initWeightRange, 221 | std::mt19937 &rng); 222 | 223 | /*! 224 | \brief Simulation step of hierarchy 225 | */ 226 | void simStep(sys::ComputeSystem &cs, const cl::Image2D &input, const cl::Image2D &actionTaken, float reward, std::mt19937 &rng, bool learn = true, bool whiten = true); 227 | 228 | /*! 229 | \brief Clear working memory 230 | */ 231 | void clearMemory(sys::ComputeSystem &cs); 232 | 233 | /*! 234 | \brief Write to stream 235 | */ 236 | void writeToStream(sys::ComputeSystem &cs, std::ostream &os) const; 237 | 238 | /*! 239 | \brief Read from stream 240 | */ 241 | void readFromStream(sys::ComputeSystem &cs, sys::ComputeProgram &program, std::istream &is); 242 | 243 | /*! 244 | \brief Get number of layers 245 | */ 246 | size_t getNumLayers() const { 247 | return _layers.size(); 248 | } 249 | 250 | /*! 251 | \brief Get access to a layer 252 | */ 253 | const Layer &getLayer(int index) const { 254 | return _layers[index]; 255 | } 256 | 257 | /*! 258 | \brief Get access to a layer desc 259 | */ 260 | const LayerDesc &getLayerDescs(int index) const { 261 | return _layerDescs[index]; 262 | } 263 | 264 | /*! 265 | \brief Get the prediction 266 | */ 267 | const cl::Image2D &getAction() const { 268 | return _layers.front()._pred.getHiddenStates()[_back]; 269 | } 270 | 271 | /*! 272 | \brief Get input whitener 273 | */ 274 | const ImageWhitener &getInputWhitener() const { 275 | return _inputWhitener; 276 | } 277 | 278 | /*! 279 | \brief Get aciton whitener 280 | */ 281 | const ImageWhitener &getActionWhitener() const { 282 | return _actionWhitener; 283 | } 284 | 285 | /*! 286 | \brief Get Q whitener 287 | */ 288 | const ImageWhitener &getQWhitener() const { 289 | return _qWhitener; 290 | } 291 | }; 292 | } -------------------------------------------------------------------------------- /NeoRL/source/TextTraining.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_TEXT_TRAINING 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | 10 | #include 11 | #include 12 | #include 13 | #include 14 | 15 | #include 16 | #include 17 | 18 | #include 19 | 20 | std::ifstream::pos_type numCharsInFile(const std::string &fileName) { 21 | std::ifstream in(fileName, std::ifstream::ate); 22 | return in.tellg(); 23 | } 24 | 25 | int main() { 26 | sf::RenderWindow window; 27 | 28 | sf::ContextSettings glContextSettings; 29 | glContextSettings.antialiasingLevel = 4; 30 | 31 | window.create(sf::VideoMode(800, 600), "Link", sf::Style::Default, glContextSettings); 32 | 33 | //window.setFramerateLimit(60); 34 | //window.setVerticalSyncEnabled(true); 35 | 36 | std::mt19937 generator(time(nullptr)); 37 | 38 | sys::ComputeSystem cs; 39 | 40 | cs.create(sys::ComputeSystem::_gpu); 41 | 42 | sys::ComputeProgram prog; 43 | 44 | prog.loadFromFile("resources/neoKernels.cl", cs); 45 | 46 | // --------------------------- Create the Sparse Coder --------------------------- 47 | 48 | unsigned long fileSize = numCharsInFile("corpus.txt"); 49 | 50 | std::cout << "File size: " << fileSize << std::endl; 51 | 52 | unsigned long trainSize = std::round(0.97 * fileSize); 53 | unsigned long testSize = fileSize - trainSize; 54 | 55 | std::cout << "Train size and test size: " << trainSize << " " << testSize << std::endl; 56 | 57 | std::uniform_real_distribution dist01(0.0f, 1.0f); 58 | 59 | std::ifstream fromFile("corpus.txt"); 60 | 61 | int minimum = 0; 62 | int maximum = 255; 63 | 64 | int numInputs = maximum - minimum + 1; 65 | 66 | int inputsRoot = std::ceil(std::sqrt(static_cast(numInputs))); 67 | 68 | std::vector layerDescs(3); 69 | 70 | layerDescs[0]._size = { 32, 32 }; 71 | layerDescs[0]._predictiveRadius = 12; 72 | layerDescs[0]._feedBackRadius = 12; 73 | layerDescs[0]._predWeightAlpha = 0.01f; 74 | 75 | layerDescs[1]._size = { 32, 32 }; 76 | 77 | layerDescs[2]._size = { 32, 32 }; 78 | 79 | neo::PredictiveHierarchy ph; 80 | 81 | ph.createRandom(cs, prog, { inputsRoot, inputsRoot }, layerDescs, { -0.01f, 0.01f }, 0.0f, generator); 82 | 83 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), inputsRoot, inputsRoot); 84 | 85 | std::vector input(inputsRoot * inputsRoot, 0.0f); 86 | std::vector pred(inputsRoot * inputsRoot, 0.0f); 87 | char predChar = 0; 88 | 89 | // ---------------------------- Game Loop ----------------------------- 90 | 91 | sf::View view = window.getDefaultView(); 92 | 93 | bool quit = false; 94 | 95 | sf::Clock clock; 96 | 97 | float dt = 0.017f; 98 | 99 | sf::Font font; 100 | if (!font.loadFromFile("C:/Windows/Fonts/Arial.ttf")) 101 | return 1; 102 | 103 | sf::Text avgText; 104 | avgText.setColor(sf::Color::Red); 105 | avgText.setFont(font); 106 | avgText.setPosition(sf::Vector2f(100.0f, 100.0f)); 107 | 108 | bool modeTest = false; 109 | 110 | std::normal_distribution noiseDist(0.0f, 1.0f); 111 | 112 | unsigned long charPosition = 0; 113 | 114 | double logLikelihoodSum = 0.0; 115 | double testSamplesSummed = 0.0; 116 | 117 | double errorSum = 0.0; 118 | 119 | float runningAverage = 1.0f; 120 | float runningAverageDecay = 0.005f; 121 | 122 | do { 123 | clock.restart(); 124 | 125 | // ----------------------------- Input ----------------------------- 126 | 127 | sf::Event windowEvent; 128 | 129 | while (window.pollEvent(windowEvent)) 130 | { 131 | switch (windowEvent.type) 132 | { 133 | case sf::Event::Closed: 134 | quit = true; 135 | break; 136 | } 137 | } 138 | 139 | if (window.hasFocus() && sf::Keyboard::isKeyPressed(sf::Keyboard::Escape)) 140 | quit = true; 141 | 142 | if (!modeTest && window.hasFocus() && sf::Keyboard::isKeyPressed(sf::Keyboard::Tilde)) { 143 | modeTest = true; 144 | 145 | fromFile.seekg(trainSize); 146 | 147 | std::cout << "Testing..." << std::endl; 148 | } 149 | 150 | // Read character 151 | char c; 152 | 153 | if (modeTest) { 154 | c = fromFile.get(); 155 | 156 | charPosition++; 157 | 158 | if (fromFile.eof()) { 159 | //charPosition = trainSize; 160 | //fromFile.seekg(trainSize); 161 | 162 | //std::cout << "Final Log Likelihood: " << std::to_string(logLikelihoodSum / testSamplesSummed) << std::endl; 163 | std::cout << "Final error: " << errorSum / testSamplesSummed << std::endl; 164 | 165 | return 0; 166 | } 167 | } 168 | else { 169 | c = fromFile.get(); 170 | 171 | charPosition++; 172 | 173 | if (charPosition >= trainSize) { 174 | charPosition = 0; 175 | fromFile.seekg(std::ios::beg); 176 | } 177 | } 178 | 179 | if (charPosition % 100 == 0) { 180 | window.clear(); 181 | 182 | window.draw(avgText); 183 | 184 | const float scale = 4.0f; 185 | 186 | window.display(); 187 | } 188 | 189 | if (!modeTest && charPosition % 50000 == 49999) { 190 | std::ofstream saveFile("neo_save1.neo"); 191 | 192 | ph.writeToStream(cs, saveFile); 193 | } 194 | 195 | for (int i = 0; i < inputsRoot * inputsRoot; i++) 196 | input[i] = 0.0f; 197 | 198 | int index = c - minimum; 199 | 200 | input[index] = 1.0f; 201 | 202 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { static_cast(inputsRoot), static_cast(inputsRoot), 1 }, 0, 0, input.data()); 203 | 204 | ph.simStep(cs, inputImage, !modeTest); 205 | 206 | cs.getQueue().enqueueReadImage(ph.getFirstLayerPred().getHiddenStates()[neo::_back], CL_TRUE, { 0, 0, 0 }, { static_cast(inputsRoot), static_cast(inputsRoot), 1 }, 0, 0, pred.data()); 207 | 208 | int predIndex = 0; 209 | 210 | for (int i = 1; i < numInputs; i++) 211 | if (pred[i] > pred[predIndex]) 212 | predIndex = i; 213 | 214 | predChar = predIndex + minimum; 215 | 216 | char nextChar = fromFile.peek(); 217 | 218 | if (!modeTest) 219 | std::cout << predChar; 220 | 221 | /*float minPred = 999999.0f; 222 | 223 | for (int i = 0; i < numInputs; i++) 224 | minPred = std::min(pred[i], minPred); 225 | 226 | // Normalize 227 | float total = 0.0f; 228 | 229 | for (int i = 0; i < numInputs; i++) { 230 | total += std::min(1.0f, std::max(0.0f, pred[i])); 231 | } 232 | 233 | float logLikelihood = std::log(std::min(1.0f, std::max(0.0f, pred[nextChar - minimum])) / total); 234 | 235 | avgText.setString("Log likelihood: " + std::to_string(logLikelihood) + " (total: " + std::to_string(total) + ")"); 236 | */ 237 | 238 | float error = nextChar == predChar ? 0.0f : 1.0f; 239 | 240 | runningAverage = (1.0f - runningAverageDecay) * runningAverage + runningAverageDecay * error; 241 | 242 | avgText.setString("Error Average: " + std::to_string(runningAverage)); 243 | 244 | if (modeTest) { 245 | //logLikelihoodSum += logLikelihood; 246 | errorSum += error; 247 | 248 | testSamplesSummed++; 249 | } 250 | } while (!quit); 251 | 252 | return 0; 253 | } 254 | 255 | #endif -------------------------------------------------------------------------------- /NeoRL/source/neo/AgentHA.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include "ComparisonSparseCoder.h" 4 | #include "Predictor.h" 5 | #include "PredictorSwarm.h" 6 | #include "ImageWhitener.h" 7 | 8 | #define USE_DETERMINISTIC_POLICY_GRADIENT 9 | 10 | namespace neo { 11 | /*! 12 | \brief Predictive hierarchy (no RL) 13 | */ 14 | class AgentHA { 15 | public: 16 | /*! 17 | \brief Layer desc 18 | */ 19 | struct LayerDesc { 20 | /*! 21 | \brief Size of layer 22 | */ 23 | cl_int2 _size; 24 | 25 | /*! 26 | \brief Radii 27 | */ 28 | cl_int _feedForwardRadius, _recurrentRadius, _lateralRadius, _feedBackRadius, _predictiveRadius; 29 | 30 | //!@{ 31 | /*! 32 | \brief Sparse coder parameters 33 | */ 34 | cl_float _scWeightAlpha; 35 | cl_float _scWeightRecurrentAlpha; 36 | cl_float _scWeightLambda; 37 | cl_float _scActiveRatio; 38 | cl_float _scBoostAlpha; 39 | //!@} 40 | 41 | //!@{ 42 | /*! 43 | \brief Predictor parameters 44 | */ 45 | cl_float _predWeightAlpha; 46 | cl_float _predWeightLambda; 47 | //!@} 48 | 49 | //!@{ 50 | /*! 51 | \brief RL 52 | */ 53 | cl_float _qAlpha; 54 | cl_float _qBiasAlpha; 55 | cl_float _qLambda; 56 | cl_int _qRadius; 57 | cl_float _qReluLeak; 58 | //!@} 59 | 60 | /*! 61 | \brief Prediction reward parameters 62 | */ 63 | cl_float _predRewardBaselineDecay; 64 | 65 | /*! 66 | \brief Initialize defaults 67 | */ 68 | LayerDesc() 69 | : _size({ 8, 8 }), 70 | _feedForwardRadius(5), _recurrentRadius(0), _lateralRadius(5), _feedBackRadius(6), _predictiveRadius(6), 71 | _scWeightAlpha(0.01f), _scWeightRecurrentAlpha(0.01f), _scWeightLambda(0.96f), 72 | _scActiveRatio(0.04f), _scBoostAlpha(0.01f), 73 | _predWeightAlpha(0.01f), _predWeightLambda(0.96f), 74 | _qAlpha(0.01f), _qBiasAlpha(0.005f), _qLambda(0.96f), _qRadius(6), _qReluLeak(0.01f), 75 | _predRewardBaselineDecay(0.01f) 76 | {} 77 | }; 78 | 79 | /*! 80 | \brief Layer 81 | */ 82 | struct Layer { 83 | //!@{ 84 | /*! 85 | \brief Sparse coder and predictor 86 | */ 87 | ComparisonSparseCoder _sc; 88 | Predictor _pred; 89 | //!@} 90 | 91 | //!@{ 92 | /*! 93 | \brief Q Hierarchy data 94 | */ 95 | DoubleBuffer3D _qWeights; 96 | DoubleBuffer2D _qBiases; 97 | DoubleBuffer2D _qStates; 98 | cl::Image2D _qErrors; 99 | //!@} 100 | 101 | //!@{ 102 | /*! 103 | \brief For prediction reward determination 104 | */ 105 | DoubleBuffer2D _predRewardBaselines; 106 | cl::Image2D _predReward; 107 | cl::Image2D _propagatedPredReward; 108 | //!@} 109 | }; 110 | 111 | private: 112 | /*! 113 | \brief Store input size 114 | */ 115 | cl_int2 _inputSize; 116 | 117 | /*! 118 | \brief Store action size 119 | */ 120 | cl_int2 _actionSize; 121 | 122 | //!@{ 123 | /*! 124 | \brief Layers and descs 125 | */ 126 | std::vector _layers; 127 | std::vector _layerDescs; 128 | //!@} 129 | 130 | //!@{ 131 | /*! 132 | \brief Kernels for hierarchy 133 | */ 134 | cl::Kernel _predictionRewardKernel; 135 | cl::Kernel _predictionRewardPropagationKernel; 136 | 137 | cl::Kernel _qForwardKernel; 138 | cl::Kernel _qLastForwardKernel; 139 | cl::Kernel _qBackwardKernel; 140 | cl::Kernel _qLastBackwardKernel; 141 | cl::Kernel _qFirstBackwardKernel; 142 | cl::Kernel _qWeightUpdateKernel; 143 | cl::Kernel _qLastWeightUpdateKernel; 144 | cl::Kernel _qActionUpdateKernel; 145 | 146 | cl::Kernel _explorationKernel; 147 | //!@} 148 | 149 | //!@{ 150 | /*! 151 | \brief Q Hierarchy data 152 | */ 153 | DoubleBuffer3D _qLastWeights; 154 | DoubleBuffer2D _qLastBiases; 155 | DoubleBuffer2D _qLastStates; 156 | 157 | cl::Image2D _qFirstErrors; 158 | 159 | cl_float _prevValue; 160 | //!@} 161 | 162 | //!@{ 163 | /*! 164 | \brief Action buffers 165 | */ 166 | cl::Image2D _action; 167 | DoubleBuffer2D _actionExploratory; 168 | //!@} 169 | 170 | /*! 171 | \brief Input whiteners 172 | */ 173 | ImageWhitener _inputWhitener; 174 | ImageWhitener _actionWhitener; 175 | 176 | public: 177 | //!@{ 178 | /*! 179 | \brief Last RL 180 | */ 181 | cl_int2 _qLastSize; 182 | cl_float _qGamma; 183 | cl_float _qLastAlpha; 184 | cl_float _qLastBiasAlpha; 185 | cl_float _qLastLambda; 186 | cl_int _qLastRadius; 187 | //!@} 188 | 189 | //!@{ 190 | /*! 191 | \brief General RL parameters 192 | */ 193 | cl_float _actionImprovementAlpha; 194 | cl_int _actionImprovementIterations; 195 | 196 | cl_float _expPert; 197 | cl_float _expBreak; 198 | //!@} 199 | 200 | //!@{ 201 | /*! 202 | \brief Whitening parameters 203 | */ 204 | cl_int _whiteningKernelRadius; 205 | cl_float _whiteningIntensity; 206 | //!@} 207 | 208 | /*! 209 | \brief Initialize defaults 210 | */ 211 | AgentHA() 212 | : _prevValue(0.0f), 213 | _qLastSize({ 8, 8 }), _qGamma(0.98f), 214 | _qLastAlpha(0.01f), _qLastBiasAlpha(0.005f), _qLastLambda(0.96f), _qLastRadius(6), 215 | _actionImprovementAlpha(0.05f), _actionImprovementIterations(4), _expPert(0.05f), _expBreak(0.01f), 216 | _whiteningKernelRadius(3), 217 | _whiteningIntensity(1024.0f) 218 | {} 219 | 220 | /*! 221 | \brief Create a comparison sparse coder with random initialization 222 | Requires the compute system, program with the NeoRL kernels, and initialization information. 223 | */ 224 | void createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 225 | cl_int2 inputSize, cl_int2 actionSize, const std::vector &layerDescs, 226 | cl_float2 initWeightRange, 227 | std::mt19937 &rng); 228 | 229 | /*! 230 | \brief Simulation step of hierarchy 231 | */ 232 | void simStep(sys::ComputeSystem &cs, float reward, const cl::Image2D &input, std::mt19937 &rng, bool learn = true); 233 | 234 | /*! 235 | \brief Clear working memory 236 | */ 237 | void clearMemory(sys::ComputeSystem &cs); 238 | 239 | /*! 240 | \brief Write to stream 241 | */ 242 | void writeToStream(sys::ComputeSystem &cs, std::ostream &os) const; 243 | 244 | /*! 245 | \brief Read from stream 246 | */ 247 | void readFromStream(sys::ComputeSystem &cs, sys::ComputeProgram &program, std::istream &is); 248 | 249 | /*! 250 | \brief Get number of layers 251 | */ 252 | size_t getNumLayers() const { 253 | return _layers.size(); 254 | } 255 | 256 | /*! 257 | \brief Get access to a layer 258 | */ 259 | const Layer &getLayer(int index) const { 260 | return _layers[index]; 261 | } 262 | 263 | /*! 264 | \brief Get access to a layer desc 265 | */ 266 | const LayerDesc &getLayerDescs(int index) const { 267 | return _layerDescs[index]; 268 | } 269 | 270 | /*! 271 | \brief Get exploratory action 272 | */ 273 | const cl::Image2D &getExploratoryAction() const { 274 | return _actionExploratory[_back]; 275 | } 276 | 277 | /*! 278 | \brief Get input whitener 279 | */ 280 | const ImageWhitener &getInputWhitener() const { 281 | return _inputWhitener; 282 | } 283 | 284 | /*! 285 | \brief Get action whitener 286 | */ 287 | const ImageWhitener &getActionWhitener() const { 288 | return _actionWhitener; 289 | } 290 | }; 291 | } -------------------------------------------------------------------------------- /NeoRL/source/deep/SDRRL.cpp: -------------------------------------------------------------------------------- 1 | #include "SDRRL.h" 2 | 3 | #include 4 | 5 | #include 6 | 7 | using namespace deep; 8 | 9 | void SDRRL::createRandom(int numStates, int numActions, int numCells, float initMinWeight, float initMaxWeight, float initMinInhibition, float initMaxInhibition, float initThreshold, std::mt19937 &generator) { 10 | std::uniform_real_distribution weightDist(initMinWeight, initMaxWeight); 11 | std::uniform_real_distribution inhibitionDist(initMinInhibition, initMaxInhibition); 12 | 13 | _numStates = numStates; 14 | 15 | _inputs.assign(numStates, 0.0f); 16 | _reconstructionError.assign(_inputs.size(), 0.0f); 17 | 18 | _cells.resize(numCells); 19 | 20 | _actions.resize(numActions * 2); 21 | 22 | _qConnections.resize(numCells); 23 | 24 | for (int i = 0; i < numCells; i++) { 25 | _cells[i]._feedForwardConnections.resize(_inputs.size()); 26 | 27 | _cells[i]._lateralConnections.resize(numCells); 28 | 29 | _cells[i]._threshold = initThreshold; 30 | 31 | for (int j = 0; j < _inputs.size(); j++) 32 | _cells[i]._feedForwardConnections[j]._weight = weightDist(generator); 33 | 34 | for (int j = 0; j < numCells; j++) 35 | _cells[i]._lateralConnections[j]._weight = inhibitionDist(generator); 36 | 37 | _cells[i]._actionConnections.resize(_actions.size()); 38 | 39 | for (int j = 0; j < _actions.size(); j++) 40 | _cells[i]._actionConnections[j]._weight = weightDist(generator); 41 | 42 | _qConnections[i]._weight = weightDist(generator); 43 | } 44 | } 45 | 46 | void SDRRL::simStep(float reward, float sparsity, float gamma, 47 | float gateFeedForwardAlpha, float gateLateralAlpha, float gateThresholdAlpha, 48 | float qAlpha, float actionAlpha, int actionDeriveIterations, float actionDeriveAlpha, float gammaLambda, 49 | float explorationStdDev, float explorationBreak, 50 | float averageSurpiseDecay, float surpriseLearnFactor, std::mt19937 &generator) 51 | { 52 | std::uniform_real_distribution dist01(0.0f, 1.0f); 53 | std::normal_distribution pertDist(0.0f, explorationStdDev); 54 | 55 | int numHalfActions = _actions.size() / 2; 56 | 57 | // Clear activations and states 58 | for (int i = 0; i < _cells.size(); i++) { 59 | _cells[i]._activation = 0.0f; 60 | _cells[i]._state = 0.0f; 61 | } 62 | 63 | // Activate 64 | for (int i = 0; i < _cells.size(); i++) { 65 | float activation = -_cells[i]._threshold; 66 | 67 | for (int j = 0; j < _inputs.size(); j++) 68 | activation += _cells[i]._feedForwardConnections[j]._weight * _reconstructionError[j]; 69 | 70 | _cells[i]._activation = activation; 71 | } 72 | 73 | for (int i = 0; i < _cells.size(); i++) { 74 | float inhibition = 0.0f; 75 | 76 | for (int j = 0; j < _cells.size(); j++) 77 | inhibition += std::abs(_cells[j]._activation) < std::abs(_cells[i]._activation) ? 1.0f : 0.0f; 78 | 79 | float state = inhibition < sparsity * _cells.size() ? 1.0f : 0.0f; 80 | } 81 | 82 | // Reconstruct 83 | for (int i = 0; i < _inputs.size(); i++) { 84 | float recon = 0.0f; 85 | 86 | for (int j = 0; j < _cells.size(); j++) 87 | recon += _cells[j]._state * _cells[j]._feedForwardConnections[i]._weight; 88 | 89 | _reconstructionError[i] = _inputs[i] - recon; 90 | } 91 | 92 | for (int i = 0; i < numHalfActions; i++) 93 | _actions[i + numHalfActions]._state = 1.0f - _actions[i]._state; 94 | 95 | // Action sampling 96 | for (int iter = 0; iter < actionDeriveIterations; iter++) { 97 | float q = 0.0f; 98 | 99 | // Forwards 100 | for (int k = 0; k < _cells.size(); k++) { 101 | if (_cells[k]._state > 0.0f) { 102 | float sum = 0.0f;// _cells[k]._actionBias._weight; 103 | 104 | for (int vi = 0; vi < _actions.size(); vi++) 105 | sum += _cells[k]._actionConnections[vi]._weight * _actions[vi]._state; 106 | 107 | _cells[k]._actionState = sigmoid(sum) * _cells[k]._state; 108 | 109 | q += _qConnections[k]._weight * _cells[k]._actionState; 110 | } 111 | else 112 | _cells[k]._actionState = 0.0f; 113 | } 114 | 115 | // Action improvement 116 | for (int k = 0; k < _cells.size(); k++) 117 | _cells[k]._actionError = _qConnections[k]._weight * _cells[k]._actionState * (1.0f - _cells[k]._actionState); 118 | 119 | for (int i = 0; i < _actions.size(); i++) { 120 | float sum = 0.0f; 121 | 122 | for (int k = 0; k < _cells.size(); k++) 123 | sum += _cells[k]._actionConnections[i]._weight * _cells[k]._actionError; 124 | 125 | _actions[i]._error = sum; 126 | } 127 | 128 | for (int i = 0; i < numHalfActions; i++) 129 | // Find action delta 130 | _actions[i]._state = std::min(1.0f, std::max(0.0f, _actions[i]._state + actionDeriveAlpha * ((_actions[i]._error - _actions[i + numHalfActions]._error) > 0.0f ? 1.0f : -1.0f))); 131 | 132 | for (int i = 0; i < numHalfActions; i++) 133 | _actions[i + numHalfActions]._state = 1.0f - _actions[i]._state; 134 | } 135 | 136 | // Exploration 137 | for (int i = 0; i < _actions.size(); i++) { 138 | if (dist01(generator) < explorationBreak) 139 | _actions[i]._exploratoryState = dist01(generator); 140 | else 141 | _actions[i]._exploratoryState = std::min(1.0f, std::max(0.0f, _actions[i]._state + pertDist(generator))); 142 | } 143 | 144 | for (int i = 0; i < numHalfActions; i++) 145 | _actions[i + numHalfActions]._exploratoryState = 1.0f - _actions[i]._exploratoryState; 146 | 147 | // Forwards 148 | float q = 0.0f; 149 | 150 | for (int k = 0; k < _cells.size(); k++) { 151 | if (_cells[k]._state > 0.0f) { 152 | float sum = 0.0f;// _cells[k]._actionBias._weight; 153 | 154 | for (int vi = 0; vi < _actions.size(); vi++) 155 | sum += _cells[k]._actionConnections[vi]._weight * _actions[vi]._exploratoryState; 156 | 157 | _cells[k]._actionState = sigmoid(sum) * _cells[k]._state; 158 | 159 | q += _qConnections[k]._weight * _cells[k]._actionState; 160 | } 161 | else 162 | _cells[k]._actionState = 0.0f; 163 | } 164 | 165 | float tdError = reward + gamma * q - _prevValue; 166 | float qAlphaTdError = qAlpha * tdError; 167 | float actionAlphaTdError = actionAlpha * tdError; 168 | float surprise = tdError * tdError; 169 | 170 | float learnPattern = sigmoid(surpriseLearnFactor * (surprise - _averageSurprise)); 171 | //std::cout << "LP: " << learnPattern << std::endl; 172 | _averageSurprise = (1.0f - averageSurpiseDecay) * _averageSurprise + averageSurpiseDecay * surprise; 173 | 174 | // Update weights 175 | for (int k = 0; k < _cells.size(); k++) { 176 | float error = _qConnections[k]._weight * _cells[k]._actionState * (1.0f - _cells[k]._actionState); 177 | 178 | //_cells[k]._actionBias._weight += actionAlphaTdError * _cells[k]._actionBias._trace; 179 | 180 | //_cells[k]._actionBias._trace = _cells[k]._actionBias._trace * gammaLambda + error; 181 | 182 | for (int vi = 0; vi < _actions.size(); vi++) { 183 | _cells[k]._actionConnections[vi]._weight += actionAlphaTdError * _cells[k]._actionConnections[vi]._trace; 184 | 185 | _cells[k]._actionConnections[vi]._trace = _cells[k]._actionConnections[vi]._trace * gammaLambda + error * _actions[vi]._exploratoryState; 186 | } 187 | 188 | _qConnections[k]._weight += qAlphaTdError * _qConnections[k]._trace; 189 | 190 | _qConnections[k]._trace = _qConnections[k]._trace * gammaLambda + _cells[k]._actionState; 191 | } 192 | 193 | float sparsitySquared = sparsity * sparsity; 194 | 195 | for (int i = 0; i < _cells.size(); i++) { 196 | // Learn SDRs 197 | if (_cells[i]._state > 0.0f) { 198 | for (int j = 0; j < _inputs.size(); j++) 199 | _cells[i]._feedForwardConnections[j]._weight += gateFeedForwardAlpha * std::exp(-std::abs(_cells[i]._activation)) * (_cells[i]._activation > 0.0f ? -1.0f : 1.0f) * _reconstructionError[j]; 200 | } 201 | 202 | _cells[i]._threshold += gateThresholdAlpha * (_cells[i]._state - sparsity); 203 | } 204 | 205 | _prevValue = q; 206 | } -------------------------------------------------------------------------------- /NeoRL/source/dirent.h: -------------------------------------------------------------------------------- 1 | /***************************************************************************** 2 | * dirent.h - dirent API for Microsoft Visual Studio 3 | * 4 | * Copyright (C) 2006 Toni Ronkko 5 | * 6 | * Permission is hereby granted, free of charge, to any person obtaining 7 | * a copy of this software and associated documentation files (the 8 | * ``Software''), to deal in the Software without restriction, including 9 | * without limitation the rights to use, copy, modify, merge, publish, 10 | * distribute, sublicense, and/or sell copies of the Software, and to 11 | * permit persons to whom the Software is furnished to do so, subject to 12 | * the following conditions: 13 | * 14 | * The above copyright notice and this permission notice shall be included 15 | * in all copies or substantial portions of the Software. 16 | * 17 | * THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, EXPRESS 18 | * OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 | * IN NO EVENT SHALL TONI RONKKO BE LIABLE FOR ANY CLAIM, DAMAGES OR 21 | * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 22 | * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 23 | * OTHER DEALINGS IN THE SOFTWARE. 24 | * 25 | * Dec 15, 2009, John Cunningham 26 | * Added rewinddir member function 27 | * 28 | * Jan 18, 2008, Toni Ronkko 29 | * Using FindFirstFileA and WIN32_FIND_DATAA to avoid converting string 30 | * between multi-byte and unicode representations. This makes the 31 | * code simpler and also allows the code to be compiled under MingW. Thanks 32 | * to Azriel Fasten for the suggestion. 33 | * 34 | * Mar 4, 2007, Toni Ronkko 35 | * Bug fix: due to the strncpy_s() function this file only compiled in 36 | * Visual Studio 2005. Using the new string functions only when the 37 | * compiler version allows. 38 | * 39 | * Nov 2, 2006, Toni Ronkko 40 | * Major update: removed support for Watcom C, MS-DOS and Turbo C to 41 | * simplify the file, updated the code to compile cleanly on Visual 42 | * Studio 2005 with both unicode and multi-byte character strings, 43 | * removed rewinddir() as it had a bug. 44 | * 45 | * Aug 20, 2006, Toni Ronkko 46 | * Removed all remarks about MSVC 1.0, which is antiqued now. Simplified 47 | * comments by removing SGML tags. 48 | * 49 | * May 14 2002, Toni Ronkko 50 | * Embedded the function definitions directly to the header so that no 51 | * source modules need to be included in the Visual Studio project. Removed 52 | * all the dependencies to other projects so that this very header can be 53 | * used independently. 54 | * 55 | * May 28 1998, Toni Ronkko 56 | * First version. 57 | *****************************************************************************/ 58 | #ifndef DIRENT_H 59 | #define DIRENT_H 60 | 61 | #include 62 | #include 63 | #include 64 | 65 | 66 | typedef struct dirent 67 | { 68 | char d_name[MAX_PATH + 1]; /* current dir entry (multi-byte char string) */ 69 | WIN32_FIND_DATAA data; /* file attributes */ 70 | } dirent; 71 | 72 | 73 | typedef struct DIR 74 | { 75 | dirent current; /* Current directory entry */ 76 | int cached; /* Indicates un-processed entry in memory */ 77 | HANDLE search_handle; /* File search handle */ 78 | char patt[MAX_PATH + 3]; /* search pattern (3 = pattern + "\\*\0") */ 79 | } DIR; 80 | 81 | 82 | /* Forward declarations */ 83 | static DIR *opendir (const char *dirname); 84 | static struct dirent *readdir (DIR *dirp); 85 | static int closedir (DIR *dirp); 86 | static void rewinddir(DIR* dirp); 87 | 88 | 89 | /* Use the new safe string functions introduced in Visual Studio 2005 */ 90 | #if defined(_MSC_VER) && _MSC_VER >= 1400 91 | # define STRNCPY(dest,src,size) strncpy_s((dest),(size),(src),_TRUNCATE) 92 | #else 93 | # define STRNCPY(dest,src,size) strncpy((dest),(src),(size)) 94 | #endif 95 | 96 | 97 | /***************************************************************************** 98 | * Open directory stream DIRNAME for read and return a pointer to the 99 | * internal working area that is used to retrieve individual directory 100 | * entries. 101 | */ 102 | static DIR *opendir(const char *dirname) 103 | { 104 | DIR *dirp; 105 | assert (dirname != NULL); 106 | assert (strlen (dirname) < MAX_PATH); 107 | 108 | /* construct new DIR structure */ 109 | dirp = (DIR*) malloc (sizeof (struct DIR)); 110 | if (dirp != NULL) { 111 | char *p; 112 | 113 | /* take directory name... */ 114 | STRNCPY (dirp->patt, dirname, sizeof(dirp->patt)); 115 | dirp->patt[MAX_PATH] = '\0'; 116 | 117 | /* ... and append search pattern to it */ 118 | p = strchr (dirp->patt, '\0'); 119 | if (dirp->patt < p && *(p-1) != '\\' && *(p-1) != ':') { 120 | *p++ = '\\'; 121 | } 122 | *p++ = '*'; 123 | *p = '\0'; 124 | 125 | /* open stream and retrieve first file */ 126 | dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->current.data); 127 | if (dirp->search_handle == INVALID_HANDLE_VALUE) { 128 | /* invalid search pattern? */ 129 | free (dirp); 130 | return NULL; 131 | } 132 | 133 | /* there is an un-processed directory entry in memory now */ 134 | dirp->cached = 1; 135 | } 136 | 137 | return dirp; 138 | } 139 | 140 | 141 | /***************************************************************************** 142 | * Read a directory entry, and return a pointer to a dirent structure 143 | * containing the name of the entry in d_name field. Individual directory 144 | * entries returned by this very function include regular files, 145 | * sub-directories, pseudo-directories "." and "..", but also volume labels, 146 | * hidden files and system files may be returned. 147 | */ 148 | static struct dirent *readdir(DIR *dirp) 149 | { 150 | assert (dirp != NULL); 151 | 152 | if (dirp->search_handle == INVALID_HANDLE_VALUE) { 153 | /* directory stream was opened/rewound incorrectly or ended normally */ 154 | return NULL; 155 | } 156 | 157 | /* get next directory entry */ 158 | if (dirp->cached != 0) { 159 | /* a valid directory entry already in memory */ 160 | dirp->cached = 0; 161 | } else { 162 | /* read next directory entry from disk */ 163 | if (FindNextFileA (dirp->search_handle, &dirp->current.data) == FALSE) { 164 | /* the very last file has been processed or an error occured */ 165 | FindClose (dirp->search_handle); 166 | dirp->search_handle = INVALID_HANDLE_VALUE; 167 | return NULL; 168 | } 169 | } 170 | 171 | /* copy as a multibyte character string */ 172 | STRNCPY ( dirp->current.d_name, 173 | dirp->current.data.cFileName, 174 | sizeof(dirp->current.d_name) ); 175 | dirp->current.d_name[MAX_PATH] = '\0'; 176 | 177 | return &dirp->current; 178 | } 179 | 180 | 181 | /***************************************************************************** 182 | * Close directory stream opened by opendir() function. Close of the 183 | * directory stream invalidates the DIR structure as well as any previously 184 | * read directory entry. 185 | */ 186 | static int closedir(DIR *dirp) 187 | { 188 | assert (dirp != NULL); 189 | 190 | /* release search handle */ 191 | if (dirp->search_handle != INVALID_HANDLE_VALUE) { 192 | FindClose (dirp->search_handle); 193 | dirp->search_handle = INVALID_HANDLE_VALUE; 194 | } 195 | 196 | /* release directory handle */ 197 | free (dirp); 198 | return 0; 199 | } 200 | 201 | 202 | /***************************************************************************** 203 | * Resets the position of the directory stream to which dirp refers to the 204 | * beginning of the directory. It also causes the directory stream to refer 205 | * to the current state of the corresponding directory, as a call to opendir() 206 | * would have done. If dirp does not refer to a directory stream, the effect 207 | * is undefined. 208 | */ 209 | static void rewinddir(DIR* dirp) 210 | { 211 | /* release search handle */ 212 | if (dirp->search_handle != INVALID_HANDLE_VALUE) { 213 | FindClose (dirp->search_handle); 214 | dirp->search_handle = INVALID_HANDLE_VALUE; 215 | } 216 | 217 | /* open new search handle and retrieve first file */ 218 | dirp->search_handle = FindFirstFileA (dirp->patt, &dirp->current.data); 219 | if (dirp->search_handle == INVALID_HANDLE_VALUE) { 220 | /* invalid search pattern? */ 221 | free (dirp); 222 | return; 223 | } 224 | 225 | /* there is an un-processed directory entry in memory now */ 226 | dirp->cached = 1; 227 | } 228 | 229 | 230 | #endif /*DIRENT_H*/ 231 | -------------------------------------------------------------------------------- /NeoRL/source/NLevelGenerator.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_N_LEVEL_GENERATOR 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | 11 | #include 12 | 13 | #include 14 | 15 | enum Section { 16 | _name = 0, _author, _type, _data, _end 17 | }; 18 | 19 | struct Level { 20 | std::string _name; 21 | std::string _author; 22 | std::string _type; 23 | 24 | std::string _tileData; 25 | std::string _objectData; 26 | }; 27 | 28 | void loadLevels(const std::string &fileName, std::vector &levels, std::unordered_set &tileDataCharset, std::unordered_set &objectDataCharset) { 29 | std::ifstream fromFile(fileName); 30 | 31 | // Skip first 50 lines 32 | for (int i = 0; i < 50; i++) { 33 | std::string line; 34 | 35 | std::getline(fromFile, line); 36 | } 37 | 38 | while (!fromFile.eof() && fromFile.good()) { 39 | std::string line; 40 | 41 | std::getline(fromFile, line); 42 | 43 | // If line starts with "$", then it is a level 44 | if (line[0] == '$') { 45 | Level l; 46 | 47 | int lastIndex = 1; 48 | 49 | for (int i = 0; i < _end; i++) { 50 | // Search for next delimiter 51 | int d; 52 | 53 | for (d = lastIndex; d < line.length(); d++) 54 | if (line[d] == '#') 55 | break; 56 | 57 | std::string data = line.substr(lastIndex, d - lastIndex); 58 | 59 | switch (i) { 60 | case _name: 61 | l._name = data; 62 | 63 | break; 64 | 65 | case _author: 66 | l._author = data; 67 | 68 | break; 69 | 70 | case _type: 71 | l._type = data; 72 | 73 | break; 74 | 75 | case _data: 76 | 77 | // Search for pipe, which splits data into 2 sections 78 | int k; 79 | 80 | for (k = 0; k < data.length(); k++) 81 | if (data[k] == '|') 82 | break; 83 | 84 | l._tileData = data.substr(0, k); 85 | l._objectData = data.substr(k + 1, data.length() - (k + 1)); 86 | 87 | // Check charsets 88 | for (int t = 0; t < l._tileData.length(); t++) { 89 | if (tileDataCharset.find(l._tileData[t]) == tileDataCharset.end()) 90 | tileDataCharset.insert(l._tileData[t]); 91 | } 92 | 93 | for (int t = 0; t < l._objectData.length(); t++) { 94 | if (objectDataCharset.find(l._objectData[t]) == objectDataCharset.end()) 95 | objectDataCharset.insert(l._objectData[t]); 96 | } 97 | 98 | break; 99 | } 100 | 101 | lastIndex = d + 1; 102 | } 103 | 104 | levels.push_back(l); 105 | } 106 | } 107 | } 108 | 109 | int main() { 110 | std::mt19937 generator(time(nullptr)); 111 | 112 | sys::ComputeSystem cs; 113 | 114 | cs.create(sys::ComputeSystem::_gpu); 115 | 116 | sys::ComputeProgram prog; 117 | 118 | prog.loadFromFile("resources/neoKernels2.cl", cs); 119 | 120 | std::vector levels; 121 | 122 | std::unordered_set tileDataCharset; 123 | std::unordered_set objectDataCharset; 124 | 125 | loadLevels("userlevels.txt", levels, tileDataCharset, objectDataCharset); 126 | 127 | // --------------------------- Create the Sparse Coder --------------------------- 128 | 129 | const int numTiles = levels.front()._tileData.length(); 130 | 131 | const int charsetSize = 128 + 2; // + 2 indicating the portion it is on (01 for tiles, 10 for objects) 132 | 133 | const cl::size_type visDim = std::ceil(std::sqrt(static_cast(charsetSize))); 134 | const int visArea = visDim * visDim; 135 | 136 | const int maxObjects = 1000; 137 | 138 | cl::Image2D input = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), visDim, visDim); 139 | 140 | std::vector inputData(visArea, 0.0f); 141 | std::vector predData(visArea, 0.0f); 142 | 143 | std::vector layerDescs(3); 144 | 145 | layerDescs[0]._size = { 16, 16 }; 146 | layerDescs[1]._size = { 16, 16 }; 147 | layerDescs[2]._size = { 16, 16 }; 148 | 149 | neo::PredictiveHierarchy ph; 150 | 151 | ph.createRandom(cs, prog, { static_cast(visDim), static_cast(visDim) }, layerDescs, { -0.01f, 0.01f }, generator); 152 | 153 | ph._whiteningKernelRadius = 1; 154 | 155 | // Learn levels 156 | std::uniform_int_distribution levelDist(0, levels.size() - 1); 157 | 158 | for (int iter = 0; iter < 20; iter++) { 159 | // Choose random level 160 | int index = levelDist(generator); 161 | 162 | const Level &l = levels[index]; 163 | 164 | // Set mode for tiles 165 | inputData[charsetSize - 2] = 0.0f; 166 | inputData[charsetSize - 1] = 1.0f; 167 | 168 | // Run through once to get PH ready 169 | inputData['#'] = 1.0f; 170 | 171 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 172 | 173 | ph.simStep(cs, input); 174 | 175 | inputData['#'] = 0.0f; 176 | 177 | // Run through tile data 178 | for (int i = 0; i < l._tileData.length(); i++) { 179 | // Set character to 1 180 | inputData[l._tileData[i]] = 1.0f; 181 | 182 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 183 | 184 | ph.simStep(cs, input); 185 | 186 | // Unset character 187 | inputData[l._tileData[i]] = 0.0f; 188 | } 189 | 190 | // Set mode for objects 191 | inputData[charsetSize - 2] = 1.0f; 192 | inputData[charsetSize - 1] = 0.0f; 193 | 194 | // Run through once to get PH ready 195 | inputData['#'] = 1.0f; 196 | 197 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, { 0, 0, 0 }, { visDim, visDim, 1 }, 0, 0, inputData.data()); 198 | 199 | ph.simStep(cs, input); 200 | 201 | inputData['#'] = 0.0f; 202 | 203 | // Run through object data 204 | for (int i = 0; i < l._objectData.length(); i++) { 205 | // Set character to 1 206 | inputData[l._objectData[i]] = 1.0f; 207 | 208 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, { 0, 0, 0 }, { visDim, visDim, 1 }, 0, 0, inputData.data()); 209 | 210 | ph.simStep(cs, input); 211 | 212 | // Unset character 213 | inputData[l._objectData[i]] = 0.0f; 214 | } 215 | 216 | std::cout << "Went over level #" << (index + 1) << " \"" << l._name << "\"" << std::endl; 217 | } 218 | 219 | // Generate new maps 220 | std::ofstream toFile("generatedNLevels.txt"); 221 | 222 | std::normal_distribution noiseDist(0.0f, 1.0f); 223 | 224 | for (int i = 0; i < 10; i++) { 225 | toFile << "$" << "Generated Level " << (i + 1) << "#NeoRL#Experimental#"; 226 | 227 | // Generated level data 228 | 229 | 230 | // Set mode for tiles 231 | inputData[charsetSize - 2] = 0.0f; 232 | inputData[charsetSize - 1] = 1.0f; 233 | 234 | // Run through once to get PH ready 235 | //cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 236 | 237 | //ph.simStep(cs, input); 238 | 239 | char prevChar = 0; 240 | 241 | for (int i = 0; i < numTiles; i++) { 242 | // Set character to 1 243 | inputData[prevChar] = 1.0f; 244 | 245 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 246 | 247 | ph.simStep(cs, input, false); 248 | 249 | // Unset character 250 | inputData[prevChar] = 0.0f; 251 | 252 | char newChar = 0; 253 | 254 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, predData.data()); 255 | 256 | for (int j = 1; j < charsetSize - 2; j++) 257 | if (predData[j] > predData[newChar]) 258 | newChar = j; 259 | 260 | // Add new character 261 | toFile << newChar; 262 | 263 | prevChar = newChar; 264 | } 265 | 266 | toFile << "|"; 267 | 268 | // Set mode for objects 269 | inputData[charsetSize - 2] = 1.0f; 270 | inputData[charsetSize - 1] = 0.0f; 271 | 272 | // Run through once to get PH ready 273 | //cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 274 | 275 | //ph.simStep(cs, input); 276 | 277 | prevChar = 0; 278 | 279 | for (int i = 0; i < maxObjects; i++) { 280 | // Set character to 1 281 | inputData[prevChar] = 1.0f; 282 | 283 | std::vector noisyInputData = inputData; 284 | 285 | for (int j = 0; j < noisyInputData.size(); j++) { 286 | noisyInputData[j] += noiseDist(generator) * 0.1f; 287 | } 288 | 289 | cs.getQueue().enqueueWriteImage(input, CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, inputData.data()); 290 | 291 | ph.simStep(cs, input, false); 292 | 293 | // Unset character 294 | inputData[prevChar] = 0.0f; 295 | 296 | char newChar = 0; 297 | 298 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, cl::array { 0, 0, 0 }, cl::array { visDim, visDim, 1 }, 0, 0, predData.data()); 299 | 300 | for (int j = 1; j < charsetSize - 2; j++) 301 | if (predData[j] > predData[newChar]) 302 | newChar = j; 303 | 304 | // If is delimiter, break 305 | if (newChar == '#') 306 | break; 307 | 308 | // Add new character 309 | toFile << newChar; 310 | 311 | prevChar = newChar; 312 | } 313 | 314 | toFile << "#" << std::endl << std::endl; 315 | } 316 | 317 | return 0; 318 | } 319 | 320 | #endif -------------------------------------------------------------------------------- /NeoRL/FindSFML.cmake: -------------------------------------------------------------------------------- 1 | # This script locates the SFML library 2 | # ------------------------------------ 3 | # 4 | # Usage 5 | # ----- 6 | # 7 | # When you try to locate the SFML libraries, you must specify which modules you want to use (system, window, graphics, network, audio, main). 8 | # If none is given, the SFML_LIBRARIES variable will be empty and you'll end up linking to nothing. 9 | # example: 10 | # find_package(SFML COMPONENTS graphics window system) // find the graphics, window and system modules 11 | # 12 | # You can enforce a specific version, either MAJOR.MINOR or only MAJOR. 13 | # If nothing is specified, the version won't be checked (ie. any version will be accepted). 14 | # example: 15 | # find_package(SFML COMPONENTS ...) // no specific version required 16 | # find_package(SFML 2 COMPONENTS ...) // any 2.x version 17 | # find_package(SFML 2.4 COMPONENTS ...) // version 2.4 or greater 18 | # 19 | # By default, the dynamic libraries of SFML will be found. To find the static ones instead, 20 | # you must set the SFML_STATIC_LIBRARIES variable to TRUE before calling find_package(SFML ...). 21 | # In case of static linking, the SFML_STATIC macro will also be defined by this script. 22 | # example: 23 | # set(SFML_STATIC_LIBRARIES TRUE) 24 | # find_package(SFML 2 COMPONENTS network system) 25 | # 26 | # On Mac OS X if SFML_STATIC_LIBRARIES is not set to TRUE then by default CMake will search for frameworks unless 27 | # CMAKE_FIND_FRAMEWORK is set to "NEVER" for example. Please refer to CMake documentation for more details. 28 | # Moreover, keep in mind that SFML frameworks are only available as release libraries unlike dylibs which 29 | # are available for both release and debug modes. 30 | # 31 | # If SFML is not installed in a standard path, you can use the SFML_ROOT CMake (or environment) variable 32 | # to tell CMake where SFML is. 33 | # 34 | # Output 35 | # ------ 36 | # 37 | # This script defines the following variables: 38 | # - For each specified module XXX (system, window, graphics, network, audio, main): 39 | # - SFML_XXX_LIBRARY_DEBUG: the name of the debug library of the xxx module (set to SFML_XXX_LIBRARY_RELEASE is no debug version is found) 40 | # - SFML_XXX_LIBRARY_RELEASE: the name of the release library of the xxx module (set to SFML_XXX_LIBRARY_DEBUG is no release version is found) 41 | # - SFML_XXX_LIBRARY: the name of the library to link to for the xxx module (includes both debug and optimized names if necessary) 42 | # - SFML_XXX_FOUND: true if either the debug or release library of the xxx module is found 43 | # - SFML_LIBRARIES: the list of all libraries corresponding to the required modules 44 | # - SFML_FOUND: true if all the required modules are found 45 | # - SFML_INCLUDE_DIR: the path where SFML headers are located (the directory containing the SFML/Config.hpp file) 46 | # 47 | # example: 48 | # find_package(SFML 2 COMPONENTS system window graphics audio REQUIRED) 49 | # include_directories(${SFML_INCLUDE_DIR}) 50 | # add_executable(myapp ...) 51 | # target_link_libraries(myapp ${SFML_LIBRARIES}) 52 | 53 | # define the SFML_STATIC macro if static build was chosen 54 | if(SFML_STATIC_LIBRARIES) 55 | add_definitions(-DSFML_STATIC) 56 | endif() 57 | 58 | # deduce the libraries suffix from the options 59 | set(FIND_SFML_LIB_SUFFIX "") 60 | if(SFML_STATIC_LIBRARIES) 61 | set(FIND_SFML_LIB_SUFFIX "${FIND_SFML_LIB_SUFFIX}-s") 62 | endif() 63 | 64 | # find the SFML include directory 65 | find_path(SFML_INCLUDE_DIR SFML/Config.hpp 66 | PATH_SUFFIXES include 67 | PATHS 68 | ${SFML_ROOT} 69 | $ENV{SFML_ROOT} 70 | ~/Library/Frameworks 71 | /Library/Frameworks 72 | /usr/local/ 73 | /usr/ 74 | /sw # Fink 75 | /opt/local/ # DarwinPorts 76 | /opt/csw/ # Blastwave 77 | /opt/) 78 | 79 | # check the version number 80 | set(SFML_VERSION_OK TRUE) 81 | if(SFML_FIND_VERSION AND SFML_INCLUDE_DIR) 82 | # extract the major and minor version numbers from SFML/Config.hpp 83 | # we have to handle framework a little bit differently : 84 | if("${SFML_INCLUDE_DIR}" MATCHES "SFML.framework") 85 | set(SFML_CONFIG_HPP_INPUT "${SFML_INCLUDE_DIR}/Headers/Config.hpp") 86 | else() 87 | set(SFML_CONFIG_HPP_INPUT "${SFML_INCLUDE_DIR}/SFML/Config.hpp") 88 | endif() 89 | FILE(READ "${SFML_CONFIG_HPP_INPUT}" SFML_CONFIG_HPP_CONTENTS) 90 | STRING(REGEX MATCH ".*#define SFML_VERSION_MAJOR ([0-9]+).*#define SFML_VERSION_MINOR ([0-9]+).*" SFML_CONFIG_HPP_CONTENTS "${SFML_CONFIG_HPP_CONTENTS}") 91 | STRING(REGEX REPLACE ".*#define SFML_VERSION_MAJOR ([0-9]+).*" "\\1" SFML_VERSION_MAJOR "${SFML_CONFIG_HPP_CONTENTS}") 92 | STRING(REGEX REPLACE ".*#define SFML_VERSION_MINOR ([0-9]+).*" "\\1" SFML_VERSION_MINOR "${SFML_CONFIG_HPP_CONTENTS}") 93 | math(EXPR SFML_REQUESTED_VERSION "${SFML_FIND_VERSION_MAJOR} * 10 + ${SFML_FIND_VERSION_MINOR}") 94 | 95 | # if we could extract them, compare with the requested version number 96 | if (SFML_VERSION_MAJOR) 97 | # transform version numbers to an integer 98 | math(EXPR SFML_VERSION "${SFML_VERSION_MAJOR} * 10 + ${SFML_VERSION_MINOR}") 99 | 100 | # compare them 101 | if(SFML_VERSION LESS SFML_REQUESTED_VERSION) 102 | set(SFML_VERSION_OK FALSE) 103 | endif() 104 | else() 105 | # SFML version is < 2.0 106 | if (SFML_REQUESTED_VERSION GREATER 19) 107 | set(SFML_VERSION_OK FALSE) 108 | set(SFML_VERSION_MAJOR 1) 109 | set(SFML_VERSION_MINOR x) 110 | endif() 111 | endif() 112 | endif() 113 | 114 | # find the requested modules 115 | set(SFML_FOUND TRUE) # will be set to false if one of the required modules is not found 116 | set(FIND_SFML_LIB_PATHS 117 | ${SFML_ROOT} 118 | $ENV{SFML_ROOT} 119 | ~/Library/Frameworks 120 | /Library/Frameworks 121 | /usr/local 122 | /usr 123 | /sw 124 | /opt/local 125 | /opt/csw 126 | /opt) 127 | foreach(FIND_SFML_COMPONENT ${SFML_FIND_COMPONENTS}) 128 | string(TOLOWER ${FIND_SFML_COMPONENT} FIND_SFML_COMPONENT_LOWER) 129 | string(TOUPPER ${FIND_SFML_COMPONENT} FIND_SFML_COMPONENT_UPPER) 130 | set(FIND_SFML_COMPONENT_NAME sfml-${FIND_SFML_COMPONENT_LOWER}${FIND_SFML_LIB_SUFFIX}) 131 | 132 | # no suffix for sfml-main, it is always a static library 133 | if(FIND_SFML_COMPONENT_LOWER STREQUAL "main") 134 | set(FIND_SFML_COMPONENT_NAME sfml-${FIND_SFML_COMPONENT_LOWER}) 135 | endif() 136 | 137 | # debug library 138 | find_library(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG 139 | NAMES ${FIND_SFML_COMPONENT_NAME}-d 140 | PATH_SUFFIXES lib64 lib 141 | PATHS ${FIND_SFML_LIB_PATHS}) 142 | 143 | # release library 144 | find_library(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE 145 | NAMES ${FIND_SFML_COMPONENT_NAME} 146 | PATH_SUFFIXES lib64 lib 147 | PATHS ${FIND_SFML_LIB_PATHS}) 148 | 149 | if (SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG OR SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE) 150 | # library found 151 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_FOUND TRUE) 152 | 153 | # if both are found, set SFML_XXX_LIBRARY to contain both 154 | if (SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG AND SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE) 155 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY debug ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG} 156 | optimized ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE}) 157 | endif() 158 | 159 | # if only one debug/release variant is found, set the other to be equal to the found one 160 | if (SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG AND NOT SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE) 161 | # debug and not release 162 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG}) 163 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG}) 164 | endif() 165 | if (SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE AND NOT SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG) 166 | # release and not debug 167 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE}) 168 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY ${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE}) 169 | endif() 170 | else() 171 | # library not found 172 | set(SFML_FOUND FALSE) 173 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_FOUND FALSE) 174 | set(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY "") 175 | set(FIND_SFML_MISSING "${FIND_SFML_MISSING} SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY") 176 | endif() 177 | 178 | # mark as advanced 179 | MARK_AS_ADVANCED(SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY 180 | SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_RELEASE 181 | SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY_DEBUG) 182 | 183 | # add to the global list of libraries 184 | set(SFML_LIBRARIES ${SFML_LIBRARIES} "${SFML_${FIND_SFML_COMPONENT_UPPER}_LIBRARY}") 185 | endforeach() 186 | 187 | # handle errors 188 | if(NOT SFML_VERSION_OK) 189 | # SFML version not ok 190 | set(FIND_SFML_ERROR "SFML found but version too low (requested: ${SFML_FIND_VERSION}, found: ${SFML_VERSION_MAJOR}.${SFML_VERSION_MINOR})") 191 | set(SFML_FOUND FALSE) 192 | elseif(NOT SFML_FOUND) 193 | # include directory or library not found 194 | set(FIND_SFML_ERROR "Could NOT find SFML (missing: ${FIND_SFML_MISSING})") 195 | endif() 196 | if (NOT SFML_FOUND) 197 | if(SFML_FIND_REQUIRED) 198 | # fatal error 199 | message(FATAL_ERROR ${FIND_SFML_ERROR}) 200 | elseif(NOT SFML_FIND_QUIETLY) 201 | # error but continue 202 | message("${FIND_SFML_ERROR}") 203 | endif() 204 | endif() 205 | 206 | # handle success 207 | if(SFML_FOUND) 208 | message(STATUS "Found SFML ${SFML_VERSION_MAJOR}.${SFML_VERSION_MINOR} in ${SFML_INCLUDE_DIR}") 209 | endif() 210 | -------------------------------------------------------------------------------- /NeoRL/source/neo/PredictorSwarm.cpp: -------------------------------------------------------------------------------- 1 | #include "PredictorSwarm.h" 2 | 3 | using namespace neo; 4 | 5 | void PredictorSwarm::createRandom(sys::ComputeSystem &cs, sys::ComputeProgram &program, 6 | const std::vector &visibleLayerDescs, cl_int2 hiddenSize, cl_float2 initWeightRange, 7 | std::mt19937 &rng) 8 | { 9 | _visibleLayerDescs = visibleLayerDescs; 10 | 11 | _hiddenSize = hiddenSize; 12 | 13 | _visibleLayers.resize(_visibleLayerDescs.size()); 14 | 15 | cl::Kernel randomUniform2DKernel = cl::Kernel(program.getProgram(), "randomUniform2D"); 16 | cl::Kernel randomUniform3DKernel = cl::Kernel(program.getProgram(), "randomUniform3D"); 17 | cl::Kernel randomUniform3DXZKernel = cl::Kernel(program.getProgram(), "randomUniform3DXZ"); 18 | 19 | cl_float4 zeroColor = { 0.0f, 0.0f, 0.0f, 0.0f }; 20 | 21 | cl::array zeroOrigin = { 0, 0, 0 }; 22 | cl::array hiddenRegion = { _hiddenSize.x, _hiddenSize.y, 1 }; 23 | 24 | // Create layers 25 | for (int vli = 0; vli < _visibleLayers.size(); vli++) { 26 | VisibleLayer &vl = _visibleLayers[vli]; 27 | VisibleLayerDesc &vld = _visibleLayerDescs[vli]; 28 | 29 | vl._hiddenToVisible = cl_float2{ static_cast(vld._size.x) / static_cast(_hiddenSize.x), 30 | static_cast(vld._size.y) / static_cast(_hiddenSize.y) 31 | }; 32 | 33 | vl._visibleToHidden = cl_float2{ static_cast(_hiddenSize.x) / static_cast(vld._size.x), 34 | static_cast(_hiddenSize.y) / static_cast(vld._size.y) 35 | }; 36 | 37 | vl._reverseRadii = cl_int2{ static_cast(std::ceil(vl._visibleToHidden.x * (vld._radius + 0.5f))), static_cast(std::ceil(vl._visibleToHidden.y * (vld._radius + 0.5f))) }; 38 | 39 | int weightDiam = vld._radius * 2 + 1; 40 | 41 | int numWeights = weightDiam * weightDiam; 42 | 43 | cl_int3 weightsSize = { _hiddenSize.x, _hiddenSize.y, numWeights }; 44 | 45 | vl._weights = createDoubleBuffer3D(cs, weightsSize, CL_RGBA, CL_FLOAT); 46 | 47 | randomUniformXZ(vl._weights[_back], cs, randomUniform3DXZKernel, weightsSize, initWeightRange, rng); 48 | 49 | vl._qTraces = createDoubleBuffer3D(cs, weightsSize, CL_R, CL_FLOAT); 50 | 51 | cs.getQueue().enqueueFillImage(vl._qTraces[_back], zeroColor, zeroOrigin, { static_cast(weightsSize.x), static_cast(weightsSize.y), static_cast(weightsSize.z) }); 52 | } 53 | 54 | // Hidden state data 55 | _hiddenStates = createDoubleBuffer2D(cs, _hiddenSize, CL_RG, CL_FLOAT); 56 | 57 | _hiddenActivations = createDoubleBuffer2D(cs, _hiddenSize, CL_RG, CL_FLOAT); 58 | 59 | _hiddenSummationTemp = createDoubleBuffer2D(cs, _hiddenSize, CL_RG, CL_FLOAT); 60 | 61 | cs.getQueue().enqueueFillImage(_hiddenStates[_back], zeroColor, zeroOrigin, hiddenRegion); 62 | cs.getQueue().enqueueFillImage(_hiddenActivations[_back], zeroColor, zeroOrigin, hiddenRegion); 63 | 64 | // Create kernels 65 | _activateKernel = cl::Kernel(program.getProgram(), "predActivateSwarm"); 66 | _solveHiddenKernel = cl::Kernel(program.getProgram(), "predSolveHiddenSwarm"); 67 | _solveHiddenNoInhibitionKernel = cl::Kernel(program.getProgram(), "predSolveHiddenNoInhibitionSwarm"); 68 | _learnWeightsTracesInhibitedKernel = cl::Kernel(program.getProgram(), "predLearnWeightsTracesSwarm"); 69 | _reconstructionErrorKernel = cl::Kernel(program.getProgram(), "predReconstructionErrorSwarm"); 70 | } 71 | 72 | void PredictorSwarm::activate(sys::ComputeSystem &cs, const cl::Image2D &targets, const std::vector &visibleStates, const std::vector &visibleStatesPrev, float activeRatio, int inhibitionRadius, float noise, std::mt19937 &rng) { 73 | // Start by clearing summation buffer 74 | { 75 | cl_float4 zeroColor = { 0.0f, 0.0f, 0.0f, 0.0f }; 76 | 77 | cl::array zeroOrigin = { 0, 0, 0 }; 78 | cl::array hiddenRegion = { _hiddenSize.x, _hiddenSize.y, 1 }; 79 | 80 | //cs.getQueue().enqueueCopyImage(_hiddenBiases[_back], _hiddenSummationTemp[_back], zeroOrigin, zeroOrigin, hiddenRegion); 81 | cs.getQueue().enqueueFillImage(_hiddenSummationTemp[_back], cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, zeroOrigin, hiddenRegion); 82 | } 83 | 84 | for (int vli = 0; vli < _visibleLayers.size(); vli++) { 85 | VisibleLayer &vl = _visibleLayers[vli]; 86 | VisibleLayerDesc &vld = _visibleLayerDescs[vli]; 87 | 88 | int argIndex = 0; 89 | 90 | _activateKernel.setArg(argIndex++, visibleStates[vli]); 91 | _activateKernel.setArg(argIndex++, _hiddenSummationTemp[_back]); 92 | _activateKernel.setArg(argIndex++, _hiddenSummationTemp[_front]); 93 | _activateKernel.setArg(argIndex++, vl._weights[_back]); 94 | _activateKernel.setArg(argIndex++, vld._size); 95 | _activateKernel.setArg(argIndex++, vl._hiddenToVisible); 96 | _activateKernel.setArg(argIndex++, vld._radius); 97 | 98 | cs.getQueue().enqueueNDRangeKernel(_activateKernel, cl::NullRange, cl::NDRange(_hiddenSize.x, _hiddenSize.y)); 99 | 100 | // Swap buffers 101 | std::swap(_hiddenSummationTemp[_front], _hiddenSummationTemp[_back]); 102 | } 103 | 104 | { 105 | std::uniform_int_distribution seedDist(0, 999); 106 | 107 | cl_uint2 seed = { seedDist(rng), seedDist(rng) }; 108 | 109 | int argIndex = 0; 110 | 111 | _solveHiddenKernel.setArg(argIndex++, _hiddenSummationTemp[_back]); 112 | _solveHiddenKernel.setArg(argIndex++, _hiddenStates[_front]); 113 | _solveHiddenKernel.setArg(argIndex++, _hiddenActivations[_front]); 114 | _solveHiddenKernel.setArg(argIndex++, _hiddenSize); 115 | _solveHiddenKernel.setArg(argIndex++, inhibitionRadius); 116 | _solveHiddenKernel.setArg(argIndex++, activeRatio); 117 | 118 | cs.getQueue().enqueueNDRangeKernel(_solveHiddenKernel, cl::NullRange, cl::NDRange(_hiddenSize.x, _hiddenSize.y)); 119 | } 120 | 121 | // Swap hidden state buffers 122 | std::swap(_hiddenStates[_front], _hiddenStates[_back]); 123 | std::swap(_hiddenActivations[_front], _hiddenActivations[_back]); 124 | } 125 | 126 | void PredictorSwarm::activateNoInhibition(sys::ComputeSystem &cs, const cl::Image2D &targets, const std::vector &visibleStates, const std::vector &visibleStatesPrev, float activeRatio, int inhibitionRadius, float noise, std::mt19937 &rng) { 127 | // Start by clearing summation buffer 128 | { 129 | cl_float4 zeroColor = { 0.0f, 0.0f, 0.0f, 0.0f }; 130 | 131 | cl::array zeroOrigin = { 0, 0, 0 }; 132 | cl::array hiddenRegion = { _hiddenSize.x, _hiddenSize.y, 1 }; 133 | 134 | //cs.getQueue().enqueueCopyImage(_hiddenBiases[_back], _hiddenSummationTemp[_back], zeroOrigin, zeroOrigin, hiddenRegion); 135 | cs.getQueue().enqueueFillImage(_hiddenSummationTemp[_back], cl_float4{ 0.0f, 0.0f, 0.0f, 0.0f }, zeroOrigin, hiddenRegion); 136 | } 137 | 138 | for (int vli = 0; vli < _visibleLayers.size(); vli++) { 139 | VisibleLayer &vl = _visibleLayers[vli]; 140 | VisibleLayerDesc &vld = _visibleLayerDescs[vli]; 141 | 142 | int argIndex = 0; 143 | 144 | _activateKernel.setArg(argIndex++, visibleStates[vli]); 145 | _activateKernel.setArg(argIndex++, _hiddenSummationTemp[_back]); 146 | _activateKernel.setArg(argIndex++, _hiddenSummationTemp[_front]); 147 | _activateKernel.setArg(argIndex++, vl._weights[_back]); 148 | _activateKernel.setArg(argIndex++, vld._size); 149 | _activateKernel.setArg(argIndex++, vl._hiddenToVisible); 150 | _activateKernel.setArg(argIndex++, vld._radius); 151 | 152 | cs.getQueue().enqueueNDRangeKernel(_activateKernel, cl::NullRange, cl::NDRange(_hiddenSize.x, _hiddenSize.y)); 153 | 154 | // Swap buffers 155 | std::swap(_hiddenSummationTemp[_front], _hiddenSummationTemp[_back]); 156 | } 157 | 158 | { 159 | int argIndex = 0; 160 | 161 | _solveHiddenNoInhibitionKernel.setArg(argIndex++, _hiddenSummationTemp[_back]); 162 | _solveHiddenNoInhibitionKernel.setArg(argIndex++, _hiddenStates[_front]); 163 | _solveHiddenNoInhibitionKernel.setArg(argIndex++, _hiddenActivations[_front]); 164 | 165 | cs.getQueue().enqueueNDRangeKernel(_solveHiddenNoInhibitionKernel, cl::NullRange, cl::NDRange(_hiddenSize.x, _hiddenSize.y)); 166 | } 167 | 168 | // Swap hidden state buffers 169 | std::swap(_hiddenStates[_front], _hiddenStates[_back]); 170 | std::swap(_hiddenActivations[_front], _hiddenActivations[_back]); 171 | } 172 | 173 | void PredictorSwarm::learn(sys::ComputeSystem &cs, float reward, float gamma, const cl::Image2D &targets, std::vector &visibleStatesPrev, cl_float2 weightAlpha, cl_float2 weightLambda, cl_float biasAlpha, cl_float activeRatio, float noise) { 174 | // Learn weights 175 | for (int vli = 0; vli < _visibleLayers.size(); vli++) { 176 | VisibleLayer &vl = _visibleLayers[vli]; 177 | VisibleLayerDesc &vld = _visibleLayerDescs[vli]; 178 | 179 | int argIndex = 0; 180 | 181 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, visibleStatesPrev[vli]); 182 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, targets); 183 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, _hiddenStates[_back]); 184 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, _hiddenActivations[_front]); 185 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, _hiddenStates[_front]); 186 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vl._weights[_back]); 187 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vl._weights[_front]); 188 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vl._qTraces[_back]); 189 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vl._qTraces[_front]); 190 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vld._size); 191 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vl._hiddenToVisible); 192 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, vld._radius); 193 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, weightAlpha); 194 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, weightLambda); 195 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, reward); 196 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, gamma); 197 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, activeRatio); 198 | _learnWeightsTracesInhibitedKernel.setArg(argIndex++, noise); 199 | 200 | cs.getQueue().enqueueNDRangeKernel(_learnWeightsTracesInhibitedKernel, cl::NullRange, cl::NDRange(_hiddenSize.x, _hiddenSize.y)); 201 | 202 | std::swap(vl._weights[_front], vl._weights[_back]); 203 | std::swap(vl._qTraces[_front], vl._qTraces[_back]); 204 | } 205 | } -------------------------------------------------------------------------------- /NeoRL/source/MNIST_Video.cpp: -------------------------------------------------------------------------------- 1 | #include "Settings.h" 2 | 3 | #if EXPERIMENT_SELECTION == EXPERIMENT_MNIST_VIDEO 4 | 5 | #include 6 | 7 | #include 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | 17 | struct Image { 18 | std::vector _intensities; 19 | }; 20 | 21 | 22 | void loadMNISTimage(std::ifstream &fromFile, int index, Image &img) { 23 | const int headerSize = 16; 24 | const int imageSize = 28 * 28; 25 | 26 | fromFile.seekg(headerSize + index * imageSize); 27 | 28 | if (img._intensities.size() != 28 * 28) 29 | img._intensities.resize(28 * 28); 30 | 31 | fromFile.read(reinterpret_cast(img._intensities.data()), 28 * 28); 32 | } 33 | 34 | int main() { 35 | std::mt19937 generator(time(nullptr)); 36 | 37 | sys::ComputeSystem cs; 38 | 39 | cs.create(sys::ComputeSystem::_gpu); 40 | 41 | sys::ComputeProgram prog; 42 | 43 | prog.loadFromFile("resources/neoKernels2.cl", cs); 44 | 45 | // --------------------------- Create the Sparse Coder --------------------------- 46 | 47 | cl::Image2D inputImage = cl::Image2D(cs.getContext(), CL_MEM_READ_WRITE, cl::ImageFormat(CL_R, CL_FLOAT), 64, 64); 48 | 49 | std::ifstream fromFile("resources/train-images.idx3-ubyte", std::ios::binary | std::ios::in); 50 | 51 | if (!fromFile.is_open()) { 52 | std::cerr << "Could not open train-images.idx3-ubyte!" << std::endl; 53 | 54 | return 1; 55 | } 56 | 57 | std::vector layerDescs(4); 58 | 59 | layerDescs[0]._size = { 64, 64 }; 60 | layerDescs[1]._size = { 48, 48 }; 61 | layerDescs[2]._size = { 32, 32 }; 62 | layerDescs[3]._size = { 24, 24 }; 63 | neo::PredictiveHierarchy ph; 64 | 65 | ph.createRandom(cs, prog, { 64, 64 }, layerDescs, { -0.5f, 0.5f }, generator); 66 | 67 | float avgError = 1.0f; 68 | 69 | float avgErrorDecay = 0.1f; 70 | 71 | sf::RenderWindow window; 72 | 73 | window.create(sf::VideoMode(1024, 512), "MNIST Video Test"); 74 | 75 | vis::Plot plot; 76 | 77 | plot._curves.push_back(vis::Curve()); 78 | 79 | plot._curves[0]._name = "Squared Error"; 80 | 81 | std::uniform_int_distribution digitDist(0, 59999); 82 | std::uniform_real dist01(0.0f, 1.0f); 83 | 84 | sf::RenderTexture rt; 85 | 86 | rt.create(64, 64); 87 | 88 | sf::Image digit0; 89 | sf::Texture digit0Tex; 90 | sf::Image digit1; 91 | sf::Texture digit1Tex; 92 | 93 | sf::Image pred; 94 | sf::Texture predTex; 95 | 96 | digit0.create(28, 28); 97 | digit1.create(28, 28); 98 | 99 | pred.create(rt.getSize().x, rt.getSize().y); 100 | 101 | const float boundingSize = (64 - 28) / 2; 102 | const float center = 32; 103 | const float minimum = center - boundingSize; 104 | const float maximum = center + boundingSize; 105 | 106 | float avgError2 = 1.0f; 107 | 108 | const float avgError2Decay = 0.01f; 109 | 110 | std::vector prediction(64 * 64, 0.0f); 111 | 112 | for (int iter = 0; iter < 10000; iter++) { 113 | // Select digit indices 114 | int d0 = digitDist(generator); 115 | int d1 = digitDist(generator); 116 | 117 | // Load digits 118 | Image img0, img1; 119 | 120 | loadMNISTimage(fromFile, d0, img0); 121 | loadMNISTimage(fromFile, d1, img1); 122 | 123 | for (int x = 0; x < digit0.getSize().x; x++) 124 | for (int y = 0; y < digit0.getSize().y; y++) { 125 | int index = x + y * digit0.getSize().x; 126 | 127 | sf::Color c = sf::Color::White; 128 | 129 | c.a = img0._intensities[index]; 130 | 131 | digit0.setPixel(x, y, c); 132 | } 133 | 134 | digit0Tex.loadFromImage(digit0); 135 | 136 | for (int x = 0; x < digit1.getSize().x; x++) 137 | for (int y = 0; y < digit1.getSize().y; y++) { 138 | int index = x + y * digit1.getSize().x; 139 | 140 | sf::Color c = sf::Color::White; 141 | 142 | c.a = img1._intensities[index]; 143 | 144 | digit1.setPixel(x, y, c); 145 | } 146 | 147 | digit1Tex.loadFromImage(digit1); 148 | 149 | sf::Vector2f vel0(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f); 150 | sf::Vector2f vel1(dist01(generator) * 2.0f - 1.0f, dist01(generator) * 2.0f - 1.0f); 151 | 152 | sf::Vector2f pos0(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum); 153 | sf::Vector2f pos1(dist01(generator) * (maximum - minimum) + minimum, dist01(generator) * (maximum - minimum) + minimum); 154 | 155 | float vel0mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel0.x * vel0.x + vel0.y + vel0.y)); 156 | 157 | vel0 *= vel0mul; 158 | 159 | float vel1mul = dist01(generator) * 6.0f / std::max(1.0f, std::sqrt(vel1.x * vel1.x + vel1.y + vel1.y)); 160 | 161 | vel1 *= vel1mul; 162 | 163 | // Render video 164 | for (int f = 0; f < 20; f++) { 165 | sf::Event windowEvent; 166 | 167 | while (window.pollEvent(windowEvent)) { 168 | switch (windowEvent.type) { 169 | case sf::Event::Closed: 170 | return 0; 171 | } 172 | } 173 | 174 | pos0 += vel0; 175 | 176 | pos1 += vel1; 177 | 178 | if (pos0.x < minimum) { 179 | pos0.x = minimum; 180 | 181 | vel0.x *= -1.0f; 182 | } 183 | else if (pos0.x > maximum) { 184 | pos0.x = maximum; 185 | 186 | vel0.x *= -1.0f; 187 | } 188 | 189 | if (pos0.y < minimum) { 190 | pos0.y = minimum; 191 | 192 | vel0.y *= -1.0f; 193 | } 194 | else if (pos0.y > maximum) { 195 | pos0.y = maximum; 196 | 197 | vel0.y *= -1.0f; 198 | } 199 | 200 | if (pos1.x < minimum) { 201 | pos1.x = minimum; 202 | 203 | vel1.x *= -1.0f; 204 | } 205 | else if (pos1.x > maximum) { 206 | pos1.x = maximum; 207 | 208 | vel1.x *= -1.0f; 209 | } 210 | 211 | if (pos1.y < minimum) { 212 | pos1.y = minimum; 213 | 214 | vel1.y *= -1.0f; 215 | } 216 | else if (pos1.y > maximum) { 217 | pos1.y = maximum; 218 | 219 | vel1.y *= -1.0f; 220 | } 221 | 222 | window.clear(); 223 | rt.clear(sf::Color::Black); 224 | 225 | sf::Sprite s0; 226 | 227 | s0.setTexture(digit0Tex); 228 | 229 | s0.setOrigin(28 / 2, 28 / 2); 230 | 231 | s0.setPosition(pos0); 232 | 233 | rt.draw(s0); 234 | 235 | sf::Sprite s1; 236 | 237 | s1.setTexture(digit1Tex); 238 | 239 | s1.setOrigin(28 / 2, 28 / 2); 240 | 241 | s1.setPosition(pos1); 242 | 243 | rt.draw(s1); 244 | 245 | rt.display(); 246 | 247 | // Get input image 248 | sf::Image res = rt.getTexture().copyToImage(); 249 | 250 | // Show RT 251 | const float scale = 4.0f; 252 | 253 | sf::Sprite s; 254 | 255 | s.setScale(scale, scale); 256 | 257 | s.setTexture(rt.getTexture()); 258 | 259 | window.draw(s); 260 | 261 | std::vector input(64 * 64); 262 | 263 | // Train 264 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::T)) { 265 | for (int x = 0; x < res.getSize().x; x++) 266 | for (int y = 0; y < res.getSize().y; y++) { 267 | input[x + y * 64] = prediction[x + y * 64]; 268 | } 269 | } 270 | else { 271 | const float predictionIncorporateRatio = 0.1f; 272 | 273 | for (int x = 0; x < res.getSize().x; x++) 274 | for (int y = 0; y < res.getSize().y; y++) { 275 | input[x + y * 64] = (1.0f - predictionIncorporateRatio) * res.getPixel(x, y).r / 255.0f + predictionIncorporateRatio * prediction[x + y * 64]; 276 | } 277 | } 278 | 279 | // Error 280 | float error = 0.0f; 281 | 282 | for (int x = 0; x < res.getSize().x; x++) 283 | for (int y = 0; y < res.getSize().y; y++) { 284 | error += std::pow(res.getPixel(x, y).r / 255.0f - prediction[x + y * 64], 2); 285 | } 286 | 287 | error /= res.getSize().x * res.getSize().y; 288 | 289 | avgError2 = (1.0f - avgError2Decay) * avgError2 + avgError2Decay * error; 290 | 291 | std::cout << "Squared Error: " << avgError2 << std::endl; 292 | 293 | cs.getQueue().enqueueWriteImage(inputImage, CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, input.data()); 294 | 295 | ph.simStep(cs, inputImage, true, true); 296 | 297 | cs.getQueue().enqueueReadImage(ph.getPrediction(), CL_TRUE, { 0, 0, 0 }, { 64, 64, 1 }, 0, 0, prediction.data()); 298 | 299 | // Show prediction 300 | for (int x = 0; x < rt.getSize().x; x++) 301 | for (int y = 0; y < rt.getSize().y; y++) { 302 | sf::Color c = sf::Color::White; 303 | 304 | c.r = c.b = c.g = std::min(1.0f, std::max(0.0f, prediction[x + y * 64])) * 255.0f; 305 | 306 | pred.setPixel(x, y, c); 307 | } 308 | 309 | predTex.loadFromImage(pred); 310 | 311 | sf::Sprite sp; 312 | 313 | sp.setTexture(predTex); 314 | 315 | sp.setScale(scale, scale); 316 | 317 | sp.setPosition(window.getSize().x - scale * rt.getSize().x, 0); 318 | 319 | window.draw(sp); 320 | 321 | /*sf::Image sdr; 322 | 323 | sdr.create(prsdr.getLayerDescs().front()._width, prsdr.getLayerDescs().front()._height); 324 | 325 | for (int x = 0; x < sdr.getSize().x; x++) 326 | for (int y = 0; y < sdr.getSize().y; y++) { 327 | sf::Color c = sf::Color::White; 328 | 329 | c.r = c.g = c.b = prsdr.getLayers().front()._sdr.getHiddenState(x, y) * 255.0f; 330 | 331 | sdr.setPixel(x, y, c); 332 | } 333 | 334 | sf::Texture sdrTex; 335 | 336 | sdrTex.loadFromImage(sdr); 337 | 338 | sf::Sprite sdrS; 339 | 340 | sdrS.setTexture(sdrTex); 341 | 342 | sdrS.setPosition(0.0f, window.getSize().y - sdrTex.getSize().y * scale); 343 | 344 | sdrS.setScale(scale, scale); 345 | 346 | window.draw(sdrS);*/ 347 | 348 | window.display(); 349 | 350 | if (sf::Keyboard::isKeyPressed(sf::Keyboard::Escape)) 351 | return 0; 352 | } 353 | } 354 | 355 | /*sf::RenderTexture rt; 356 | 357 | rt.create(1024, 1024); 358 | 359 | sf::Texture lineGradientTexture; 360 | 361 | lineGradientTexture.loadFromFile("resources/lineGradient.png"); 362 | 363 | sf::Font tickFont; 364 | 365 | tickFont.loadFromFile("resources/arial.ttf"); 366 | 367 | plot.draw(rt, lineGradientTexture, tickFont, 1.0f, sf::Vector2f(0.0f, step), sf::Vector2f(0.0f, 1.0f), sf::Vector2f(128.0f, 128.0f), sf::Vector2f(500.0f, 0.1f), 2.0f, 3.0f, 1.5f, 3.0f, 20.0f, 6); 368 | 369 | rt.display(); 370 | 371 | rt.getTexture().copyToImage().saveToFile("plot.png");*/ 372 | 373 | return 0; 374 | } 375 | 376 | #endif -------------------------------------------------------------------------------- /NeoRL/source/vis/Plot.cpp: -------------------------------------------------------------------------------- 1 | #include "Plot.h" 2 | 3 | #include 4 | 5 | using namespace vis; 6 | 7 | void Plot::draw(sf::RenderTarget &target, const sf::Texture &lineGradientTexture, const sf::Font &tickFont, float tickTextScale, 8 | const sf::Vector2f &domain, const sf::Vector2f &range, const sf::Vector2f &margins, const sf::Vector2f &tickIncrements, float axesSize, float lineSize, float tickSize, float tickLength, float textTickOffset, int precision) 9 | { 10 | target.clear(_backgroundColor); 11 | 12 | sf::Vector2f plotSize = sf::Vector2f(target.getSize().x - margins.x, target.getSize().y - margins.y); 13 | 14 | sf::Vector2f origin = sf::Vector2f(margins.x, target.getSize().y - margins.y); 15 | 16 | // Draw curves 17 | for (int c = 0; c < _curves.size(); c++) { 18 | if (_curves[c]._points.empty()) 19 | continue; 20 | 21 | sf::VertexArray vertexArray; 22 | 23 | vertexArray.resize((_curves[c]._points.size() - 1) * 6); 24 | 25 | int index = 0; 26 | 27 | // Go through points 28 | for (int p = 0; p < _curves[c]._points.size() - 1; p++) { 29 | Point &point = _curves[c]._points[p]; 30 | Point &pointNext = _curves[c]._points[p + 1]; 31 | 32 | sf::Vector2f difference = pointNext._position - point._position; 33 | sf::Vector2f direction = vectorNormalize(difference); 34 | 35 | sf::Vector2f renderPointFirst, renderPointSecond; 36 | 37 | bool pointVisible = point._position.x >= domain.x && point._position.x <= domain.y && 38 | point._position.y >= range.x && point._position.y <= range.y; 39 | 40 | bool pointNextVisible = pointNext._position.x >= domain.x && pointNext._position.x <= domain.y && 41 | pointNext._position.y >= range.x && pointNext._position.y <= range.y; 42 | 43 | if (pointVisible || pointNextVisible) { 44 | sf::Vector2f renderPoint = sf::Vector2f(origin.x + (point._position.x - domain.x) / (domain.y - domain.x) * plotSize.x, 45 | origin.y - (point._position.y - range.x) / (range.y - range.x) * plotSize.y); 46 | 47 | sf::Vector2f renderPointNext = sf::Vector2f(origin.x + (pointNext._position.x - domain.x) / (domain.y - domain.x) * plotSize.x, 48 | origin.y - (pointNext._position.y - range.x) / (range.y - range.x) * plotSize.y); 49 | 50 | sf::Vector2f renderDirection = vectorNormalize(renderPointNext - renderPoint); 51 | 52 | sf::Vector2f sizeOffset; 53 | sf::Vector2f sizeOffsetNext; 54 | 55 | if (p > 0) { 56 | sf::Vector2f renderPointPrev = sf::Vector2f(origin.x + (_curves[c]._points[p - 1]._position.x - domain.x) / (domain.y - domain.x) * plotSize.x, 57 | origin.y - (_curves[c]._points[p - 1]._position.y - range.x) / (range.y - range.x) * plotSize.y); 58 | 59 | sf::Vector2f averageDirection = (renderDirection + vectorNormalize(renderPoint - renderPointPrev)) * 0.5f; 60 | 61 | sizeOffset = vectorNormalize(sf::Vector2f(-averageDirection.y, averageDirection.x)); 62 | } 63 | else 64 | sizeOffset = vectorNormalize(sf::Vector2f(-renderDirection.y, renderDirection.x)); 65 | 66 | if (p < _curves[c]._points.size() - 2) { 67 | sf::Vector2f renderPointNextNext = sf::Vector2f(origin.x + (_curves[c]._points[p + 2]._position.x - domain.x) / (domain.y - domain.x) * plotSize.x, 68 | origin.y - (_curves[c]._points[p + 2]._position.y - range.x) / (range.y - range.x) * plotSize.y); 69 | 70 | sf::Vector2f averageDirection = (renderDirection + vectorNormalize(renderPointNextNext - renderPointNext)) * 0.5f; 71 | 72 | sizeOffsetNext = vectorNormalize(sf::Vector2f(-averageDirection.y, averageDirection.x)); 73 | } 74 | else 75 | sizeOffsetNext = vectorNormalize(sf::Vector2f(-renderDirection.y, renderDirection.x)); 76 | 77 | sf::Vector2f perpendicular = vectorNormalize(sf::Vector2f(-renderDirection.y, renderDirection.x)); 78 | 79 | sizeOffset *= 1.0f / vectorDot(perpendicular, sizeOffset) * lineSize * 0.5f; 80 | sizeOffsetNext *= 1.0f / vectorDot(perpendicular, sizeOffsetNext) * lineSize * 0.5f; 81 | 82 | vertexArray[index].position = renderPoint - sizeOffset; 83 | vertexArray[index].texCoords = sf::Vector2f(0.0f, 0.0f); 84 | vertexArray[index].color = point._color; 85 | 86 | index++; 87 | 88 | vertexArray[index].position = renderPointNext - sizeOffsetNext; 89 | vertexArray[index].texCoords = sf::Vector2f(0.0f, 0.0f); 90 | vertexArray[index].color = pointNext._color; 91 | 92 | index++; 93 | 94 | vertexArray[index].position = renderPointNext + sizeOffsetNext; 95 | vertexArray[index].texCoords = sf::Vector2f(0.0f, lineGradientTexture.getSize().y); 96 | vertexArray[index].color = pointNext._color; 97 | 98 | index++; 99 | 100 | vertexArray[index].position = renderPoint - sizeOffset; 101 | vertexArray[index].texCoords = sf::Vector2f(0.0f, 0.0f); 102 | vertexArray[index].color = point._color; 103 | 104 | index++; 105 | 106 | vertexArray[index].position = renderPointNext + sizeOffsetNext; 107 | vertexArray[index].texCoords = sf::Vector2f(0.0f, lineGradientTexture.getSize().y); 108 | vertexArray[index].color = pointNext._color; 109 | 110 | index++; 111 | 112 | vertexArray[index].position = renderPoint + sizeOffset; 113 | vertexArray[index].texCoords = sf::Vector2f(0.0f, lineGradientTexture.getSize().y); 114 | vertexArray[index].color = point._color; 115 | 116 | index++; 117 | } 118 | } 119 | 120 | vertexArray.resize(index); 121 | 122 | vertexArray.setPrimitiveType(sf::PrimitiveType::Triangles); 123 | 124 | if (_curves[c]._shadow != 0.0f) { 125 | sf::VertexArray shadowArray = vertexArray; 126 | 127 | for (int v = 0; v < shadowArray.getVertexCount(); v++) { 128 | shadowArray[v].position += _curves[c]._shadowOffset; 129 | shadowArray[v].color = sf::Color(0, 0, 0, _curves[c]._shadow * 255.0f); 130 | } 131 | 132 | target.draw(shadowArray, &lineGradientTexture); 133 | } 134 | 135 | target.draw(vertexArray, &lineGradientTexture); 136 | } 137 | 138 | // Mask off parts of the curve that go beyond bounds 139 | sf::RectangleShape leftMask; 140 | leftMask.setSize(sf::Vector2f(margins.x, target.getSize().y)); 141 | leftMask.setFillColor(_backgroundColor); 142 | 143 | target.draw(leftMask); 144 | 145 | sf::RectangleShape rightMask; 146 | rightMask.setSize(sf::Vector2f(target.getSize().x, margins.y)); 147 | rightMask.setPosition(sf::Vector2f(0.0f, target.getSize().y - margins.y)); 148 | rightMask.setFillColor(_backgroundColor); 149 | 150 | target.draw(rightMask); 151 | 152 | // Draw axes 153 | sf::RectangleShape xAxis; 154 | xAxis.setSize(sf::Vector2f(plotSize.x + axesSize * 0.5f, axesSize)); 155 | xAxis.setPosition(sf::Vector2f(origin.x - axesSize * 0.5f, origin.y - axesSize * 0.5f)); 156 | xAxis.setFillColor(_axesColor); 157 | 158 | target.draw(xAxis); 159 | 160 | sf::RectangleShape yAxis; 161 | yAxis.setSize(sf::Vector2f(axesSize, plotSize.y + axesSize * 0.5f)); 162 | yAxis.setPosition(sf::Vector2f(origin.x - axesSize * 0.5f, origin.y - axesSize * 0.5f - plotSize.y)); 163 | yAxis.setFillColor(_axesColor); 164 | 165 | target.draw(yAxis); 166 | 167 | // Draw ticks 168 | { 169 | float xDistance = domain.y - domain.x; 170 | int xTicks = std::floor(xDistance / tickIncrements.x); 171 | float xTickOffset = 0.0f;// std::fmod(domain.x, tickIncrements.x); 172 | 173 | if (xTickOffset < 0.0f) 174 | xTickOffset += tickIncrements.x; 175 | 176 | float xTickRenderOffset = xTickOffset / xDistance; 177 | 178 | float xTickRenderDistance = tickIncrements.x / xDistance * plotSize.x; 179 | 180 | std::ostringstream os; 181 | 182 | os.precision(precision); 183 | 184 | for (int t = 0; t < xTicks; t++) { 185 | sf::RectangleShape xTick; 186 | xTick.setSize(sf::Vector2f(axesSize, tickLength)); 187 | xTick.setPosition(sf::Vector2f(origin.x + xTickRenderOffset + xTickRenderDistance * t - tickSize * 0.5f, origin.y)); 188 | xTick.setFillColor(_axesColor); 189 | 190 | target.draw(xTick); 191 | 192 | float value = domain.x + xTickOffset + t * tickIncrements.x; 193 | 194 | os.str(""); 195 | os << value; 196 | 197 | sf::Text xTickText; 198 | xTickText.setString(os.str()); 199 | xTickText.setFont(tickFont); 200 | xTickText.setPosition(sf::Vector2f(xTick.getPosition().x, xTick.getPosition().y + tickLength + textTickOffset)); 201 | xTickText.setRotation(45.0f); 202 | xTickText.setColor(_axesColor); 203 | xTickText.setScale(sf::Vector2f(tickTextScale, tickTextScale)); 204 | 205 | target.draw(xTickText); 206 | } 207 | } 208 | 209 | { 210 | float yDistance = range.y - range.x; 211 | int yTicks = std::floor(yDistance / tickIncrements.y); 212 | float yTickOffset = 0.0f;// std::fmod(range.x, tickIncrements.y); 213 | 214 | if (yTickOffset < 0.0f) 215 | yTickOffset += tickIncrements.y; 216 | 217 | float yTickRenderOffset = yTickOffset / yDistance; 218 | 219 | float yTickRenderDistance = tickIncrements.y / yDistance * plotSize.y; 220 | 221 | std::ostringstream os; 222 | 223 | os.precision(precision); 224 | 225 | for (int t = 0; t < yTicks; t++) { 226 | sf::RectangleShape yTick; 227 | yTick.setSize(sf::Vector2f(tickLength, axesSize)); 228 | yTick.setPosition(sf::Vector2f(origin.x - tickLength, origin.y - yTickRenderOffset - yTickRenderDistance * t - tickSize * 0.5f)); 229 | yTick.setFillColor(_axesColor); 230 | 231 | target.draw(yTick); 232 | 233 | float value = range.x + yTickOffset + t * tickIncrements.y; 234 | 235 | os.str(""); 236 | os << value; 237 | 238 | sf::Text yTickText; 239 | yTickText.setString(os.str()); 240 | yTickText.setFont(tickFont); 241 | sf::FloatRect bounds = yTickText.getLocalBounds(); 242 | yTickText.setPosition(sf::Vector2f(yTick.getPosition().x - bounds.width * 0.5f - tickLength * 0.5f - textTickOffset, yTick.getPosition().y - bounds.height * 0.5f)); 243 | yTickText.setRotation(0.0f); 244 | yTickText.setColor(_axesColor); 245 | yTickText.setScale(sf::Vector2f(tickTextScale, tickTextScale)); 246 | 247 | target.draw(yTickText); 248 | } 249 | } 250 | } 251 | 252 | float vis::vectorMagnitude(const sf::Vector2f &vector) { 253 | return std::sqrt(vector.x * vector.x + vector.y * vector.y); 254 | } 255 | 256 | sf::Vector2f vis::vectorNormalize(const sf::Vector2f &vector) { 257 | float magnitude = vectorMagnitude(vector); 258 | 259 | return vector / magnitude; 260 | } 261 | 262 | float vis::vectorDot(const sf::Vector2f &left, const sf::Vector2f &right) { 263 | return left.x * right.x + left.y * right.y; 264 | } --------------------------------------------------------------------------------