├── .appveyor.yml ├── .github └── ISSUE_TEMPLATE │ ├── Bug_report.md │ └── Feature_request.md ├── .gitignore ├── .npmignore ├── .travis.yml ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE ├── PULL_REQUEST_TEMPLATE.md ├── README.md ├── docs ├── fnn │ ├── bnn.md │ ├── dnn.md │ └── net.md ├── graph.md ├── mat-ops.md ├── mat.md ├── rnn │ ├── lstm.md │ └── rnn.md └── utils.md ├── jasmine.json ├── package-lock.json ├── package.json ├── src ├── fnn │ ├── ann.ts │ ├── bnn.spec.ts │ ├── bnn.ts │ ├── dnn.spec.ts │ ├── dnn.ts │ ├── fnn-model.spec.ts │ ├── fnn-model.ts │ └── utils │ │ ├── sample.ts │ │ └── training-set.ts ├── graph.spec.ts ├── graph.ts ├── index.ts ├── mat.spec.ts ├── mat.ts ├── net.ts ├── rand-mat.spec.ts ├── rand-mat.ts ├── rnn │ ├── lstm.spec.ts │ ├── lstm.ts │ ├── rnn-model.spec.ts │ ├── rnn-model.ts │ ├── rnn.spec.ts │ └── rnn.ts ├── solver.ts ├── test-examples.spec.ts ├── utils.spec.ts ├── utils.ts └── utils │ ├── assertable.spec.ts │ ├── assertable.ts │ ├── inner-state.ts │ ├── mat-ops.spec.ts │ ├── mat-ops.ts │ └── net-opts.ts ├── tsconfig.json ├── tslint-google.json └── tslint.json /.appveyor.yml: -------------------------------------------------------------------------------- 1 | platform: 2 | - x86 3 | - x64 4 | # Install scripts. (runs after repo cloning) 5 | install: 6 | # Get the latest stable version of Node 0.STABLE.latest 7 | - ps: Install-Product node $env:nodejs_version 8 | # Output useful info for debugging. 9 | - node --version 10 | - npm --version 11 | # Typical npm stuff. 12 | - npm install 13 | - npm test 14 | # Post-install test scripts. 15 | # test_script: 16 | # # Output useful info for debugging. 17 | # - node --version 18 | # - npm --version 19 | build: off 20 | cache: node_modules 21 | matrix: 22 | fast_finish: true 23 | environment: 24 | # - my_var1: value1 25 | matrix: 26 | - nodejs_version: "10.0.0" # 27 | - nodejs_version: "9.11.1" # same results as 8.10.x, 8.11.x, 9.0.x, 9.1.x, 9.2.x, 9.3.x, 9.4.x, 9.5.x, 9.6.x, 9.7.x, 9.8.x, 9.9.x, 9.10.x, 9.11.x 28 | - nodejs_version: "8.9.4" # same results as 8.7.x, 8.8.x, 8.9.x 29 | - nodejs_version: "8.6.0" # same results as 8.3.x, 8.4.x, 8.5.x 30 | - nodejs_version: "8.2.1" # same results as 8.0.x, 8.1.x, 8.2.x 31 | - nodejs_version: "7.10.1" # same results as 7.6.x, 7.7.x, 7.8.x, 7.9.x, 7.10.x 32 | - nodejs_version: "7.5.0" # same results as 7.1.x, 7.2.x, 7.3.x, 7.4.x 33 | - nodejs_version: "6.12.3" # same results as 6.5.x, 6.6.x, 6.7.x, 6.8.x, 6.9.x, 6.10.x, 6.11.x, 6.12.x 34 | - nodejs_version: "6.4.0" # same results as 6.0.x, 6.1.x, 6.2.x, 6.3.x 35 | skip_commits: 36 | files: 37 | - .github/* 38 | - docs/* 39 | - examples/* 40 | - .gitignore 41 | - .npmignore 42 | - .travis.yml 43 | - CODE_OF_CONDUCT.md 44 | - LICENSE 45 | - CONTRIBUTING.md 46 | - PULL_REQUEST_TEMPLATE.md 47 | - README.md 48 | - TODO.md 49 | - tslint* -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | 5 | --- 6 | 7 | **Describe the bug** 8 | A clear and concise description of what the bug is. 9 | 10 | **To Reproduce** 11 | Steps to reproduce the behavior: 12 | 1. Go to '...' 13 | 2. Click on '....' 14 | 3. Scroll down to '....' 15 | 4. See error 16 | 17 | **Expected behavior** 18 | A clear and concise description of what you expected to happen. 19 | 20 | **Screenshots** 21 | If applicable, add screenshots to help explain your problem. 22 | 23 | **Desktop (please complete the following information):** 24 | - OS: [e.g. iOS] 25 | - Browser [e.g. chrome, safari] 26 | - Version [e.g. 22] 27 | 28 | **Smartphone (please complete the following information):** 29 | - Device: [e.g. iPhone6] 30 | - OS: [e.g. iOS8.1] 31 | - Browser [e.g. stock browser, safari] 32 | - Version [e.g. 22] 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/Feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | 5 | --- 6 | 7 | **Is your feature request related to a problem? Please describe.** 8 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 9 | 10 | **Describe the solution you'd like** 11 | A clear and concise description of what you want to happen. 12 | 13 | **Describe alternatives you've considered** 14 | A clear and concise description of any alternative solutions or features you've considered. 15 | 16 | **Additional context** 17 | Add any other context or screenshots about the feature request here. 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Directories 2 | .vscode 3 | coverage 4 | dist 5 | logs 6 | node_modules 7 | test 8 | 9 | # Config Files 10 | 11 | # Other Files 12 | *.js 13 | *.js.map 14 | *.d.ts 15 | **/*.js 16 | **/*.js.map 17 | **/*.d.ts 18 | *.log* 19 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | # Directories 2 | .github 3 | node_modules 4 | examples 5 | docs 6 | src 7 | 8 | # Config Files 9 | .appveyor.yml 10 | .travis.yml 11 | CODE_OF_CONDUCT.md 12 | CONTRIBUTING.md 13 | jasmine.json 14 | PULL_REQUEST_TEMPLATE.md 15 | tsconfig.json 16 | tslint-google.json 17 | tslint.json 18 | 19 | # Other Files 20 | TODO.md 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: # Test only current versions of NodeJS >= 8.0 (Mid 2017) to reduce server loads 4 | - stable 5 | - "10.0.0" # 6 | - "9.11.1" # same results as 8.10, 8.11, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9, 9.10., 9.11 7 | - "8.9.4" # same results as 8.7, 8.8, 8.9 8 | - "8.6.0" # same results as 8.3, 8.4, 8.5 9 | - "8.2.1" # same results as 8.0, 8.1, 8.2 10 | # - "7.10.1" # same results as 7.6, 7.7, 7.8, 7.9, 7.10 11 | # - "7.5.0" # same results as 7.1, 7.2, 7.3, 7.4 12 | # - "6.12.3" # same results as 6.5, 6.6, 6.7, 6.8, 6.9, 6.10, 6.11, 6.12 13 | # - "6.4.0" # same results as 6.0, 6.1, 6.2, 6.3 14 | script: 15 | - npm run test 16 | matrix: 17 | fast_finish: true 18 | include: # Test additional versions in Matrix NodeJS < 8.0 && >= 6.0 19 | - node_js: "node" 20 | env: PRETEST=true 21 | # - node_js: "10.0.0" # 22 | # env: TEST=true ALLOW_FAILURE=true 23 | # - node_js: "9.11.1" # same results as 8.10, 8.11, 9.0, 9.1, 9.2, 9.3, 9.4, 9.5, 9.6, 9.7, 9.8, 9.9, 9.10., 9.11 24 | # env: TEST=true ALLOW_FAILURE=true 25 | # - node_js: "8.9.4" # same results as 8.7, 8.8, 8.9 26 | # env: TEST=true ALLOW_FAILURE=true 27 | # - node_js: "8.6.0" # same results as 8.3, 8.4, 8.5 28 | # env: TEST=true ALLOW_FAILURE=true 29 | # - node_js: "8.2.1" # same results as 8.0, 8.1, 8.2 30 | # env: TEST=true ALLOW_FAILURE=true 31 | - node_js: "7.10.1" # same results as 7.6, 7.7, 7.8, 7.9, 7.10 32 | env: TEST=true ALLOW_FAILURE=true 33 | - node_js: "7.5.0" # same results as 7.1, 7.2, 7.3, 7.4 34 | env: TEST=true ALLOW_FAILURE=true 35 | - node_js: "6.12.3" # same results as 6.5, 6.6, 6.7, 6.8, 6.9, 6.10, 6.11, 6.12 36 | env: TEST=true ALLOW_FAILURE=true 37 | - node_js: "6.4.0" # same results as 6.0, 6.1, 6.2, 6.3 38 | env: TEST=true ALLOW_FAILURE=true 39 | branches: 40 | only: 41 | - master 42 | os: 43 | - linux 44 | - osx 45 | cache: 46 | directories: 47 | - node_modules 48 | after_success: 49 | # - npm run report-coverage 50 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | ## 1. Purpose 4 | 5 | A primary goal of recurrent-js is to be inclusive to the largest number of contributors, with the most varied and diverse backgrounds possible. As such, we are committed to providing a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion (or lack thereof). 6 | 7 | This code of conduct outlines our expectations for all those who participate in our community, as well as the consequences for unacceptable behavior. 8 | 9 | We invite all those who participate in recurrent-js to help us create safe and positive experiences for everyone. 10 | 11 | ## 2. Open Source Citizenship 12 | 13 | A supplemental goal of this Code of Conduct is to increase open source citizenship by encouraging participants to recognize and strengthen the relationships between our actions and their effects on our community. 14 | 15 | Communities mirror the societies in which they exist and positive action is essential to counteract the many forms of inequality and abuses of power that exist in society. 16 | 17 | If you see someone who is making an extra effort to ensure our community is welcoming, friendly, and encourages all participants to contribute to the fullest extent, we want to know. 18 | 19 | ## 3. Expected Behavior 20 | 21 | The following behaviors are expected and requested of all community members: 22 | 23 | * Participate in an authentic and active way. In doing so, you contribute to the health and longevity of this community. 24 | * Exercise consideration and respect in your speech and actions. 25 | * Attempt collaboration before conflict. 26 | * Refrain from demeaning, discriminatory, or harassing behavior and speech. 27 | * Be mindful of your surroundings and of your fellow participants. Alert community leaders if you notice a dangerous situation, someone in distress, or violations of this Code of Conduct, even if they seem inconsequential. 28 | * Remember that community event venues may be shared with members of the public; please be respectful to all patrons of these locations. 29 | 30 | ## 4. Unacceptable Behavior 31 | 32 | The following behaviors are considered harassment and are unacceptable within our community: 33 | 34 | * Violence, threats of violence or violent language directed against another person. 35 | * Sexist, racist, homophobic, transphobic, ableist or otherwise discriminatory jokes and language. 36 | * Posting or displaying sexually explicit or violent material. 37 | * Posting or threatening to post other people’s personally identifying information ("doxing"). 38 | * Personal insults, particularly those related to gender, sexual orientation, race, religion, or disability. 39 | * Inappropriate photography or recording. 40 | * Inappropriate physical contact. You should have someone’s consent before touching them. 41 | * Unwelcome sexual attention. This includes, sexualized comments or jokes; inappropriate touching, groping, and unwelcomed sexual advances. 42 | * Deliberate intimidation, stalking or following (online or in person). 43 | * Advocating for, or encouraging, any of the above behavior. 44 | * Sustained disruption of community events, including talks and presentations. 45 | 46 | ## 5. Consequences of Unacceptable Behavior 47 | 48 | Unacceptable behavior from any community member, including sponsors and those with decision-making authority, will not be tolerated. 49 | 50 | Anyone asked to stop unacceptable behavior is expected to comply immediately. 51 | 52 | If a community member engages in unacceptable behavior, the community organizers may take any action they deem appropriate, up to and including a temporary ban or permanent expulsion from the community without warning (and without refund in the case of a paid event). 53 | 54 | ## 6. Reporting Guidelines 55 | 56 | If you are subject to or witness unacceptable behavior, or have any other concerns, please notify a community organizer as soon as possible. [mvrahden](mailto:menno.vrahden@gmail.com). 57 | 58 | 59 | 60 | Additionally, community organizers are available to help community members engage with local law enforcement or to otherwise help those experiencing unacceptable behavior feel safe. In the context of in-person events, organizers will also provide escorts as desired by the person experiencing distress. 61 | 62 | ## 7. Addressing Grievances 63 | 64 | If you feel you have been falsely or unfairly accused of violating this Code of Conduct, you should notify mvahden with a concise description of your grievance. Your grievance will be handled in accordance with our existing governing policies. 65 | 66 | 67 | 68 | ## 8. Scope 69 | 70 | We expect all community participants (contributors, paid or otherwise; sponsors; and other guests) to abide by this Code of Conduct in all community venues–online and in-person–as well as in all one-on-one communications pertaining to community business. 71 | 72 | This code of conduct and its related procedures also applies to unacceptable behavior occurring outside the scope of community activities when such behavior has the potential to adversely affect the safety and well-being of community members. 73 | 74 | ## 9. Contact info 75 | 76 | [mvrahden](mailto:menno.vrahden@gmail.com) 77 | 78 | ## 10. License and attribution 79 | 80 | This Code of Conduct is distributed under a [Creative Commons Attribution-ShareAlike license](http://creativecommons.org/licenses/by-sa/3.0/). 81 | 82 | Portions of text derived from the [Django Code of Conduct](https://www.djangoproject.com/conduct/) and the [Geek Feminism Anti-Harassment Policy](http://geekfeminism.wikia.com/wiki/Conference_anti-harassment/Policy). 83 | 84 | Retrieved on November 22, 2016 from [http://citizencodeofconduct.org/](http://citizencodeofconduct.org/) 85 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thanks for taking the time to contribute to recurrent-js. Follow these guidelines to make the process smoother: 4 | 5 | 1. One feature per pull request. 6 | Each PR should have one focus, and all the code changes should be supporting that one feature or bug fix. Using a [separate branch](https://guides.github.com/introduction/flow/index.html) for each feature should help you manage developing multiple features at once. 7 | 8 | 2. Follow the style of the file when it comes to syntax like curly braces and indents. 9 | 10 | 3. Add a test for the feature or fix, if possible. See the `.spec` files for existing tests and README describing how to run these tests. 11 | 12 | This Contributing guidelines were aligned to those of the [brain.js](https://github.com/BrainJS/brain.js) project. Thanks! 13 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2015 Andrej Karpathy, 2017 Menno van Rahden 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ![A GIF or MEME to give some spice of the internet](url) 5 | 6 | ## Description 7 | 8 | 9 | ## Motivation and Context 10 | 11 | 12 | [issue](https://github.com/BrainJS/brain.js/issues/###) 13 | 14 | ## How Has This Been Tested? 15 | 16 | 17 | 18 | 19 | ## Screenshots (if appropriate): 20 | 21 | ## Types of changes 22 | 23 | - [ ] Bug fix (non-breaking change which fixes an issue) 24 | - [ ] New feature (non-breaking change which adds functionality) 25 | - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) 26 | 27 | ## Author's Checklist: 28 | 29 | 30 | - [ ] My code focuses on the main motivation and avoids scope creep. 31 | - [ ] My code passes current tests and adds new tests where possible. 32 | - [ ] My code is [SOLID](https://en.wikipedia.org/wiki/SOLID_(object-oriented_design)) and [DRY](https://en.wikipedia.org/wiki/Don%27t_repeat_yourself). 33 | - [ ] I have updated the documentation as needed. 34 | 35 | ## Reviewer's Checklist: 36 | - [ ] I kept my comments to the author positive, specific, and productive. 37 | - [ ] I tested the code and didn't find any new problems. 38 | - [ ] I think the motivation is good for the project. 39 | - [ ] I think the code works to satisfies the motivation. 40 | 41 | 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # recurrent-js 2 | [![Build Status](https://travis-ci.org/mvrahden/recurrent-js.svg?branch=master)](https://travis-ci.org/mvrahden/recurrent-js) 3 | [![Build status](https://ci.appveyor.com/api/projects/status/7qkcof8t6b0io44f/branch/master?svg=true)](https://ci.appveyor.com/project/mvrahden/recurrent-js/branch/master) 4 | [![js-google-style](https://img.shields.io/badge/code%20style-google-blue.svg)](https://google.github.io/styleguide/jsguide.html) 5 | [![dependency-free](https://img.shields.io/badge/dependencies-none-brightgreen.svg)]() 6 | 7 | [docs-utils]: https://github.com/mvrahden/recurrent-js/blob/master/docs/utils.md 8 | [docs-mat]: https://github.com/mvrahden/recurrent-js/blob/master/docs/mat.md 9 | [docs-mat-ops]: https://github.com/mvrahden/recurrent-js/blob/master/docs/mat-ops.md 10 | [docs-graph]: https://github.com/mvrahden/recurrent-js/blob/master/docs/graph.md 11 | [docs-net]: https://github.com/mvrahden/recurrent-js/blob/master/docs/fnn/net.md 12 | [docs-dnn]: https://github.com/mvrahden/recurrent-js/blob/master/docs/fnn/dnn.md 13 | [docs-bnn]: https://github.com/mvrahden/recurrent-js/blob/master/docs/fnn/bnn.md 14 | [docs-rnn]: https://github.com/mvrahden/recurrent-js/blob/master/docs/rnn/rnn.md 15 | [docs-lstm]: https://github.com/mvrahden/recurrent-js/blob/master/docs/rnn/lstm.md 16 | 17 | **Call For Volunteers:** Due to my lack of time, I'm desperately looking for voluntary help. Should you be interested in the training of neural networks (even though you're a newbie) and willing to develop this educational project a little further, please contact me :) There are some points on the agenda, that I'd still like to see implemented to make this project a nice library for abstract educational purposes. 18 | 19 | > INACTIVE: Due to lack of time and help 20 | 21 | **The recurrent-js library** – Various amazingly simple to build and train neural network architectures. This Library is for **educational purposes** only. The library is an object-oriented neural network approach (baked with [Typescript](https://github.com/Microsoft/TypeScript)), containing stateless and stateful neural network architectures. It is a redesigned and extended version of _Andrej Karpathy's_ RecurrentJS library that implements the following: 22 | 23 | * Vanilla Feedforward Neural Network (Net) 24 | * Deep **Recurrent Neural Networks** (RNN) 25 | * Deep **Long Short-Term Memory** Networks (LSTM) 26 | * **Bonus #1**: Deep **Feedforward Neural Networks** (DNN) 27 | * **Bonus #2**: Deep **Bayesian Neural Networks** (BNN) 28 | * In fact, the library is more general because it has functionality to construct arbitrary **expression graphs** over which the library can perform **automatic differentiation** similar to what you may find in Theano for Python, or in Torch etc. Currently, the code uses this very general functionality to implement RNN/LSTM, but one can build arbitrary Neural Networks and do automatic backprop. 29 | 30 | ## For Production Use 31 | 32 | ### What does the Library has to offer? 33 | 34 | The following sections provide an overview of the available Classes and Interfaces. 35 | The class names are linked to more detailed descriptions of the specific classes. 36 | 37 | #### Utility Classes: 38 | 39 | * **[Utils][docs-utils]** - Collection of Utility functions: Array creation & manipulation, Statistical evaluation methods etc. 40 | * **[Mat][docs-mat]** - Matrix Class holding weights and their derivatives for the neural networks. 41 | * **RandMat** - A convenient subclass of `Mat`. `RandMat` objects are automatically populated with random values on their creation. 42 | * **[MatOps][docs-mat-ops]** - Class with matrix operations (add, multiply, sigmoid etc.) and their respective derivative functions. 43 | * **[Graph][docs-graph]** - Graph memorizing the sequences of matrix operations and matching their respective derivative functions for backpropagation. 44 | * **NetOpts** - Standardized `Interface` for the initial configuration of all Neural Networks. 45 | 49 | * **InnerState** - Standardized `Interface` for stateful networks memorizing the previous state of activations. 50 | 51 | #### Neural Network Classes: 52 | * stateless: 53 | * **[Net][docs-net]** - shallow Vanilla Feedforward Neural Network (1 hidden layer). 54 | * **[DNN][docs-dnn]** - Deep Feedforward Neural Network. 55 | * **[BNN][docs-bnn]** - Deep Bayesian Neural Network. 56 | * stateful (*Still old API!*): 57 | * **[RNN][docs-rnn]** - Deep Recurrent Neural Network. 58 | * **[LSTM][docs-lstm]** - Long Short Term Memory Network. 59 | 60 | ### How to install as dependency 61 | 62 | Download available `@npm`: [recurrent-js](https://www.npmjs.com/package/recurrent-js) 63 | 64 | Install via command line: 65 | ``` 66 | npm install --save recurrent-js@latest 67 | ``` 68 | 69 | The project directly ships with the transpiled Javascript code. 70 | For TypeScript development it also contains Map-files and Declaration-files. 71 | 72 | ### How to import? 73 | 74 | The aforementioned classes can be imported from this `npm` module, e.g.: 75 | ```typescript 76 | import { NetOpts, DNN } from 'recurrent-js'; 77 | ``` 78 | 79 | For JavaScript usage `require` classes from this `npm` module as follows: 80 | ```javascript 81 | // NetOpts is an interface (Typescript only), but it gives clues about the required Object-properties (keys) 82 | const DNN = require('recurrent-js').DNN; 83 | ``` 84 | 85 | ### How to train? 86 | 87 | Training of neural networks is achieved by iteratively reinforcing wanted neural activations or by suppressing unwanted activation paths through adjusting their respective slopes. 88 | The training is achieved via an expression `Graph`, which memorizes the sequence of matrix operations being executed during the forward-pass operation of a neural network. 89 | The results of the Matrix operations are contained in `Mat`-objects, which contain the resulting values (`w`) and their corresponding derivatives (`dw`). 90 | The `Graph`-object can be used to calculate the resulting gradient and propagate a loss value back into the memorized sequence of matrix operations. 91 | The update of the weights of the neural connections will then lead to supporting wanted neural network activity and suppressing unwanted activation behavior. 92 | The described backpropagation can be achieved as follows: 93 | 94 | ```typescript 95 | import { Graph, DNN } from 'recurrent-js'; 96 | 97 | /* define network structure configuration */ 98 | const netOpts = { 99 | architecture: { inputSize: 2, hiddenUnits: [2, 3], outputSize: 3 }, 100 | training: { loss: 1e-11 } 101 | }; 102 | 103 | /* instantiate network */ 104 | const net = new DNN(netOpts); 105 | 106 | /* make it trainable */ 107 | net.setTrainability(true); 108 | 109 | /** 110 | * Perform an iterative training by first forward passing an input 111 | * and second backward propagating the according target output. 112 | * You'll receive the squared loss, that gives you a hint of the networks 113 | * approximation quality. 114 | * Repeat this action until the quality of the output of the forward pass 115 | * suits your needs, or the mean squared error is small enough, e.g. < 1. 116 | */ 117 | do { 118 | const someInput = [0, 1]; /* an array of intput values */ 119 | const someExpectedOutput = [0, 1, 0]; /* an array of target output */ 120 | 121 | const someOutput = net.forward(someInput); 122 | 123 | net.backward(someExpectedOutput /* , alpha?: number */); 124 | const squaredLoss = net.getSquaredLoss(someInput, someExpectedOutput); 125 | } while(squaredLoss > 0.1); 126 | /** 127 | * --> Keep in mind: you actually want a low MEAN squaredLoss, this is 128 | * left out in this example, to keep the focus on the important parts 129 | */ 130 | 131 | ``` 132 | **HINT #1**: providing an additional *custom learning rate* (`alpha`) for the backpropagation can accelerate the training. For further info please consult the respective`test-examples.spec.ts` file. 133 | 134 | **HINT #2**: The *Recurrent Neural Network Architectures* (RNN, LSTM) are not yet updated to this new training API. Due to my current lack of time, this likely won't change for a while... (unless this repo gets some voluntary help). Please consult the README of the [commit v.1.6.2](https://github.com/mvrahden/recurrent-js/tree/4065e644a36a26ae31598070dd0197008fe1a88b) for the details of the former training style. Thanks! 135 | 136 | Should you want to get some deeper insights on "how to train the network", it is recommendable to have a look into the source of the DQN-Solver from the [reinforce-js](https://github.com/mvrahden/reinforce-js) library (`learnFromSarsaTuple`-Method). 137 | 138 | ## Example Applications 139 | 140 | This project is an integral part of the `reinforce-js` library. 141 | As such it is vividly demonstrated in the `learning-agents` model. 142 | 143 | - [learning-agents](https://mvrahden.github.io/learning-agents) (GitHub Page) 144 | - [reinforce-js](https://github.com/mvrahden/reinforce-js) (GitHub Repository) 145 | 146 | 147 | ## Community Contribution 148 | 149 | Everybody is more than welcome to contribute and extend the functionality! 150 | 151 | Please feel free to contribute to this project as much as you wish to. 152 | 153 | 1. clone from GitHub via `git clone https://github.com/mvrahden/recurrent-js.git` 154 | 2. `cd` into the directory and `npm install` for initialization 155 | 3. Try to `npm run test`. If everything is green, you're ready to go :sunglasses: 156 | 157 | Before triggering a pull-request, please make sure that you've run all the tests via the *testing command*: 158 | 159 | ``` 160 | npm run test 161 | ``` 162 | 163 | This project relies on Visual Studio Codes built-in Typescript linting facilities. It primarily follows the [Google TypeScript Style-Guide](https://github.com/google/ts-style) through the provided *tslint-google.json* configuration file. 164 | 165 | ## License 166 | 167 | As of License-File: [MIT](LICENSE) 168 | -------------------------------------------------------------------------------- /docs/fnn/bnn.md: -------------------------------------------------------------------------------- 1 | # Class: `BNN` 2 | 3 | to be continued... 4 | -------------------------------------------------------------------------------- /docs/fnn/dnn.md: -------------------------------------------------------------------------------- 1 | # Class: `DNN` 2 | 3 | to be continued... 4 | -------------------------------------------------------------------------------- /docs/fnn/net.md: -------------------------------------------------------------------------------- 1 | # Class: `Net` 2 | 3 | to be continued... 4 | -------------------------------------------------------------------------------- /docs/graph.md: -------------------------------------------------------------------------------- 1 | # Class: `Graph` 2 | 3 | The Graph class is important for the neural networks in a way that it keeps track of the relations between the matrices used in these networks. 4 | The following sections further describe the `Graph` class and its usage. 5 | 6 | ## Class Structure 7 | * Constructor: `Graph()` 8 | * Provided Matrix Operations: 9 | * Delegate the actual Matrix Operation call 10 | * Keeps protocol of the sequence of matrix operations 11 | * Each operation returns a new `Mat`-Object, containing the specific results. 12 | * Available Matrix Operations are: 13 | * `rowPluck(m: Mat, rowIndex: number): Mat` 14 | * `gauss(m: Mat, std: Mat): Mat` 15 | * `tanh(m: Mat): Mat` 16 | * `sig(m: Mat): Mat` 17 | * `relu(m: Mat): Mat` 18 | * `add(mat1: Mat, mat2: Mat): Mat` 19 | * `mul(mat1: Mat, mat2: Mat): Mat` 20 | * `dot(mat1: Mat, mat2: Mat): Mat` 21 | * `eltmul(mat1: Mat, mat2: Mat): Mat` 22 | * `memorizeOperationSequence(isMemorizing: boolean): void`: Switch, whether the graph should keep a protocol of the operation sequence for backpropagation 23 | * `isMemorizingSequence(): boolean`: Get current memorization state 24 | * `forgetCurrentSequence(): void`: Clear the graph memory 25 | * `backward(): void`: Calls the Backpropagation Stack in reverse (LIFO) order of Matrix Operation Derivatives. 26 | 27 | ## Usage 28 | 29 | ### Object creation 30 | 31 | Create a graph that does (or does not) memorize the sequence of Matrix Operations. 32 | 33 | ```typescript 34 | const graph = new Graph(); 35 | 36 | /* OPTIONAL: Set Backprop-state to `true` */ 37 | graph.memorizeOperationSequence(true); 38 | ``` 39 | 40 | ### Matrix Operation Call e.g. `sig(m: Mat): Mat` 41 | 42 | Create a graph and inject a `Mat` object to call a sigmoid operation on its respective elements. 43 | A graph object - with backpropagation enabled - memorizes the derivatives of the matrix operations in the order they have been called. 44 | 45 | ```typescript 46 | const graph = new Graph(); 47 | /* OPTIONAL: Set Backprop-state to `true` */ 48 | graph.memorizeOperationSequence(true); 49 | 50 | const mat = new Mat(4, 1); 51 | mat.setFrom([0.1, 0.3, 0.9, 0.4]); /* fill Matrix with values */ 52 | 53 | /* 54 | * - calls sigmoid operation on matrix, 55 | * - registers the matrix operation to the backpropagation stack (if activated) and 56 | * - returns a new matrix object with respective results 57 | */ 58 | const result = graph.sig(mat); 59 | ``` 60 | 61 | ### Call of the `backward(): void` Method 62 | 63 | After the execution of a sequence of matrix operations via a `Graph`-object, this graph is then able to execute the backpropagation process in reverse (LIFO) order. 64 | 65 | **Prerequisite:** The `Graph`-object needs to memorize the sequence of matrix operations. 66 | * Call `graph.memorizeOperationSequence(true);` for that 67 | * Check the memorization state of graph with `graph.isMemorizingSequence();` 68 | 69 | ```typescript 70 | const graph = new Graph(); 71 | 72 | const mat = new Mat(4, 1); 73 | mat.setFrom([0.1, 0.3, 0.9, 0.4]); /* fill Matrix with values */ 74 | 75 | /* IMPORTANT: Set Backprop-state to `true` to be able to memorize matrix operation sequence */ 76 | graph.memorizeOperationSequence(true); 77 | 78 | /* now: perform some sequence of matrix operations */ 79 | const result = graph.sig(mat); 80 | 81 | /* manipulate the derivatives of `result`-Matrix */ 82 | result.dw[1] = -0.8; 83 | 84 | /* propagate the modification back, by calling the memorized sequence of matrox operations */ 85 | graph.backward(); 86 | 87 | /* OPTIONAL: reset the graphs memory/protocol */ 88 | graph.forgetCurrentSequence(); 89 | ``` 90 | -------------------------------------------------------------------------------- /docs/mat-ops.md: -------------------------------------------------------------------------------- 1 | # Class: `MatOps` 2 | 3 | The class `MatOps` holds all the necessary matrix operations for assambling a forward-pass and their respective derivatives (`get...Backprop`) to perform backpropagation. 4 | All methods are static (stateindependent) methods. 5 | 6 | ## Class Structure 7 | 8 | * Each Matrix Operation: 9 | * executes the actual Matrix Operation 10 | * throws an Error Message if dimensions are not aligned 11 | * returns a new `Mat`-instance with resulting values 12 | * Each `get...Backprop`-Function: 13 | * returns a derivative Function, that keeps the **references** of the given inputs 14 | * Available Matrix Operations: 15 | * `[static] rowPluck(m: Mat, rowIndex: number): Mat` 16 | * `[static] getRowPluckBackprop(m: Mat, rowIndex: number, out: Mat): Mat` 17 | * `[static] gauss(m: Mat, std: Mat): Mat` 18 | * `[static] tanh(m: Mat): Mat` 19 | * `[static] sig(m: Mat): Mat` 20 | * `[static] relu(m: Mat): Mat` 21 | * `[static] add(mat1: Mat, mat2: Mat): Mat` 22 | * `[static] mul(mat1: Mat, mat2: Mat): Mat` 23 | * `[static] dot(mat1: Mat, mat2: Mat): Mat` 24 | * `[static] eltmul(mat1: Mat, mat2: Mat): Mat` 25 | * Available `get...Backprop`-Functions: 26 | * `[static] getTanhBackprop(m: Mat, out: Mat): Function` 27 | * `[static] getSigmoidBackprop(m: Mat, out: Mat): Function` 28 | * `[static] getReluBackprop(m: Mat, out: Mat): Function` 29 | * `[static] getAddBackprop(mat1: Mat, mat2: Mat, out: Mat): Function` 30 | * `[static] getMulBackprop(mat1: Mat, mat2: Mat, out: Mat): Function` 31 | * `[static] getDotBackprop(mat1: Mat, mat2: Mat, out: Mat): Function` 32 | * `[static] getEltmulBackprop(mat1: Mat, mat2: Mat, out: Mat): Function` 33 | 34 | ## Usage 35 | 36 | Import the `MatOps`-class and use the provided **static** methods. 37 | Each method comes with its respective description. 38 | 39 | ### Matrix Operation 40 | 41 | All Matrix Operations are **non-destructive**, meaning: they all return a new `Mat`-instance with the resulting values. 42 | 43 | ```typescript 44 | import { MatOps } from 'recurrent-js'; 45 | 46 | /* 47 | * const input = new Mat(1, 4); 48 | * input.setFrom([0, 1, 2, 3]); 49 | */ 50 | 51 | const mat = MatOps.tanh(input); 52 | ``` 53 | 54 | ### Backprop/Derivative-Functions 55 | 56 | ```typescript 57 | import { MatOps } from 'recurrent-js'; 58 | 59 | /* 60 | * const input = new Mat(1, 4); 61 | * input.setFrom([0, 1, 2, 3]); 62 | * 63 | * const mat = MatOps.tanh(input); 64 | */ 65 | 66 | /* 67 | * Inject: 68 | * - input: the input that lead to an output 69 | * - out: the corresponding output 70 | */ 71 | const tanhBackprop = MatOps.getTanhBackprop(input, mat); 72 | 73 | /* ready to perform backprop */ 74 | tanhBackprop(); 75 | ``` 76 | -------------------------------------------------------------------------------- /docs/mat.md: -------------------------------------------------------------------------------- 1 | # Class: `Mat` 2 | 3 | The Mat class is important for the neural networks as it holds the weights and their associated derivative values in form of matrix values. 4 | The following sections further describe the `Mat` and its usage. 5 | 6 | ## Class Structure 7 | * Constructor: `Mat(rows: number, cols: number)` 8 | * Available Properties: 9 | * `w`: 1d array, holding the actual values of the matrix 10 | * `dw`: 1d array, holding the derivatives of the values of a matrix after calling derivative operations 11 | * Administrative Methods: 12 | * `get(row: number, col: number): number` 13 | * `set(row: number, col: number, v: number): void` 14 | * `setFrom(arr: Array | Float64Array): void` 15 | 16 | * `setColumn(m: Mat, colIndex: number): void` 17 | * `equals(m: Mat): boolean` 18 | * `[static] toJSON(m: Mat | any): {rows, cols, w}` 19 | * `[static] fromJSON(m: {rows, cols, w}): Mat` 20 | * For Backpropagation: 21 | * `update(alpha: number): void` 22 | 23 | ## Usage 24 | 25 | ### Object creation 26 | 27 | Create a matrix with defined rows and columns. 28 | 29 | ```typescript 30 | const matrix = new Mat(4, 1); 31 | ``` 32 | 33 | ### Set Values from an Array | Float64Array 34 | 35 | When copying the values from an Array to the matrix object, the array needs to be of length `rows * cols` and have the values ordered according to the sequence of the rows. (e.g. `[ row1 row2 ... ]`) 36 | 37 | ```typescript 38 | matrix.setFrom([1, 4, 3, 2]); 39 | ``` 40 | 41 | ### Matrix Operation Call e.g. `sig(m: Mat): Mat` 42 | 43 | A `Mat` object executing a sigmoid operation on its respective elements. 44 | 45 | ```typescript 46 | const mat = new Mat(4, 1); 47 | 48 | mat.setFrom([0.1, 0.3, 0.9, 0.4]); 49 | 50 | const result = Mat.sig(mat); 51 | ``` 52 | 53 | ### Call of the `update(aplha: number): void` Method 54 | 55 | To train a neural network, the weights need to manipulated in a way. 56 | The `update()`-Method is a way to discount the values (kept in `w`) by the values of their respective derivatives (kept in `dw`). 57 | The underlying mechanism subtracts the discounted derivative value from the real value like: `w[i] = w[i] - (dq[i] * alpha)` 58 | 59 | ```typescript 60 | mat.update(0.01); 61 | ``` 62 | -------------------------------------------------------------------------------- /docs/rnn/lstm.md: -------------------------------------------------------------------------------- 1 | # Class: `LSTM` 2 | 3 | to be continued... 4 | -------------------------------------------------------------------------------- /docs/rnn/rnn.md: -------------------------------------------------------------------------------- 1 | # Class: `RNN` 2 | 3 | to be continued... 4 | -------------------------------------------------------------------------------- /docs/utils.md: -------------------------------------------------------------------------------- 1 | # Class: `Utils` 2 | 3 | The class `Utils` provides a collection of useful statistical tools, evaluation tools and array tools. 4 | 5 | ## Class Structure 6 | 7 | * Random Number Functions 8 | * `[static] randf(min: number, max: number): number` 9 | * `[static] randi(min: number, max: number): number` 10 | * `[static] randn(mu: number, std: number): number` 11 | * `[static] skewedRandn(mu: number, std: number, skew: number): number` 12 | * Statistic Tools 13 | * `[static] sum(arr: Array | Float64Array): number` 14 | * `[static] mean(arr: Array | Float64Array): number` 15 | * `[static] median(arr: Array | Float64Array): number` 16 | * `[static] mode(arr: Array | Float64Array, precision?: number): Array` 17 | * `[static] var(arr: Array | Float64Array, normalization?: 'uncorrected' | 'biased' | 'unbiased'): number` 18 | * `[static] std(arr: Array | Float64Array, normalization?: 'uncorrected' | 'biased' | 'unbiased'): number` 19 | * Array Filler 20 | * `[static] fillRandn(arr: Array | Float64Array, mu: number, std: number): void` 21 | * `[static] fillRand(arr: Array | Float64Array, min: number, max: number): void` 22 | * `[static] fillConst(arr: Array | Float64Array, c: number): void` 23 | * `[static] fillArray(n: number, val: number): Array | Float64Array` 24 | * Array Creation 25 | * `[static] zeros(n: number): Array | Float64Array` 26 | * `[static] ones(n: number): Array | Float64Array` 27 | * Output Functions 28 | * `[static] softmax(arr: Array | Float64Array): Array | Float64Array` 29 | * `[static] argmax(arr: Array | Float64Array): number` 30 | * `[static] sampleWeighted(arr: Array | Float64Array): number` 31 | 32 | ## Usage 33 | 34 | Import the `Utils`-class and use the provided **static** methods. 35 | Each method comes with its respective description. 36 | 37 | ```typescript 38 | import { Utils } from 'recurrent-js'; 39 | 40 | const randomFloat = Utils.randf(0, 10); 41 | const randomInt = Utils.randi(0, 10); 42 | const randomNormal = Utils.randi(0, 1); 43 | 44 | const sum = Utils.sum([1, 2, 4, 10]); 45 | ``` -------------------------------------------------------------------------------- /jasmine.json: -------------------------------------------------------------------------------- 1 | { 2 | "spec_dir": "dist", 3 | "spec_files": [ 4 | "**/*[sS]pec.js" 5 | ], 6 | "helpers": [ 7 | "helpers/**/*.js" 8 | ], 9 | "stopSpecOnExpectationFailure": false, 10 | "random": false 11 | } 12 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recurrent-js", 3 | "version": "1.7.4", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "@types/jasmine": { 8 | "version": "2.8.8", 9 | "resolved": "https://registry.npmjs.org/@types/jasmine/-/jasmine-2.8.8.tgz", 10 | "integrity": "sha512-OJSUxLaxXsjjhob2DBzqzgrkLmukM3+JMpRp0r0E4HTdT1nwDCWhaswjYxazPij6uOdzHCJfNbDjmQ1/rnNbCg==", 11 | "dev": true 12 | }, 13 | "balanced-match": { 14 | "version": "1.0.0", 15 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 16 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 17 | "dev": true 18 | }, 19 | "brace-expansion": { 20 | "version": "1.1.8", 21 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.8.tgz", 22 | "integrity": "sha1-wHshHHyVLsH479Uad+8NHTmQopI=", 23 | "dev": true, 24 | "requires": { 25 | "balanced-match": "^1.0.0", 26 | "concat-map": "0.0.1" 27 | } 28 | }, 29 | "concat-map": { 30 | "version": "0.0.1", 31 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 32 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 33 | "dev": true 34 | }, 35 | "exit": { 36 | "version": "0.1.2", 37 | "resolved": "https://registry.npmjs.org/exit/-/exit-0.1.2.tgz", 38 | "integrity": "sha1-BjJjj42HfMghB9MKD/8aF8uhzQw=", 39 | "dev": true 40 | }, 41 | "fs.realpath": { 42 | "version": "1.0.0", 43 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 44 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 45 | "dev": true 46 | }, 47 | "glob": { 48 | "version": "7.1.2", 49 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 50 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 51 | "dev": true, 52 | "requires": { 53 | "fs.realpath": "^1.0.0", 54 | "inflight": "^1.0.4", 55 | "inherits": "2", 56 | "minimatch": "^3.0.4", 57 | "once": "^1.3.0", 58 | "path-is-absolute": "^1.0.0" 59 | } 60 | }, 61 | "inflight": { 62 | "version": "1.0.6", 63 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 64 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 65 | "dev": true, 66 | "requires": { 67 | "once": "^1.3.0", 68 | "wrappy": "1" 69 | } 70 | }, 71 | "inherits": { 72 | "version": "2.0.3", 73 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 74 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 75 | "dev": true 76 | }, 77 | "jasmine": { 78 | "version": "2.99.0", 79 | "resolved": "https://registry.npmjs.org/jasmine/-/jasmine-2.99.0.tgz", 80 | "integrity": "sha1-jKctEC5jm4Z8ZImFbg4YqceqQrc=", 81 | "dev": true, 82 | "requires": { 83 | "exit": "^0.1.2", 84 | "glob": "^7.0.6", 85 | "jasmine-core": "~2.99.0" 86 | } 87 | }, 88 | "jasmine-core": { 89 | "version": "2.99.1", 90 | "resolved": "https://registry.npmjs.org/jasmine-core/-/jasmine-core-2.99.1.tgz", 91 | "integrity": "sha1-5kAN8ea1bhMLYcS80JPap/boyhU=", 92 | "dev": true 93 | }, 94 | "minimatch": { 95 | "version": "3.0.4", 96 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 97 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 98 | "dev": true, 99 | "requires": { 100 | "brace-expansion": "^1.1.7" 101 | } 102 | }, 103 | "once": { 104 | "version": "1.4.0", 105 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 106 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 107 | "dev": true, 108 | "requires": { 109 | "wrappy": "1" 110 | } 111 | }, 112 | "path-is-absolute": { 113 | "version": "1.0.1", 114 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 115 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 116 | "dev": true 117 | }, 118 | "rimraf": { 119 | "version": "2.6.2", 120 | "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.6.2.tgz", 121 | "integrity": "sha512-lreewLK/BlghmxtfH36YYVg1i8IAce4TI7oao75I1g245+6BctqTVQiBP3YUJ9C6DQOXJmkYR9X9fCLtCOJc5w==", 122 | "dev": true, 123 | "requires": { 124 | "glob": "^7.0.5" 125 | } 126 | }, 127 | "typescript": { 128 | "version": "2.9.2", 129 | "resolved": "https://registry.npmjs.org/typescript/-/typescript-2.9.2.tgz", 130 | "integrity": "sha512-Gr4p6nFNaoufRIY4NMdpQRNmgxVIGMs4Fcu/ujdYk3nAZqk7supzBE9idmvfZIlH/Cuj//dvi+019qEue9lV0w==", 131 | "dev": true 132 | }, 133 | "wrappy": { 134 | "version": "1.0.2", 135 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 136 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 137 | "dev": true 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "recurrent-js", 3 | "version": "1.7.4", 4 | "description": "Various amazingly simple to build and train neural network architectures. The library is an object-oriented neural network approach (baked with Typescript), containing stateless and stateful neural network architectures.", 5 | "keywords": [ 6 | "ai", 7 | "artificial intelligence", 8 | "recurrent", 9 | "stateless", 10 | "stateful", 11 | "recurrent neural network", 12 | "rnn", 13 | "deep learning", 14 | "dnn", 15 | "bayesian network", 16 | "bnn", 17 | "neural network", 18 | "artificial neural network", 19 | "deep neural network", 20 | "feedforward neural network", 21 | "lstm", 22 | "long short-term memory", 23 | "expression graphs", 24 | "automatic differentiation", 25 | "backprop", 26 | "backpropagation" 27 | ], 28 | "main": "./dist/index.js", 29 | "types": "./dist/index.d.ts", 30 | "scripts": { 31 | "prepare": "npm run test", 32 | "test": "./node_modules/.bin/jasmine --config=jasmine.json", 33 | "pretest": "npm run build", 34 | "build": "./node_modules/.bin/tsc -p .", 35 | "prebuild": "./node_modules/.bin/rimraf dist", 36 | "posttest": "./node_modules/.bin/rimraf dist/*.spec.* dist/*.doubles.*" 37 | }, 38 | "author": "Menno van Rahden", 39 | "homepage": "https://github.com/mvrahden/recurrent-js#readme", 40 | "bugs": { 41 | "url": "https://github.com/mvrahden/recurrent-js/issues" 42 | }, 43 | "repository": { 44 | "type": "git", 45 | "url": "git+https://github.com/mvrahden/recurrent-js.git" 46 | }, 47 | "license": "MIT", 48 | "devDependencies": { 49 | "@types/jasmine": "^2.8.8", 50 | "jasmine": "^2.99.0", 51 | "rimraf": "^2.6.2", 52 | "typescript": "^2.9.2" 53 | }, 54 | "dependencies": {} 55 | } 56 | -------------------------------------------------------------------------------- /src/fnn/ann.ts: -------------------------------------------------------------------------------- 1 | export interface ANN { 2 | forward(input: Array | Float64Array): Array | Float64Array; 3 | backward(expectedOutput: Array | Float64Array, alpha?: number): void; 4 | setTrainability(isTrainable: boolean): void; 5 | getSquaredLossFor(input: Array | Float64Array, expectedOutput: Array | Float64Array): number; 6 | } 7 | -------------------------------------------------------------------------------- /src/fnn/bnn.spec.ts: -------------------------------------------------------------------------------- 1 | import { Utils } from '..'; 2 | 3 | const patchFillRandn = () => { 4 | spyOn(Utils, 'fillRandn').and.callFake(fillConstOnes); 5 | }; 6 | 7 | const fillConstOnes = (arr) => { 8 | Utils.fillConst(arr, 1); 9 | }; 10 | -------------------------------------------------------------------------------- /src/fnn/bnn.ts: -------------------------------------------------------------------------------- 1 | import { Mat, Utils, NetOpts } from './..'; 2 | import { DNN } from './dnn'; 3 | 4 | export class BNN extends DNN { 5 | 6 | private hiddenStd: Array; 7 | 8 | /** 9 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 10 | * @param {{ hidden: { Wh, bh }, decoder: { Wh, b }}} opt Specs of the Neural Net. 11 | */ 12 | constructor(opt: { hidden: { Wh, bh }, decoder: { Wh, b } }); 13 | /** 14 | * Generates a Neural Net with given specs. 15 | * @param {NetOpts} opt Specs of the Neural Net. [defaults to: needsBackprop = false, mu = 0, std = 0.01] 16 | */ 17 | constructor(opt: NetOpts); 18 | constructor(opt: any) { 19 | super(opt); 20 | } 21 | 22 | protected initializeModelAsFreshInstance(opt: NetOpts) { 23 | super.initializeModelAsFreshInstance(opt); 24 | this.initializeHiddenLayerStds(opt); 25 | } 26 | 27 | /** 28 | * Assign a STD per hidden Unit per Layer 29 | */ 30 | private initializeHiddenLayerStds(opt: NetOpts) { 31 | this.hiddenStd = new Array(this.architecture.hiddenUnits.length); 32 | 33 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 34 | this.hiddenStd[i] = new Mat(this.architecture.hiddenUnits[i], 1); 35 | } 36 | 37 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 38 | Utils.fillRandn(this.hiddenStd[i].w, 0, 0.001); 39 | } 40 | } 41 | 42 | /** 43 | * Compute forward pass of Neural Network 44 | * @param state 1D column vector with observations 45 | * @param graph optional: inject Graph to append Operations 46 | * @returns Output of type `Mat` 47 | */ 48 | public specificForwardpass(state: Mat): Mat[] { 49 | const activations = this.computeHiddenActivations(state); 50 | 51 | // Add random normal distributed noise to activations 52 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 53 | activations[i] = this.graph.gauss(activations[i], this.hiddenStd[i]); 54 | } 55 | return activations; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/fnn/dnn.spec.ts: -------------------------------------------------------------------------------- 1 | import { DNN, Mat, NetOpts, Utils, Graph } from '..'; 2 | 3 | describe('Deep Neural Network (DNN):', () => { 4 | 5 | let sut: DNN; 6 | const config = { 7 | architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } 8 | }; 9 | 10 | describe('Instantiation:', () => { 11 | 12 | describe('Configuration with NetOpts:', () => { 13 | 14 | beforeEach(() => { 15 | sut = new DNN(config); 16 | }); 17 | 18 | it('fresh instance >> on creation >> should hold model with hidden layer, containing arrays of weight and bias matrices', () => { 19 | expect(sut.model).toBeDefined(); 20 | expect(sut.model.hidden).toBeDefined(); 21 | expect(sut.model.hidden.Wh).toBeDefined(); 22 | expect(sut.model.hidden.Wh.length).toBe(2); 23 | expect(sut.model.hidden.bh).toBeDefined(); 24 | expect(sut.model.hidden.bh.length).toBe(2); 25 | }); 26 | 27 | it('fresh instance >> on creation >> should hold model with decoder layer, containing weight and bias matrices', () => { 28 | expect(sut.model.decoder).toBeDefined(); 29 | expect(sut.model.decoder.Wh).toBeDefined(); 30 | expect(sut.model.decoder.b).toBeDefined(); 31 | }); 32 | 33 | describe('Hidden Layer:', () => { 34 | 35 | it('fresh instance >> on creation >> model should hold hidden layer containing weight matrices with expected dimensions', () => { 36 | expectHiddenStatelessWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength(2, [3, 4]); 37 | }); 38 | 39 | it('fresh instance >> on creation >> model should hold hidden layer containing bias matrices with expected dimensions', () => { 40 | expectHiddenBiasMatricesToHaveRowsOfSizeOfPrecedingLayerAndColsOfSize1(2, [3, 4]); 41 | }); 42 | 43 | const expectHiddenStatelessWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength = (inputSize: number, hiddenUnits: Array) => { 44 | let precedingLayerSize = inputSize; 45 | let expectedRows, expectedCols; 46 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 47 | expectedRows = hiddenUnits[i]; 48 | expectedCols = precedingLayerSize; 49 | expect(sut.model.hidden.Wh[i].rows).toBe(expectedRows); 50 | expect(sut.model.hidden.Wh[i].cols).toBe(expectedCols); 51 | precedingLayerSize = expectedRows; 52 | } 53 | }; 54 | 55 | const expectHiddenBiasMatricesToHaveRowsOfSizeOfPrecedingLayerAndColsOfSize1 = (inputSize: number, hiddenUnits: Array) => { 56 | let expectedRows; 57 | const expectedCols = 1; 58 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 59 | expectedRows = hiddenUnits[i]; 60 | expect(sut.model.hidden.bh[i].rows).toBe(expectedRows); 61 | expect(sut.model.hidden.bh[i].cols).toBe(expectedCols); 62 | } 63 | }; 64 | }); 65 | }); 66 | }); 67 | 68 | describe('Backpropagation:', () => { 69 | 70 | beforeEach(() => { 71 | sut = new DNN(config); 72 | }); 73 | 74 | it('given an fresh instance with having trainability set >> backward >> should throw an error', () => { 75 | const act = () => { sut.backward([]); }; 76 | expect(act).toThrowError(/Trainability is not enabled/); 77 | }); 78 | 79 | 80 | describe('Backward Pass:', () => { 81 | 82 | beforeEach(() => { 83 | sut.setTrainability(true); 84 | }); 85 | 86 | it('given an instance without forward pass >> backward >> should throw error ', () => { 87 | const act = () => { sut.backward([]); }; 88 | expect(act).toThrowError(/forward()/); 89 | }); 90 | 91 | describe('With Forward Pass:', () => { 92 | 93 | beforeEach(() => { 94 | const someInput = [1, 0]; 95 | sut.forward(someInput); 96 | patchBackwardSequenceAsSpies(); 97 | }); 98 | 99 | it('given an instance with forward pass >> backward >> should have c', () => { 100 | sut.backward([0, 1, 0]); 101 | 102 | expect(sut['graph'].backward).toHaveBeenCalled(); 103 | }); 104 | 105 | it('given an instance with forward pass >> backward >> should have called `sut.graph.backward`', () => { 106 | sut.backward([0, 1, 0]); 107 | 108 | expect(sut['graph'].backward).toHaveBeenCalled(); 109 | }); 110 | 111 | it('given an instance with forward pass >> backward >> should have called `sut.graph.forgetCurrentSequence`', () => { 112 | sut.backward([0, 1, 0]); 113 | 114 | expect(sut['graph'].forgetCurrentSequence).toHaveBeenCalled(); 115 | }); 116 | 117 | it('given an instance with forward pass >> backward >> should have called `sut.update`', () => { 118 | sut.backward([0, 1, 0]); 119 | 120 | expect(sut['updateWeights']).toHaveBeenCalled(); 121 | }); 122 | 123 | it('given an instance with forward pass >> backward >> should propagate the prediction quality loss into decoder layer', () => { 124 | sut.backward([0, 1, 0]); 125 | 126 | expect(sut['propagateLossIntoDecoderLayer']).toHaveBeenCalled(); 127 | }); 128 | }); 129 | }); 130 | 131 | const patchBackwardSequenceAsSpies = (): void => { 132 | spyOn(sut['graph'], 'backward'); 133 | spyOn(sut['graph'], 'forgetCurrentSequence'); 134 | // TypeScript workaround for private methods 135 | spyOn(sut, 'updateWeights' as any); 136 | spyOn(sut, 'propagateLossIntoDecoderLayer' as any); 137 | }; 138 | 139 | }); 140 | 141 | describe('Forward Pass:', () => { 142 | 143 | let input: Array; 144 | 145 | beforeEach(() => { 146 | patchFillRandn(); 147 | sut = new DNN(config); 148 | input = [0, 1]; 149 | }); 150 | 151 | it('given fresh network instance and some input vector >> forward pass >> should call activation function as often as number of hidden layer', () => { 152 | patchNetworkGraphAsSpy(); 153 | sut.forward(input); 154 | 155 | expect(sut['graph'].tanh).toHaveBeenCalledTimes(2); 156 | }); 157 | 158 | it('given fresh network instance and some input vector >> forward pass >> should return output with given dimensions', () => { 159 | const output = sut.forward(input); 160 | 161 | expect(output.length).toBe(3); 162 | }); 163 | 164 | it('given fresh network instance and some input vector >> forward pass >> should return Array filled with a value close to 3.91795', () => { 165 | const output = sut.forward(input); 166 | 167 | expect(output[0]).toBeCloseTo(3.91795); 168 | expect(output[1]).toBeCloseTo(3.91795); 169 | expect(output[2]).toBeCloseTo(3.91795); 170 | }); 171 | 172 | const patchFillRandn = () => { 173 | spyOn(Utils, 'fillRandn').and.callFake(fillConstOnes); 174 | }; 175 | 176 | const patchNetworkGraphAsSpy = () => { 177 | spyOn(sut['graph'], 'tanh').and.callFake(fillMatConstOnes); 178 | }; 179 | 180 | const fillConstOnes = (arr) => { 181 | Utils.fillConst(arr, 1); 182 | }; 183 | 184 | const fillMatConstOnes = (mat) => { 185 | const out = new Mat(mat.rows, 1); 186 | fillConstOnes(out.w); 187 | return out; 188 | }; 189 | }); 190 | }); 191 | -------------------------------------------------------------------------------- /src/fnn/dnn.ts: -------------------------------------------------------------------------------- 1 | import { Mat, NetOpts } from './..'; 2 | import { FNNModel } from './fnn-model'; 3 | 4 | export class DNN extends FNNModel { 5 | 6 | /** 7 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 8 | * @param {{ hidden: { Wh, bh }, decoder: { Wh, b } }} opt Specs of the Neural Net. 9 | */ 10 | constructor(opt: { hidden: { Wh, bh }, decoder: { Wh, b } }); 11 | /** 12 | * Generates a Neural Net with given specs. 13 | * @param {NetOpts} opt Specs of the Neural Net. [defaults to: needsBackprop = false, mu = 0, std = 0.01] 14 | */ 15 | constructor(opt: NetOpts); 16 | constructor(opt: any) { 17 | super(opt); 18 | } 19 | 20 | /** 21 | * Compute forward pass of Neural Network 22 | * @param state 1D column vector with observations 23 | * @param graph optional: inject Graph to append Operations 24 | * @returns Output of type `Mat` 25 | */ 26 | public specificForwardpass(state: Mat): Mat[] { 27 | const activations = this.computeHiddenActivations(state); 28 | return activations; 29 | } 30 | 31 | protected computeHiddenActivations(state: Mat): Mat[] { 32 | const hiddenActivations = new Array(); 33 | for (let d = 0; d < this.architecture.hiddenUnits.length; d++) { 34 | const inputVector = d === 0 ? state : hiddenActivations[d - 1]; 35 | const weightedInput = this.graph.mul(this.model.hidden.Wh[d], inputVector); 36 | const biasedWeightedInput = this.graph.add(weightedInput, this.model.hidden.bh[d]); 37 | const activation = this.graph.tanh(biasedWeightedInput); 38 | hiddenActivations.push(activation); 39 | } 40 | return hiddenActivations; 41 | } 42 | 43 | public static toJSON(dnn: DNN): { hidden: { Wh, bh }, decoder: { Wh, b } } { 44 | const json = { hidden: { Wh: [], bh: [] }, decoder: { Wh: null, b: null } }; 45 | for (let i = 0; i < dnn.model.hidden.Wh.length; i++) { 46 | json.hidden.Wh[i] = Mat.toJSON(dnn.model.hidden.Wh[i]); 47 | json.hidden.bh[i] = Mat.toJSON(dnn.model.hidden.bh[i]); 48 | } 49 | json.decoder.Wh = Mat.toJSON(dnn.model.decoder.Wh); 50 | json.decoder.b = Mat.toJSON(dnn.model.decoder.b); 51 | return json; 52 | } 53 | 54 | public static fromJSON(json: { hidden: { Wh, bh }, decoder: { Wh, b } }): DNN { 55 | return new DNN(json); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/fnn/fnn-model.spec.ts: -------------------------------------------------------------------------------- 1 | import { DNN } from '../.'; 2 | 3 | /** 4 | * Tests are being executed on a DNN instance. 5 | * DNN Class fully delegates instantiation to FNNModel Class. 6 | */ 7 | describe('Feedforward Neural Network Model:', () => { 8 | let sut: DNN; 9 | 10 | describe('Instantiation:', () => { 11 | 12 | describe('Configuration with NetOpts:', () => { 13 | 14 | describe('Initialization of NetOpts-Properties:', () => { 15 | 16 | it('given NetOpts with architecture >> on creation >> should define `sut.architecture` accordingly', () => { 17 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 18 | 19 | sut = new DNN(config); 20 | 21 | expect(sut['architecture']).toBeDefined(); 22 | expect(sut['architecture'].inputSize).toBe(2); 23 | expect(sut['architecture'].hiddenUnits[0]).toBe(3); 24 | expect(sut['architecture'].hiddenUnits[1]).toBe(4); 25 | expect(sut['architecture'].outputSize).toBe(3); 26 | }); 27 | 28 | it('given NetOpts without `training` >> on creation >> should define `sut.training` with default values', () => { 29 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 30 | 31 | sut = new DNN(config); 32 | 33 | expect(sut['training']).toBeDefined(); 34 | expect(sut['training'].alpha).toBe(0.01); 35 | expect(sut['training'].loss).toBeDefined(); 36 | }); 37 | 38 | it('given NetOpts with `training` >> on creation >> should define `sut.training` with given values', () => { 39 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 }, training: { loss: 1 } }; 40 | 41 | sut = new DNN(config); 42 | 43 | expect(sut['training']).toBeDefined(); 44 | expect(sut['training'].alpha).toBe(0.01); 45 | expect(sut['training'].loss).toBeDefined(); 46 | expect(sut['training'].loss).toBe(1); 47 | }); 48 | 49 | it('given NetOpts with `training` >> on creation >> should define `sut.training` with given values', () => { 50 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 }, training: { alpha: 1 } }; 51 | 52 | sut = new DNN(config); 53 | 54 | expect(sut['training']).toBeDefined(); 55 | expect(sut['training'].alpha).toBe(1); 56 | expect(sut['training'].loss).toBeDefined(); 57 | expect(sut['training'].loss).toBe(1e-6); 58 | }); 59 | 60 | }); 61 | 62 | describe('Decoder Layer:', () => { 63 | 64 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 65 | 66 | beforeEach(() => { 67 | sut = new DNN(config); 68 | }); 69 | 70 | it('fresh instance >> on creation >> model should hold decoder layer containing weight matrix with given dimensions', () => { 71 | expectDecoderWeightMatrixToHaveDimensionsOf(3, 4); 72 | }); 73 | 74 | it('fresh instance >> on creation >> model should hold decoder layer containing bias matrix with given dimensions', () => { 75 | expectDecoderBiasMatrixToHaveDimensionsOf(3, 1); 76 | }); 77 | 78 | const expectDecoderWeightMatrixToHaveDimensionsOf = (expectedRows: number, expectedCols: number) => { 79 | expect(sut.model.decoder.Wh.rows).toBe(expectedRows); 80 | expect(sut.model.decoder.Wh.cols).toBe(expectedCols); 81 | }; 82 | 83 | const expectDecoderBiasMatrixToHaveDimensionsOf = (expectedRows: number, expectedCols: number) => { 84 | expect(sut.model.decoder.b.rows).toBe(expectedRows); 85 | expect(sut.model.decoder.b.cols).toBe(expectedCols); 86 | }; 87 | }); 88 | }); 89 | 90 | describe('Configuration with JSON Object', () => { 91 | 92 | }); 93 | }); 94 | 95 | describe('Backpropagation:', () => { 96 | 97 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 98 | 99 | beforeEach(() => { 100 | sut = new DNN(config); 101 | 102 | spyOnUpdateMethods(); 103 | }); 104 | 105 | describe('Backward:', () => { 106 | 107 | describe('Hidden Layer:', () => { 108 | 109 | it('after forward pass >> backward >> should call graph to execute', () => { 110 | 111 | }); 112 | 113 | }); 114 | 115 | describe('Decoder Layer:', () => { 116 | 117 | }); 118 | }); 119 | 120 | describe('Update:', () => { 121 | 122 | describe('Hidden Layer:', () => { 123 | 124 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer', () => { 125 | sut['updateWeights'](); 126 | 127 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 128 | }); 129 | 130 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer with given value', () => { 131 | sut['updateWeights'](); 132 | 133 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 134 | }); 135 | 136 | it('fresh instance >> update with given value >> should call update methods of weight and bias matrices of all hidden layer', () => { 137 | const alpha = 0.02; 138 | sut['updateWeights'](alpha); 139 | 140 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 141 | }); 142 | 143 | it('fresh instance >> update with given value >> should call update methods of weight and bias matrices of all hidden layer with given value', () => { 144 | const alpha = 0.02; 145 | sut['updateWeights'](alpha); 146 | 147 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.02); 148 | }); 149 | 150 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 151 | expect(sut.model.hidden.Wh[0].update).toHaveBeenCalled(); 152 | expect(sut.model.hidden.Wh[1].update).toHaveBeenCalled(); 153 | expect(sut.model.hidden.bh[0].update).toHaveBeenCalled(); 154 | expect(sut.model.hidden.bh[1].update).toHaveBeenCalled(); 155 | }; 156 | 157 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 158 | expect(sut.model.hidden.Wh[0].update).toHaveBeenCalledWith(value); 159 | expect(sut.model.hidden.Wh[1].update).toHaveBeenCalledWith(value); 160 | expect(sut.model.hidden.bh[0].update).toHaveBeenCalledWith(value); 161 | expect(sut.model.hidden.bh[1].update).toHaveBeenCalledWith(value); 162 | }; 163 | }); 164 | 165 | describe('Decoder Layer:', () => { 166 | 167 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer', () => { 168 | sut['updateWeights'](); 169 | 170 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 171 | }); 172 | 173 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer with given value', () => { 174 | sut['updateWeights'](); 175 | 176 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 177 | }); 178 | 179 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 180 | expect(sut.model.decoder.Wh.update).toHaveBeenCalled(); 181 | expect(sut.model.decoder.b.update).toHaveBeenCalled(); 182 | }; 183 | 184 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 185 | expect(sut.model.decoder.Wh.update).toHaveBeenCalledWith(value); 186 | expect(sut.model.decoder.b.update).toHaveBeenCalledWith(value); 187 | }; 188 | }); 189 | }); 190 | 191 | const spyOnUpdateMethods = () => { 192 | spyOn(sut.model.hidden.Wh[0], 'update'); 193 | spyOn(sut.model.hidden.Wh[1], 'update'); 194 | spyOn(sut.model.hidden.bh[0], 'update'); 195 | spyOn(sut.model.hidden.bh[1], 'update'); 196 | 197 | spyOn(sut.model.decoder.Wh, 'update'); 198 | spyOn(sut.model.decoder.b, 'update'); 199 | }; 200 | }); 201 | }); 202 | 203 | -------------------------------------------------------------------------------- /src/fnn/fnn-model.ts: -------------------------------------------------------------------------------- 1 | import { Graph, Mat, RandMat, NetOpts } from './..'; 2 | import { ANN } from './ann'; 3 | import { Assertable } from './../utils/assertable'; 4 | 5 | export abstract class FNNModel extends Assertable implements ANN { 6 | 7 | protected architecture: { inputSize: number, hiddenUnits: Array, outputSize: number }; 8 | protected training: { alpha: number, lossClamp: number, loss: number }; 9 | 10 | public model: { hidden: { Wh: Array, bh: Array }, decoder: { Wh: Mat, b: Mat } } = { hidden: { Wh: [], bh: [] }, decoder: { Wh: null, b: null } }; 11 | 12 | protected graph: Graph; 13 | protected previousOutput: Mat; 14 | 15 | /** 16 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 17 | * @param {{ hidden: { Wh, bh }, decoder: { Wh: Mat, b: Mat } }} opt Specs of the Neural Net. 18 | */ 19 | constructor(opt: { hidden: { Wh, bh }, decoder: { Wh: Mat, b: Mat } }); 20 | /** 21 | * Generates a Neural Net with given specs. 22 | * @param {NetOpts} opt Specs of the Neural Net. [defaults to: needsBackprop = false, mu = 0, std = 0.01] 23 | */ 24 | constructor(opt: NetOpts); 25 | constructor(opt: any) { 26 | super(); 27 | this.initializeNeuralNetworkFromGivenOptions(opt); 28 | } 29 | 30 | private initializeNeuralNetworkFromGivenOptions(opt: any): void { 31 | this.graph = new Graph(); 32 | if (FNNModel.isFromJSON(opt)) { 33 | this.initializeModelFromJSONObject(opt); 34 | } 35 | else if (FNNModel.isFreshInstanceCall(opt)) { 36 | this.initializeModelAsFreshInstance(opt); 37 | } 38 | else { 39 | FNNModel.assert(false, 'Improper input for DNN.'); 40 | } 41 | } 42 | 43 | protected static isFromJSON(opt: any): boolean { 44 | return FNNModel.has(opt, ['hidden', 'decoder']) 45 | && FNNModel.has(opt.hidden, ['Wh', 'bh']) 46 | && FNNModel.has(opt.decoder, ['Wh', 'b']); 47 | } 48 | 49 | protected initializeModelFromJSONObject(opt: { hidden: { Wh, bh }, decoder: { Wh, b } }): void { 50 | this.initializeHiddenLayerFromJSON(opt); 51 | this.model.decoder.Wh = Mat.fromJSON(opt['decoder']['Wh']); 52 | this.model.decoder.b = Mat.fromJSON(opt['decoder']['b']); 53 | } 54 | 55 | protected initializeHiddenLayerFromJSON(opt: { hidden: { Wh: Array; bh: Array; }; decoder: { Wh: Mat; b: Mat; }; }): void { 56 | FNNModel.assert(!Array.isArray(opt['hidden']['Wh']), 'Wrong JSON Format to recreate Hidden Layer.'); 57 | for (let i = 0; i < opt.hidden.Wh.length; i++) { 58 | this.model.hidden.Wh[i] = Mat.fromJSON(opt.hidden.Wh[i]); 59 | this.model.hidden.bh[i] = Mat.fromJSON(opt.hidden.bh[i]); 60 | } 61 | } 62 | 63 | protected static isFreshInstanceCall(opt: any): boolean { 64 | return FNNModel.has(opt, ['architecture']) && FNNModel.has(opt.architecture, ['inputSize', 'hiddenUnits', 'outputSize']); 65 | } 66 | 67 | protected initializeModelAsFreshInstance(opt: NetOpts): void { 68 | this.architecture = this.determineArchitectureProperties(opt); 69 | this.training = this.determineTrainingProperties(opt); 70 | 71 | const mu = opt['mu'] ? opt['mu'] : 0; 72 | const std = opt['std'] ? opt['std'] : 0.1; 73 | 74 | this.model = this.initializeFreshNetworkModel(); 75 | 76 | this.initializeHiddenLayer(mu, std); 77 | 78 | this.initializeDecoder(mu, std); 79 | } 80 | 81 | protected determineArchitectureProperties(opt: NetOpts): { inputSize: number, hiddenUnits: Array, outputSize: number } { 82 | const out = { inputSize: null, hiddenUnits: null, outputSize: null }; 83 | out.inputSize = typeof opt.architecture.inputSize === 'number' ? opt.architecture.inputSize : 1; 84 | out.hiddenUnits = Array.isArray(opt.architecture.hiddenUnits) ? opt.architecture.hiddenUnits : [1]; 85 | out.outputSize = typeof opt.architecture.outputSize === 'number' ? opt.architecture.outputSize : 1; 86 | return out; 87 | } 88 | 89 | protected determineTrainingProperties(opt: NetOpts): { alpha: number, lossClamp: number, loss: number } { 90 | const out = { alpha: null, lossClamp: null, loss: null }; 91 | if (!opt.training) { 92 | // patch `opt` 93 | opt.training = out; 94 | } 95 | 96 | out.alpha = typeof opt.training.alpha === 'number' ? opt.training.alpha : 0.01; 97 | out.lossClamp = typeof opt.training.lossClamp === 'number' ? opt.training.lossClamp : 1; 98 | out.loss = typeof opt.training.loss === 'number' ? opt.training.loss : 1e-6; 99 | 100 | return out; 101 | } 102 | 103 | protected initializeFreshNetworkModel(): { hidden: { Wh: Array; bh: Array; }; decoder: { Wh: Mat; b: Mat; }; } { 104 | return { 105 | hidden: { 106 | Wh: new Array(this.architecture.hiddenUnits.length), 107 | bh: new Array(this.architecture.hiddenUnits.length) 108 | }, 109 | decoder: { 110 | Wh: null, 111 | b: null 112 | } 113 | }; 114 | } 115 | 116 | protected initializeHiddenLayer(mu: number, std: number): void { 117 | let hiddenSize; 118 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 119 | const previousSize = this.getPrecedingLayerSize(i); 120 | hiddenSize = this.architecture.hiddenUnits[i]; 121 | this.model.hidden.Wh[i] = new RandMat(hiddenSize, previousSize, mu, std); 122 | this.model.hidden.bh[i] = new Mat(hiddenSize, 1); 123 | } 124 | } 125 | 126 | /** 127 | * According to the given hiddenLayer Index, get the size of preceding layer. 128 | * @param i current hidden layer index 129 | */ 130 | private getPrecedingLayerSize(i: number) { 131 | return i === 0 ? this.architecture.inputSize : this.architecture.hiddenUnits[i - 1]; 132 | } 133 | 134 | protected initializeDecoder(mu: number, std: number): void { 135 | this.model.decoder.Wh = new RandMat(this.architecture.outputSize, this.architecture.hiddenUnits[this.architecture.hiddenUnits.length - 1], mu, std); 136 | this.model.decoder.b = new Mat(this.architecture.outputSize, 1); 137 | } 138 | 139 | /** 140 | * Sets the neural network into a trainable state. 141 | * Also cleans the memory of forward pass operations, meaning that the last forward pass cannot be used for backpropagation. 142 | * @param isTrainable 143 | */ 144 | public setTrainability(isTrainable: boolean): void { 145 | this.graph.forgetCurrentSequence(); 146 | this.graph.memorizeOperationSequence(isTrainable); 147 | } 148 | 149 | /** 150 | * 151 | * @param expectedOutput Corresponding target for previous Input of forward-pass 152 | * @param alpha update factor 153 | * @returns squared summed loss 154 | */ 155 | public backward(expectedOutput: Array | Float64Array, alpha?: number): void { 156 | FNNModel.assert(this.graph.isMemorizingSequence(), '['+ this.constructor.name +'] Trainability is not enabled.'); 157 | FNNModel.assert(typeof this.previousOutput !== 'undefined', '['+ this.constructor.name +'] Please execute `forward()` before calling `backward()`'); 158 | this.propagateLossIntoDecoderLayer(expectedOutput); 159 | this.backwardGraph(); 160 | this.updateWeights(alpha); 161 | this.resetGraph(); 162 | } 163 | 164 | private backwardGraph(): void { 165 | this.graph.backward(); 166 | } 167 | 168 | private resetGraph(): void { 169 | this.graph.forgetCurrentSequence(); 170 | } 171 | 172 | private propagateLossIntoDecoderLayer(expected: Array | Float64Array): void { 173 | let loss; 174 | for (let i = 0; i < this.architecture.outputSize; i++) { 175 | loss = this.previousOutput.w[i] - expected[i]; 176 | if (Math.abs(loss) <= this.training.loss) { 177 | continue; 178 | } else { 179 | loss = this.clipLoss(loss); 180 | this.previousOutput.dw[i] = loss; 181 | } 182 | } 183 | } 184 | 185 | private clipLoss(loss: number): number { 186 | if (loss > this.training.lossClamp) { return this.training.lossClamp; } 187 | else if (loss < -this.training.lossClamp) { return -this.training.lossClamp; } 188 | return loss; 189 | } 190 | 191 | protected updateWeights(alpha?: number): void { 192 | alpha = alpha ? alpha : this.training.alpha; 193 | this.updateHiddenLayer(alpha); 194 | this.updateDecoderLayer(alpha); 195 | } 196 | 197 | protected updateHiddenLayer(alpha: number): void { 198 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 199 | this.model.hidden.Wh[i].update(alpha); 200 | this.model.hidden.bh[i].update(alpha); 201 | } 202 | } 203 | 204 | protected updateDecoderLayer(alpha: number): void { 205 | this.model.decoder.Wh.update(alpha); 206 | this.model.decoder.b.update(alpha); 207 | } 208 | 209 | /** 210 | * Compute forward pass of Neural Network 211 | * @param input 1D column vector with observations 212 | * @param graph optional: inject Graph to append Operations 213 | * @returns Output of type `Mat` 214 | */ 215 | public forward(input: Array | Float64Array): Array | Float64Array { 216 | const mat = this.transformArrayToMat(input); 217 | const activations = this.specificForwardpass(mat); 218 | const outputMat = this.computeOutput(activations); 219 | const output = this.transformMatToArray(outputMat); 220 | this.previousOutput = outputMat; 221 | return output; 222 | } 223 | 224 | private transformArrayToMat(input: Array | Float64Array): Mat { 225 | const mat = new Mat(this.architecture.inputSize, 1); 226 | mat.setFrom(input); 227 | return mat; 228 | } 229 | 230 | private transformMatToArray(input: Mat): Array | Float64Array { 231 | const arr = input.w.slice(0); 232 | return arr; 233 | } 234 | 235 | protected abstract specificForwardpass(state: Mat): Array; 236 | 237 | protected computeOutput(hiddenUnitActivations: Array): Mat { 238 | const weightedInputs = this.graph.mul(this.model.decoder.Wh, hiddenUnitActivations[hiddenUnitActivations.length - 1]); 239 | return this.graph.add(weightedInputs, this.model.decoder.b); 240 | } 241 | 242 | public getSquaredLossFor(input: number[] | Float64Array, expectedOutput: number[] | Float64Array): number { 243 | const trainability = this.graph.isMemorizingSequence(); 244 | this.setTrainability(false); 245 | const lossSum = this.calculateLossSumByForwardPass(input, expectedOutput); 246 | this.setTrainability(trainability); 247 | return lossSum * lossSum; 248 | } 249 | 250 | private calculateLossSumByForwardPass(input: Array | Float64Array, expected: Array | Float64Array): number { 251 | let lossSum = 0; 252 | const actualOutput = this.forward(input); 253 | for (let i = 0; i < this.architecture.outputSize; i++) { 254 | const loss = actualOutput[i] - expected[i]; 255 | lossSum += loss; 256 | } 257 | return lossSum; 258 | } 259 | 260 | private static has(obj: any, keys: Array): boolean { 261 | FNNModel.assert(obj, 'Improper input for DNN.'); 262 | for (const key of keys) { 263 | if (Object.hasOwnProperty.call(obj, key)) { continue; } 264 | return false; 265 | } 266 | return true; 267 | } 268 | } 269 | -------------------------------------------------------------------------------- /src/fnn/utils/sample.ts: -------------------------------------------------------------------------------- 1 | export interface Sample { 2 | input: Array; 3 | output: Array; 4 | } 5 | -------------------------------------------------------------------------------- /src/fnn/utils/training-set.ts: -------------------------------------------------------------------------------- 1 | import { Sample } from "./sample"; 2 | 3 | export class TrainingSet { 4 | 5 | private samples: Array; 6 | 7 | setSamples(samples: Array): void { 8 | this.samples = samples; 9 | } 10 | 11 | length(): number { 12 | return this.samples.length; 13 | } 14 | 15 | getInputForSample(i: number): Array { 16 | return this.samples[i].input; 17 | } 18 | 19 | getExpectedOutputForSample(i: number): Array { 20 | return this.samples[i].output; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/graph.spec.ts: -------------------------------------------------------------------------------- 1 | import { Graph, Mat } from '.'; 2 | import { MatOps } from './utils/mat-ops'; 3 | 4 | /** 5 | * TEST GRAPH derivative functions --> Outsource to Mat?? 6 | */ 7 | 8 | describe('Graph Operations:', () => { 9 | 10 | let sut: Graph; 11 | let mat: Mat; 12 | 13 | beforeEach(() => { 14 | // Turn Matrix into Test Double through Method-Patching 15 | initializeMatrixSpyFunctions(); 16 | 17 | sut = new Graph(); 18 | mat = new Mat(2, 4); 19 | }); 20 | 21 | describe('Matrix Operation Call:', () => { 22 | 23 | describe('Single Matrix Operations:', () => { 24 | 25 | it('given a graph and a matrix >> rowPluck >> should have called MatOps.rowPluck', () => { 26 | const anyIndex = 0; 27 | sut.rowPluck(mat, anyIndex); 28 | 29 | expect(MatOps.rowPluck).toHaveBeenCalled(); 30 | expect(MatOps.rowPluck).toHaveBeenCalledWith(mat, 0); 31 | }); 32 | 33 | describe('Monadic Matrix Operations', () => { 34 | 35 | it('given a graph and a matrix >> tanh >> should have called MatOps.tanh', () => { 36 | sut.tanh(mat); 37 | 38 | expectSpyMethodToHaveBeenCalled(MatOps.tanh); 39 | }); 40 | 41 | it('given a graph and a matrix >> sig >> should have called MatOps.sig', () => { 42 | sut.sig(mat); 43 | 44 | expectSpyMethodToHaveBeenCalled(MatOps.sig); 45 | }); 46 | 47 | it('given a graph and a matrix >> relu >> should have called MatOps.relu', () => { 48 | sut.relu(mat); 49 | 50 | expectSpyMethodToHaveBeenCalled(MatOps.relu); 51 | }); 52 | 53 | const expectSpyMethodToHaveBeenCalled = (spy: Function): void => { 54 | expect(spy).toHaveBeenCalled(); 55 | expect(spy).toHaveBeenCalledWith(mat); 56 | }; 57 | }); 58 | }); 59 | 60 | describe('Dual Matrix Operations:', () => { 61 | 62 | let mat2: Mat; 63 | 64 | beforeEach(() => { 65 | mat2 = new Mat(2, 4); 66 | }); 67 | 68 | it('given a graph and 2 matrices >> mul >> should have called MatOps.mul', () => { 69 | sut.mul(mat, mat2); 70 | 71 | expectSpyMethodToHaveBeenCalled(MatOps.mul); 72 | }); 73 | 74 | it('given a graph and 2 matrices >> add >> should have called MatOps.add', () => { 75 | sut.add(mat, mat2); 76 | 77 | expectSpyMethodToHaveBeenCalled(MatOps.add); 78 | }); 79 | 80 | it('given a graph and 2 matrices >> dot >> should have called MatOps.dot', () => { 81 | sut.dot(mat, mat2); 82 | 83 | expectSpyMethodToHaveBeenCalled(MatOps.dot); 84 | }); 85 | 86 | it('given a graph and 2 matrices >> eltmul >> should have called MatOps.eltmul', () => { 87 | sut.eltmul(mat, mat2); 88 | 89 | expectSpyMethodToHaveBeenCalled(MatOps.eltmul); 90 | }); 91 | 92 | const expectSpyMethodToHaveBeenCalled = (spy: Function): void => { 93 | expect(spy).toHaveBeenCalled(); 94 | expect(spy).toHaveBeenCalledWith(mat, mat2); 95 | }; 96 | }); 97 | }); 98 | 99 | describe('Backpropagation Stack:', () => { 100 | 101 | describe('Without Backpropagation:', () => { 102 | 103 | beforeEach(() => { 104 | // Turn Graph Property into Test Double through Method-Patching 105 | spyOn(sut['backpropagationStack'], 'push'); 106 | }); 107 | 108 | describe('Single Matrix Operations:', () => { 109 | 110 | it('given a graph without backpropagation >> gauss >> should add function on stack', () => { 111 | const std = new Mat(2, 4); 112 | sut.gauss(mat, std); // Exception to the rule, gauss adds noise but does not change the slope. 113 | 114 | expectOperationNotToBePushedToBackpropagationStack(); 115 | }); 116 | 117 | it('given a graph without backpropagation >> rowPluck >> should add function on stack', () => { 118 | const rowIndex = 1; 119 | sut.rowPluck(mat, rowIndex); 120 | 121 | expectOperationNotToBePushedToBackpropagationStack(); 122 | }); 123 | 124 | it('given a graph without backpropagation >> tanh >> should add function on stack', () => { 125 | sut.tanh(mat); 126 | 127 | expectOperationNotToBePushedToBackpropagationStack(); 128 | }); 129 | 130 | it('given a graph without backpropagation >> sig >> should add function on stack', () => { 131 | sut.sig(mat); 132 | 133 | expectOperationNotToBePushedToBackpropagationStack(); 134 | }); 135 | 136 | it('given a graph without backpropagation >> relu >> should add function on stack', () => { 137 | sut.relu(mat); 138 | 139 | expectOperationNotToBePushedToBackpropagationStack(); 140 | }); 141 | }); 142 | 143 | describe('Dual Matrix Operations:', () => { 144 | 145 | let mat2: Mat; 146 | 147 | beforeEach(() => { 148 | mat2 = new Mat(2, 4); 149 | }); 150 | 151 | it('given a graph without backpropagation >> add >> should add function on stack', () => { 152 | sut.add(mat, mat2); 153 | 154 | expectOperationNotToBePushedToBackpropagationStack(); 155 | }); 156 | 157 | it('given a graph without backpropagation >> mul >> should add function on stack', () => { 158 | sut.mul(mat, mat2); 159 | 160 | expectOperationNotToBePushedToBackpropagationStack(); 161 | }); 162 | 163 | it('given a graph without backpropagation >> dot >> should add function on stack', () => { 164 | sut.dot(mat, mat2); 165 | 166 | expectOperationNotToBePushedToBackpropagationStack(); 167 | }); 168 | 169 | it('given a graph without backpropagation >> eltmul >> should add function on stack', () => { 170 | sut.eltmul(mat, mat2); 171 | 172 | expectOperationNotToBePushedToBackpropagationStack(); 173 | }); 174 | }); 175 | 176 | }); 177 | 178 | describe('With Backpropagation:', () => { 179 | 180 | beforeEach(() => { 181 | sut = new Graph(); 182 | sut.memorizeOperationSequence(true); 183 | // Turn Graph Property into Test Double through Method-Patching 184 | spyOn(sut['backpropagationStack'], 'push'); 185 | }); 186 | 187 | describe('Single Matrix Operations:', () => { 188 | 189 | it('given a graph with backpropagation >> gauss >> should NOT add function on stack', () => { 190 | const std = new Mat(2, 4); 191 | sut.gauss(mat, std); // Exception to the rule, gauss adds noise but does not change the slope. 192 | 193 | expectOperationNotToBePushedToBackpropagationStack(); 194 | }); 195 | 196 | it('given a graph with backpropagation >> rowPluck >> should add function on stack', () => { 197 | const rowIndex = 1; 198 | sut.rowPluck(mat, rowIndex); 199 | 200 | expectOperationToBePushedToBackpropagationStack(); 201 | }); 202 | 203 | it('given a graph with backpropagation >> tanh >> should add function on stack', () => { 204 | sut.tanh(mat); 205 | 206 | expectOperationToBePushedToBackpropagationStack(); 207 | }); 208 | 209 | it('given a graph with backpropagation >> sig >> should add function on stack', () => { 210 | sut.sig(mat); 211 | 212 | expectOperationToBePushedToBackpropagationStack(); 213 | }); 214 | 215 | it('given a graph with backpropagation >> relu >> should add function on stack', () => { 216 | sut.relu(mat); 217 | 218 | expectOperationToBePushedToBackpropagationStack(); 219 | }); 220 | }); 221 | 222 | describe('Dual Matrix Operations:', () => { 223 | 224 | let mat2: Mat; 225 | 226 | beforeEach(() => { 227 | mat2 = new Mat(2, 4); 228 | }); 229 | 230 | it('given a graph with backpropagation >> add >> should add function on stack', () => { 231 | sut.add(mat, mat2); 232 | 233 | expectOperationToBePushedToBackpropagationStack(); 234 | }); 235 | 236 | it('given a graph with backpropagation >> mul >> should add function on stack', () => { 237 | sut.mul(mat, mat2); 238 | 239 | expectOperationToBePushedToBackpropagationStack(); 240 | }); 241 | 242 | it('given a graph with backpropagation >> dot >> should add function on stack', () => { 243 | sut.dot(mat, mat2); 244 | 245 | expectOperationToBePushedToBackpropagationStack(); 246 | }); 247 | 248 | it('given a graph with backpropagation >> eltmul >> should add function on stack', () => { 249 | sut.eltmul(mat, mat2); 250 | 251 | expectOperationToBePushedToBackpropagationStack(); 252 | }); 253 | }); 254 | 255 | }); 256 | 257 | const expectOperationToBePushedToBackpropagationStack = () => { 258 | expect(sut['backpropagationStack'].push).toHaveBeenCalled(); 259 | }; 260 | 261 | const expectOperationNotToBePushedToBackpropagationStack = () => { 262 | expect(sut['backpropagationStack'].push).not.toHaveBeenCalled(); 263 | }; 264 | }); 265 | 266 | describe('Sequence Memorization:', () => { 267 | 268 | it('given fresh instance >> set sequence memorization to true >> should to return true', () => { 269 | sut.memorizeOperationSequence(true); 270 | 271 | expect(sut.isMemorizingSequence()).toBe(true); 272 | }); 273 | 274 | it('given fresh instance >> set sequence memorization to true >> should to return true', () => { 275 | sut.memorizeOperationSequence(false); 276 | 277 | expect(sut.isMemorizingSequence()).toBe(false); 278 | }); 279 | }); 280 | 281 | describe('Reset Sequence:', () => { 282 | 283 | it('instance populated with operations in backpropagationStack >> forgetSequence >> should have an empty `backpropagationStack`', () => { 284 | sut.add(mat, mat); 285 | sut.add(mat, mat); 286 | sut.add(mat, mat); 287 | 288 | sut.forgetCurrentSequence(); 289 | 290 | expect(sut['backpropagationStack'].length).toBe(0); 291 | }); 292 | }); 293 | 294 | const initializeMatrixSpyFunctions = (): void => { 295 | spyOn(MatOps, 'rowPluck'); 296 | spyOn(MatOps, 'tanh'); 297 | spyOn(MatOps, 'sig'); 298 | spyOn(MatOps, 'relu'); 299 | spyOn(MatOps, 'mul'); 300 | spyOn(MatOps, 'add'); 301 | spyOn(MatOps, 'dot'); 302 | spyOn(MatOps, 'eltmul'); 303 | }; 304 | }); 305 | -------------------------------------------------------------------------------- /src/graph.ts: -------------------------------------------------------------------------------- 1 | import { Mat } from '.'; 2 | import { MatOps } from './utils/mat-ops'; 3 | 4 | export class Graph { 5 | private needsBackpropagation: boolean; 6 | 7 | private readonly backpropagationStack: Array; 8 | 9 | /** 10 | * Initializes a Graph to memorize Matrix Operation Sequences. 11 | */ 12 | constructor() { 13 | this.needsBackpropagation = false; 14 | 15 | this.backpropagationStack = new Array(); 16 | } 17 | 18 | /** 19 | * Switch whether to memorize the operation sequence for Backpropagation (true) or ignore it (false). 20 | * @param {boolean} isMemorizing true or false [defaults to false] 21 | */ 22 | public memorizeOperationSequence(isMemorizing: boolean = false): void { 23 | this.needsBackpropagation = isMemorizing; 24 | } 25 | 26 | /** 27 | * Gives back the state of either memorizing or not a sequence of operations 28 | */ 29 | public isMemorizingSequence(): boolean { 30 | return this.needsBackpropagation; 31 | } 32 | 33 | /** 34 | * Clears the memorized sequence of operations 35 | */ 36 | public forgetCurrentSequence(): void { 37 | this.backpropagationStack.length = 0; // reset array 38 | } 39 | 40 | /** 41 | * Executes the memorized sequence of derivative operations in LIFO order 42 | */ 43 | public backward(): void { 44 | for (let i = this.backpropagationStack.length - 1; i >= 0; i--) { 45 | this.backpropagationStack[i](); 46 | } 47 | } 48 | 49 | /** 50 | * Non-destructively pluck a row of m with rowIndex 51 | * @param m 52 | * @param rowIndex 53 | */ 54 | public rowPluck(m: Mat, rowIndex: number): Mat { 55 | const out = MatOps.rowPluck(m, rowIndex); 56 | this.addRowPluckToBackpropagationStack(m, rowIndex, out); 57 | return out; 58 | } 59 | 60 | private addRowPluckToBackpropagationStack(m: Mat, rowIndex: number, out: Mat) { 61 | if (this.needsBackpropagation) { 62 | const backward = MatOps.getRowPluckBackprop(m, rowIndex, out); 63 | this.backpropagationStack.push(backward); 64 | } 65 | } 66 | 67 | /** 68 | * Non-destructively pluck a row of m with rowIndex 69 | * @param m 70 | * @param rowIndex 71 | */ 72 | public gauss(m: Mat, std: Mat): Mat { 73 | const out = MatOps.gauss(m, std); 74 | return out; 75 | } 76 | 77 | /** 78 | * Non-destructive elementwise tanh 79 | * @param m 80 | */ 81 | public tanh(m: Mat): Mat { 82 | const out = MatOps.tanh(m); 83 | this.addTanhToBackpropagationStack(m, out); 84 | return out; 85 | } 86 | 87 | private addTanhToBackpropagationStack(m: Mat, out: Mat) { 88 | if (this.needsBackpropagation) { 89 | const backward = MatOps.getTanhBackprop(m, out); 90 | this.backpropagationStack.push(backward); 91 | } 92 | } 93 | 94 | /** 95 | * Non-destructive elementwise sigmoid 96 | * @param m 97 | */ 98 | public sig(m: Mat): Mat { 99 | const out = MatOps.sig(m); 100 | this.addSigmoidToBackpropagationStack(m, out); 101 | return out; 102 | } 103 | 104 | private addSigmoidToBackpropagationStack(m: Mat, out: Mat) { 105 | if (this.needsBackpropagation) { 106 | const backward = MatOps.getSigmoidBackprop(m, out); 107 | this.backpropagationStack.push(backward); 108 | } 109 | } 110 | 111 | /** 112 | * Non-destructive elementwise ReLU (rectified linear unit) 113 | * @param m 114 | */ 115 | public relu(m: Mat): Mat { 116 | const out = MatOps.relu(m); 117 | this.addReluToBackpropagationStack(m, out); 118 | return out; 119 | } 120 | 121 | private addReluToBackpropagationStack(m: Mat, out: Mat) { 122 | if (this.needsBackpropagation) { 123 | const backward = MatOps.getReluBackprop(m, out); 124 | this.backpropagationStack.push(backward); 125 | } 126 | } 127 | 128 | /** 129 | * Non-destructive elementwise addition 130 | * @param m1 131 | * @param m2 132 | */ 133 | public add(m1: Mat, m2: Mat): Mat { 134 | const out = MatOps.add(m1, m2); 135 | this.addAdditionToBackpropagationStack(m1, m2, out); 136 | return out; 137 | } 138 | 139 | private addAdditionToBackpropagationStack(m1: Mat, m2: Mat, out: Mat) { 140 | if (this.needsBackpropagation) { 141 | const backward = MatOps.getAddBackprop(m1, m2, out); 142 | this.backpropagationStack.push(backward); 143 | } 144 | } 145 | 146 | /** 147 | * Non-destructive matrix multiplication 148 | * @param m1 149 | * @param m2 150 | */ 151 | public mul(m1: Mat, m2: Mat): Mat { 152 | const out = MatOps.mul(m1, m2); 153 | this.addMultiplyToBackpropagationStack(m1, m2, out); 154 | return out; 155 | } 156 | 157 | private addMultiplyToBackpropagationStack(m1: Mat, m2: Mat, out: Mat) { 158 | if (this.needsBackpropagation) { 159 | const backward = MatOps.getMulBackprop(m1, m2, out); 160 | this.backpropagationStack.push(backward); 161 | } 162 | } 163 | 164 | /** 165 | * Non-destructive Dot product. 166 | * @param m1 167 | * @param m2 168 | */ 169 | public dot(m1: Mat, m2: Mat): Mat { 170 | const out = MatOps.dot(m1, m2); 171 | this.addDotToBackpropagationStack(m1, m2, out); 172 | return out; 173 | } 174 | 175 | private addDotToBackpropagationStack(m1: Mat, m2: Mat, out: Mat) { 176 | if (this.needsBackpropagation) { 177 | const backward = MatOps.getDotBackprop(m1, m2, out); 178 | this.backpropagationStack.push(backward); 179 | } 180 | } 181 | 182 | /** 183 | * Non-destructively elementwise multiplication 184 | * @param m1 185 | * @param m2 186 | */ 187 | public eltmul(m1: Mat, m2: Mat): Mat { 188 | const out = MatOps.eltmul(m1, m2); 189 | this.addEltmulToBackpropagationStack(m1, m2, out); 190 | return out; 191 | } 192 | 193 | private addEltmulToBackpropagationStack(m1: Mat, m2: Mat, out: Mat) { 194 | if (this.needsBackpropagation) { 195 | const backward = MatOps.getEltmulBackprop(m1, m2, out); 196 | this.backpropagationStack.push(backward); 197 | } 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import { BNN } from './fnn/bnn'; 2 | import { DNN } from './fnn/dnn'; 3 | import { Graph } from './graph'; 4 | import { InnerState } from './utils/inner-state'; 5 | import { LSTM } from './rnn/lstm'; 6 | import { Mat } from './mat'; 7 | import { MatOps } from './utils/mat-ops'; 8 | import { Net } from './net'; 9 | import { NetOpts } from './utils/net-opts'; 10 | import { Utils } from './utils'; 11 | import { RandMat } from './rand-mat'; 12 | import { RNN } from './rnn/rnn'; 13 | 14 | export { BNN, DNN, Graph, InnerState, LSTM, Mat, MatOps, Net, NetOpts, Utils, RandMat, RNN }; 15 | -------------------------------------------------------------------------------- /src/mat.spec.ts: -------------------------------------------------------------------------------- 1 | import { Mat, Utils } from '.'; 2 | 3 | 4 | describe('Matrix Object:', () => { 5 | 6 | let sut: Mat; 7 | 8 | describe('Instantiation:', () => { 9 | 10 | beforeEach(() => { 11 | sut = new Mat(2, 3); 12 | }); 13 | 14 | it('fresh instance >> on creation >> should have expected rows, cols and array length', () => { 15 | expect(sut.rows).toBe(2); 16 | expect(sut.cols).toBe(3); 17 | expect(sut['_length']).toBe(6); 18 | }); 19 | 20 | it('fresh instance >> on creation >> should have array of values [w] and derivatives [dw] with given length', () => { 21 | expect(sut.w.length).toBe(6); 22 | expect(sut.dw.length).toBe(6); 23 | }); 24 | 25 | it('fresh instance >> on creation >> should have array of values [w] populated with zeros', () => { 26 | expectValuesToBe(0); 27 | }); 28 | 29 | it('fresh instance >> on creation >> should have array of derivatives [dw] populated with zeros', () => { 30 | expectDerivativesToBe(0); 31 | }); 32 | 33 | const expectValuesToBe = (value: number) => { 34 | for (let i = 0; i < sut.w.length; i++) { 35 | expect(sut.w[i]).toBe(value); 36 | } 37 | }; 38 | 39 | const expectDerivativesToBe = (value: number) => { 40 | for (let i = 0; i < sut.w.length; i++) { 41 | expect(sut.w[i]).toBe(value); 42 | } 43 | }; 44 | 45 | }); 46 | 47 | describe('Get and Set:', () => { 48 | 49 | beforeEach(() => { 50 | sut = new Mat(2, 3); 51 | sut.setFrom([0, 3, 2, 4, 5, 1]); 52 | }); 53 | 54 | it('fresh instance >> setFrom >> values should be populated in same order as given Array', () => { 55 | expectValuesToBe([0, 3, 2, 4, 5, 1]); 56 | }); 57 | 58 | it('fresh instance >> setFrom >> derivatives should stay untouched', () => { 59 | expectDerivativesToBe([0, 0, 0, 0, 0, 0]); 60 | }); 61 | 62 | it('with [0, 3, 2, 4, 5, 1] populated instance >> get value with given row and col >> should return value at position', () => { 63 | expectValueAtGivenPosition({ row: 1, col: 2 }, 1); 64 | expectValueAtGivenPosition({ row: 0, col: 1 }, 3); 65 | }); 66 | 67 | it('with [0, 3, 2, 4, 5, 1] populated instance >> set value at given row and col >> should mutate value at given position', () => { 68 | sut.set(1, 2, 4); 69 | expect(sut.w[5]).toBe(4); 70 | expect(sut.dw[5]).toBe(0); 71 | 72 | sut.set(0, 1, 4); 73 | expect(sut.w[1]).toBe(4); 74 | expect(sut.dw[1]).toBe(0); 75 | }); 76 | 77 | it('with [0, 3, 2, 4, 5, 1] populated instance >> set value at given row and col >> should not mutate derivative at position', () => { 78 | sut.set(1, 2, 4); 79 | expect(sut.dw[5]).toBe(0); 80 | 81 | sut.set(0, 1, 4); 82 | expect(sut.dw[1]).toBe(0); 83 | }); 84 | 85 | it('with [0, 3, 2, 4, 5, 1] populated instance >> setColumn with given values >> should mutate the values of given column', () => { 86 | const m = new Mat(2, 1); 87 | m.setFrom([10, 20]); 88 | 89 | sut.setColumn(m, 1); 90 | 91 | expectColumnToContain(1, [10, 20]); 92 | }); 93 | 94 | const expectColumnToContain = (col: number, expected: Array) => { 95 | for (let i = 0; i < expected.length; i++) { 96 | expect(sut.get(i, col)).toBe(expected[i]); 97 | } 98 | }; 99 | 100 | const expectValueAtGivenPosition = (given: any, expected: number) => { 101 | const actual = sut.get(given.row, given.col); 102 | expect(actual).toBe(expected); 103 | }; 104 | 105 | const expectValuesToBe = (expected: Array) => { 106 | for (let i = 0; i < expected.length; i++) { 107 | expect(sut.w[i]).toBe(expected[i]); 108 | } 109 | }; 110 | 111 | const expectDerivativesToBe = (expected: Array) => { 112 | for (let i = 0; i < expected.length; i++) { 113 | expect(sut.dw[i]).toBe(expected[i]); 114 | } 115 | }; 116 | }); 117 | 118 | describe('Compare:', () => { 119 | 120 | beforeEach(() => { 121 | sut = new Mat(3, 2); 122 | sut.setFrom([0, 1, 2, 3, 4, 5]); 123 | }); 124 | 125 | it('given two unequally dimensioned matrices >> equals >> should return false', () => { 126 | const mat2 = new Mat(2, 3); 127 | mat2.setFrom([0, 1, 2, 3, 4, 5]); 128 | 129 | const actual = sut.equals(mat2); 130 | 131 | expect(actual).toBe(false); 132 | }); 133 | 134 | it('given two unequally populated instances >> equals >> should return false', () => { 135 | const mat2 = new Mat(3, 2); 136 | mat2.setFrom([0, 0, 0, 0, 0, 0]); 137 | 138 | const actual = sut.equals(mat2); 139 | 140 | expect(actual).toBe(false); 141 | }); 142 | 143 | it('given two equally populated instances >> equals >> should return true', () => { 144 | const mat2 = new Mat(3, 2); 145 | mat2.setFrom([0, 1, 2, 3, 4, 5]); 146 | 147 | const actual = sut.equals(mat2); 148 | 149 | expect(actual).toBe(true); 150 | }); 151 | }); 152 | 153 | describe('Backpropagation:', () => { 154 | 155 | beforeEach(() => { 156 | sut = new Mat(2, 3); 157 | sut.setFrom([0, 1, 2, 3, 4, 5]); 158 | Utils.fillConst(sut.dw, 1); 159 | }); 160 | 161 | it('instance values populated from 1 to 5 and derivative values populated with "ones" >> update >> should decrease values by discounted derivatives', () => { 162 | sut.update(0.1); 163 | 164 | expectValuesToBe([-0.1, 0.9, 1.9, 2.9, 3.9, 4.9]); 165 | }); 166 | 167 | it('instance values populated from 1 to 5 and derivative values populated with "ones" >> update >> should reset derivative values to zero', () => { 168 | sut.update(0.1); 169 | 170 | expectDerivativesToBe([0, 0, 0, 0, 0, 0]); 171 | }); 172 | 173 | const expectValuesToBe = (expected: Array) => { 174 | for (let i = 0; i < expected.length; i++) { 175 | expect(sut.w[i]).toBe(expected[i]); 176 | } 177 | }; 178 | 179 | const expectDerivativesToBe = (expected: Array) => { 180 | for (let i = 0; i < expected.length; i++) { 181 | expect(sut.dw[i]).toBe(expected[i]); 182 | } 183 | }; 184 | }); 185 | 186 | describe('STATIC:', () => { 187 | 188 | const sut = Mat; 189 | 190 | describe('JSON:', () => { 191 | 192 | describe('fromJSON:', () => { 193 | 194 | let actual: Mat; 195 | 196 | it('json object >> fromJSON >> should return a matrix with given dimensions', () => { 197 | const json = { rows: 2, cols: 3, w: [0, 1, 2, 3, 4, 5] }; 198 | actual = sut.fromJSON(json); 199 | 200 | expect(actual.rows).toBe(2); 201 | expect(actual.cols).toBe(3); 202 | }); 203 | 204 | it('json object >> fromJSON >> should return a matrix with values populated', () => { 205 | const json = { rows: 2, cols: 3, w: [0, 1, 2, 3, 4, 5] }; 206 | actual = sut.fromJSON(json); 207 | 208 | expectValuesToBe([0, 1, 2, 3, 4, 5]); 209 | }); 210 | 211 | it('json object >> fromJSON >> should return a matrix with derivatives populated with zero', () => { 212 | const json = { rows: 2, cols: 3, w: [0, 1, 2, 3, 4, 5] }; 213 | actual = sut.fromJSON(json); 214 | 215 | expectDerivativesToBe([0, 0, 0, 0, 0, 0]); 216 | }); 217 | 218 | const expectValuesToBe = (expected: Array) => { 219 | for (let i = 0; i < expected.length; i++) { 220 | expect(actual.w[i]).toBe(expected[i]); 221 | } 222 | }; 223 | 224 | const expectDerivativesToBe = (expected: Array) => { 225 | for (let i = 0; i < expected.length; i++) { 226 | expect(actual.dw[i]).toBe(expected[i]); 227 | } 228 | }; 229 | }); 230 | 231 | describe('toJSON:', () => { 232 | 233 | let m: Mat; 234 | let actual: any; 235 | 236 | beforeEach(() => { 237 | m = new Mat(2, 3); 238 | m.setFrom([0, 1, 2, 2, 1, 0]); 239 | }); 240 | 241 | it('matrix populated with [0, 1, 2, 2, 1, 0] >> toJSON >> should return a json object with given rows and cols', () => { 242 | actual = sut.toJSON(m); 243 | 244 | expect(actual.rows).toBe(2); 245 | expect(actual.cols).toBe(3); 246 | }); 247 | 248 | it('matrix populated with [0, 1, 2, 2, 1, 0] >> toJSON >> should return a json object with values [w]', () => { 249 | actual = sut.toJSON(m); 250 | 251 | expectValuesToBe([0, 1, 2, 2, 1, 0]); 252 | }); 253 | 254 | const expectValuesToBe = (expected: Array) => { 255 | for (let i = 0; i < expected.length; i++) { 256 | expect(actual.w[i]).toBe(expected[i]); 257 | } 258 | }; 259 | }); 260 | }); 261 | }); 262 | }); 263 | -------------------------------------------------------------------------------- /src/mat.ts: -------------------------------------------------------------------------------- 1 | import { Utils } from '.'; 2 | import { Assertable } from './utils/assertable'; 3 | 4 | export class Mat extends Assertable { 5 | 6 | public readonly rows: number; 7 | public readonly cols: number; 8 | private readonly _length: number; // length of 1d-representation of Mat 9 | 10 | public readonly w: Array | Float64Array; 11 | public readonly dw: Array | Float64Array; 12 | 13 | /** 14 | * 15 | * @param rows rows of Matrix 16 | * @param cols columns of Matrix 17 | */ 18 | constructor(rows: number, cols: number) { 19 | super(); 20 | this.rows = rows; 21 | this.cols = cols; 22 | this._length = rows * cols; 23 | this.w = Utils.zeros(this._length); 24 | this.dw = Utils.zeros(this._length); 25 | } 26 | 27 | /** 28 | * Accesses the value of given row and column. 29 | * @param row 30 | * @param col 31 | * @returns the value of given row and column 32 | */ 33 | public get(row: number, col: number): number { 34 | const ix = this.getIndexBy(row, col); 35 | Mat.assert(ix >= 0 && ix < this.w.length, '[class:Mat] get: index out of bounds.'); 36 | return this.w[ix]; 37 | } 38 | 39 | /** 40 | * Mutates the value of given row and column. 41 | * @param row 42 | * @param col 43 | * @param v 44 | */ 45 | public set(row: number, col: number, v: number): void { 46 | const ix = this.getIndexBy(row, col); 47 | Mat.assert(ix >= 0 && ix < this.w.length, '[class:Mat] set: index out of bounds.'); 48 | this.w[ix] = v; 49 | } 50 | 51 | /** 52 | * Gets Index by Row-major order 53 | * @param row 54 | * @param col 55 | */ 56 | protected getIndexBy(row: number, col: number): number { 57 | return (row * this.cols) + col; 58 | } 59 | 60 | /** 61 | * Sets values according to the given Array. 62 | * @param arr 63 | */ 64 | public setFrom(arr: Array | Float64Array): void { 65 | for (let i = 0; i < arr.length; i++) { 66 | this.w[i] = arr[i]; 67 | } 68 | } 69 | 70 | /** 71 | * Overrides the values from the column of the matrix 72 | * @param m 73 | * @param colIndex 74 | */ 75 | public setColumn(m: Mat, colIndex: number): void { 76 | Mat.assert(m.w.length === this.rows, '[class:Mat] setColumn: dimensions misaligned.'); 77 | for (let i = 0; i < m.w.length; i++) { 78 | this.w[(this.cols * i) + colIndex] = m.w[i]; 79 | } 80 | } 81 | 82 | /** 83 | * Checks equality of matrices. 84 | * The check includes the value equality and a dimensionality check. 85 | * Derivatives are not considered. 86 | * @param {Mat} m Matrix to be compared with 87 | * @returns {boolean} true if equal and false otherwise 88 | */ 89 | public equals(m: Mat): boolean { 90 | if(this.rows !== m.rows || this.cols !== m.cols) { 91 | return false; 92 | } 93 | for(let i = 0; i < this._length; i++) { 94 | if(this.w[i] !== m.w[i]) { 95 | return false; 96 | } 97 | } 98 | return true; 99 | } 100 | 101 | public static toJSON(m: Mat | any): {rows, cols, w} { 102 | const json = {rows: 0, cols: 0, w: []}; 103 | json.rows = m.rows || m.n; 104 | json.cols = m.cols || m.d; 105 | json.w = m.w; 106 | return json; 107 | } 108 | 109 | public static fromJSON(json: {rows, n?, cols, d?, w}): Mat { 110 | const rows = json.rows || json.n; 111 | const cols = json.cols || json.d; 112 | const mat = new Mat(rows, cols); 113 | for (let i = 0; i < mat._length; i++) { 114 | mat.w[i] = json.w[i]; 115 | } 116 | return mat; 117 | } 118 | 119 | /** 120 | * Discounts all values as follows: w[i] = w[i] - (alpha * dw[i]) 121 | * @param alpha discount factor 122 | */ 123 | public update(alpha: number): void { 124 | for (let i = 0; i < this._length; i++) { 125 | if (this.dw[i] !== 0) { 126 | this.w[i] = this.w[i] - alpha * this.dw[i]; 127 | this.dw[i] = 0; 128 | } 129 | } 130 | } 131 | 132 | } 133 | -------------------------------------------------------------------------------- /src/net.ts: -------------------------------------------------------------------------------- 1 | import { Mat, RandMat, Graph, NetOpts } from '.'; 2 | 3 | export class Net { 4 | public W1: Mat; 5 | public b1: Mat; 6 | public W2: Mat; 7 | public b2: Mat; 8 | 9 | /** 10 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 11 | * @param {{W1, b1, W2, b2}} opt Specs of the Neural Net. 12 | */ 13 | constructor(opt: { W1, b1, W2, b2 }); 14 | /** 15 | * Generates a Neural Net with given specs. 16 | * @param {NetOpts} opt Specs of the Neural Net. 17 | */ 18 | constructor(opt: NetOpts); 19 | constructor(opt: any) { 20 | if (this.isFromJSON(opt)) { 21 | this.initializeFromJSONObject(opt); 22 | } else if (this.isFreshInstanceCall(opt)) { 23 | this.initializeAsFreshInstance(opt); 24 | } else { 25 | this.initializeAsFreshInstance({ architecture: { inputSize: 1, hiddenUnits: [1], outputSize: 1 } }); 26 | } 27 | } 28 | 29 | private isFromJSON(opt: any) { 30 | return Net.has(opt, ['W1', 'b1', 'W2', 'b2']); 31 | } 32 | 33 | private isFreshInstanceCall(opt: NetOpts) { 34 | return Net.has(opt, ['architecture']) && Net.has(opt.architecture, ['inputSize', 'hiddenUnits', 'outputSize']); 35 | } 36 | 37 | private initializeFromJSONObject(opt: { W1, b1, W2, b2 }) { 38 | this.W1 = Mat.fromJSON(opt['W1']); 39 | this.b1 = Mat.fromJSON(opt['b1']); 40 | this.W2 = Mat.fromJSON(opt['W2']); 41 | this.b2 = Mat.fromJSON(opt['b2']); 42 | } 43 | 44 | private initializeAsFreshInstance(opt: NetOpts) { 45 | let mu = 0; 46 | let std = 0.01; 47 | if(Net.has(opt, ['other'])) { 48 | mu = opt.other['mu'] ? opt.other['mu'] : mu; 49 | std = opt.other['std'] ? opt.other['std'] : std; 50 | } 51 | const firstLayer = 0; // only consider the first layer => shallowness 52 | this.W1 = new RandMat(opt.architecture['hiddenUnits'][firstLayer], opt.architecture['inputSize'], mu, std); 53 | this.b1 = new Mat(opt.architecture['hiddenUnits'][firstLayer], 1); 54 | this.W2 = new RandMat(opt.architecture['outputSize'], opt.architecture['hiddenUnits'][firstLayer], mu, std); 55 | this.b2 = new Mat(opt.architecture['outputSize'], 1); 56 | } 57 | 58 | /** 59 | * Updates all weights 60 | * @param alpha discount factor for weight updates 61 | */ 62 | public update(alpha: number): void { 63 | this.W1.update(alpha); 64 | this.b1.update(alpha); 65 | this.W2.update(alpha); 66 | this.b2.update(alpha); 67 | } 68 | 69 | public static toJSON(net: Net): {} { 70 | const json = {}; 71 | json['W1'] = Mat.toJSON(net.W1); 72 | json['b1'] = Mat.toJSON(net.b1); 73 | json['W2'] = Mat.toJSON(net.W2); 74 | json['b2'] = Mat.toJSON(net.b2); 75 | return json; 76 | } 77 | 78 | /** 79 | * Compute forward pass of Neural Network 80 | * @param state 1D column vector with observations 81 | * @param graph optional: inject Graph to append Operations 82 | * @returns output of type `Mat` 83 | */ 84 | public forward(state: Mat, graph: Graph): Mat { 85 | const weightedInput = graph.mul(this.W1, state); 86 | 87 | const a1mat = graph.add(weightedInput, this.b1); 88 | 89 | const h1mat = graph.tanh(a1mat); 90 | 91 | const a2Mat = this.computeOutput(h1mat, graph); 92 | return a2Mat; 93 | } 94 | 95 | private computeOutput(hiddenUnits: Mat, graph: Graph) { 96 | const weightedActivation = graph.mul(this.W2, hiddenUnits); 97 | // a2 = Output Vector of Weight2 (W2) and hyperbolic Activation (h1) 98 | const a2Mat = graph.add(weightedActivation, this.b2); 99 | return a2Mat; 100 | } 101 | 102 | public static fromJSON(json: { W1, b1, W2, b2 }): Net { 103 | return new Net(json); 104 | } 105 | 106 | private static has(obj: any, keys: Array) { 107 | for (const key of keys) { 108 | if (Object.hasOwnProperty.call(obj, key)) { continue; } 109 | return false; 110 | } 111 | return true; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/rand-mat.spec.ts: -------------------------------------------------------------------------------- 1 | import { RandMat, Utils, Mat } from '.'; 2 | 3 | describe('RandMat:', () => { 4 | let sut: Mat; 5 | 6 | describe('Check usage of Utils:', () => { 7 | 8 | beforeEach(() => { 9 | spyOn(Utils, 'fillRandn'); 10 | sut = new RandMat(2, 4, 0, 1); 11 | }); 12 | 13 | it('given a fresh instance >> on creation >> should have created Mat of expected size', () => { 14 | expect(sut.rows).toBe(2); 15 | expect(sut.cols).toBe(4); 16 | }); 17 | 18 | it('given a fresh instance >> on creation >> should have called Utils.fillRandn', () => { 19 | expect(Utils.fillRandn).toHaveBeenCalled(); 20 | expect(Utils.fillRandn).toHaveBeenCalledWith(sut.w, 0, 1); 21 | }); 22 | }); 23 | 24 | describe('Check population of values:', () => { 25 | 26 | beforeEach(() => { 27 | spyOn(Utils, 'randn').and.callFake((mu: number, std: number) => { return mu + std; }); 28 | }); 29 | 30 | it('given a fresh instance >> on creation >> should have populated Matrix with values', () => { 31 | sut = new RandMat(1, 3, 1, 0.123); 32 | 33 | expectMatrixToBePopulatedWith([1.123, 1.123, 1.123]); 34 | }); 35 | 36 | const expectMatrixToBePopulatedWith = (expected: Array) => { 37 | for (let i = 0; i < sut.w.length; i++) { 38 | expect(sut.w[i]).toBeCloseTo(expected[i], 5); 39 | } 40 | }; 41 | }); 42 | }); 43 | 44 | -------------------------------------------------------------------------------- /src/rand-mat.ts: -------------------------------------------------------------------------------- 1 | import { Mat, Utils } from '.'; 2 | 3 | export class RandMat extends Mat { 4 | 5 | /** 6 | * 7 | * @param rows length of Matrix 8 | * @param cols depth of Matrix 9 | * @param mu Population mean for initialization 10 | * @param std Standard deviation for initialization 11 | */ 12 | constructor(rows: number, cols: number, mu: number, std: number) { 13 | super(rows, cols); 14 | Utils.fillRandn(this.w, mu, std); 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/rnn/lstm.spec.ts: -------------------------------------------------------------------------------- 1 | import { LSTM } from '..'; 2 | 3 | describe('Long Short-Term Memory Network (LSTM):', () => { 4 | 5 | let sut: LSTM; 6 | 7 | describe('Instantiation:', () => { 8 | 9 | describe('Configuration with NetOpts:', () => { 10 | 11 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 12 | 13 | beforeEach(() => { 14 | sut = new LSTM(config); 15 | }); 16 | 17 | describe('Hidden Layer:', () => { 18 | 19 | it('fresh instance >> on creation >> model should hold hidden layer containing weight matrices with expected dimensions', () => { 20 | expectHiddenWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength(2, [3, 4]); 21 | }); 22 | 23 | it('fresh instance >> on creation >> model should hold hidden layer containing weight matrices for preceding activations with expected quadratic dimensions', () => { 24 | expectHiddenWeightMatricesForPrecedingActivationToHaveQuadraticDimensionsAccordingToHiddenUnits([3, 4]); 25 | }); 26 | 27 | it('fresh instance >> on creation >> model should hold hidden layer containing bias matrices with expected dimensions', () => { 28 | expectHiddenBiasMatricesToHaveRowsOfSizeOfPrecedingLayerAndColsOfSize1(2, [3, 4]); 29 | }); 30 | 31 | const expectHiddenWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength = (inputSize: number, hiddenUnits: Array) => { 32 | let precedingLayerSize = inputSize; 33 | let expectedRows, expectedCols; 34 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 35 | expectedRows = hiddenUnits[i]; 36 | expectedCols = precedingLayerSize; 37 | expect(sut.model.hidden.input.Wx[i].rows).toBe(expectedRows); 38 | expect(sut.model.hidden.input.Wx[i].cols).toBe(expectedCols); 39 | 40 | expect(sut.model.hidden.output.Wx[i].rows).toBe(expectedRows); 41 | expect(sut.model.hidden.output.Wx[i].cols).toBe(expectedCols); 42 | 43 | expect(sut.model.hidden.forget.Wx[i].rows).toBe(expectedRows); 44 | expect(sut.model.hidden.forget.Wx[i].cols).toBe(expectedCols); 45 | 46 | expect(sut.model.hidden.cell.Wx[i].rows).toBe(expectedRows); 47 | expect(sut.model.hidden.cell.Wx[i].cols).toBe(expectedCols); 48 | precedingLayerSize = expectedRows; 49 | } 50 | }; 51 | 52 | const expectHiddenWeightMatricesForPrecedingActivationToHaveQuadraticDimensionsAccordingToHiddenUnits = (expected: Array) => { 53 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 54 | expect(sut.model.hidden.input.Wh[i].rows).toBe(expected[i]); 55 | expect(sut.model.hidden.input.Wh[i].cols).toBe(expected[i]); 56 | 57 | expect(sut.model.hidden.output.Wh[i].rows).toBe(expected[i]); 58 | expect(sut.model.hidden.output.Wh[i].cols).toBe(expected[i]); 59 | 60 | expect(sut.model.hidden.forget.Wh[i].rows).toBe(expected[i]); 61 | expect(sut.model.hidden.forget.Wh[i].cols).toBe(expected[i]); 62 | 63 | expect(sut.model.hidden.cell.Wh[i].rows).toBe(expected[i]); 64 | expect(sut.model.hidden.cell.Wh[i].cols).toBe(expected[i]); 65 | } 66 | }; 67 | 68 | const expectHiddenBiasMatricesToHaveRowsOfSizeOfPrecedingLayerAndColsOfSize1 = (inputSize: number, hiddenUnits: Array) => { 69 | let expectedRows; 70 | const expectedCols = 1; 71 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 72 | expectedRows = hiddenUnits[i]; 73 | expect(sut.model.hidden.input.bh[i].rows).toBe(expectedRows); 74 | expect(sut.model.hidden.input.bh[i].cols).toBe(expectedCols); 75 | 76 | expect(sut.model.hidden.output.bh[i].rows).toBe(expectedRows); 77 | expect(sut.model.hidden.output.bh[i].cols).toBe(expectedCols); 78 | 79 | expect(sut.model.hidden.forget.bh[i].rows).toBe(expectedRows); 80 | expect(sut.model.hidden.forget.bh[i].cols).toBe(expectedCols); 81 | 82 | expect(sut.model.hidden.cell.bh[i].rows).toBe(expectedRows); 83 | expect(sut.model.hidden.cell.bh[i].cols).toBe(expectedCols); 84 | } 85 | }; 86 | }); 87 | }); 88 | }); 89 | 90 | describe('Backpropagation:', () => { 91 | 92 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 93 | 94 | beforeEach(() => { 95 | sut = new LSTM(config); 96 | 97 | spyOnUpdateMethods(); 98 | }); 99 | 100 | describe('Update:', () => { 101 | 102 | describe('Hidden Layer:', () => { 103 | 104 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer', () => { 105 | sut.update(0.01); 106 | 107 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 108 | }); 109 | 110 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer with given value', () => { 111 | sut.update(0.01); 112 | 113 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 114 | }); 115 | 116 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 117 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 118 | expect(sut.model.hidden.input.Wx[i].update).toHaveBeenCalled(); 119 | expect(sut.model.hidden.input.Wh[i].update).toHaveBeenCalled(); 120 | expect(sut.model.hidden.input.bh[i].update).toHaveBeenCalled(); 121 | 122 | expect(sut.model.hidden.output.Wx[i].update).toHaveBeenCalled(); 123 | expect(sut.model.hidden.output.Wh[i].update).toHaveBeenCalled(); 124 | expect(sut.model.hidden.output.bh[i].update).toHaveBeenCalled(); 125 | 126 | expect(sut.model.hidden.forget.Wx[i].update).toHaveBeenCalled(); 127 | expect(sut.model.hidden.forget.Wh[i].update).toHaveBeenCalled(); 128 | expect(sut.model.hidden.forget.bh[i].update).toHaveBeenCalled(); 129 | 130 | expect(sut.model.hidden.cell.Wx[i].update).toHaveBeenCalled(); 131 | expect(sut.model.hidden.cell.Wh[i].update).toHaveBeenCalled(); 132 | expect(sut.model.hidden.cell.bh[i].update).toHaveBeenCalled(); 133 | } 134 | }; 135 | 136 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 137 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 138 | expect(sut.model.hidden.input.Wx[i].update).toHaveBeenCalledWith(value); 139 | expect(sut.model.hidden.input.Wh[i].update).toHaveBeenCalledWith(value); 140 | expect(sut.model.hidden.input.bh[i].update).toHaveBeenCalledWith(value); 141 | 142 | expect(sut.model.hidden.output.Wx[i].update).toHaveBeenCalledWith(value); 143 | expect(sut.model.hidden.output.Wh[i].update).toHaveBeenCalledWith(value); 144 | expect(sut.model.hidden.output.bh[i].update).toHaveBeenCalledWith(value); 145 | 146 | expect(sut.model.hidden.forget.Wx[i].update).toHaveBeenCalledWith(value); 147 | expect(sut.model.hidden.forget.Wh[i].update).toHaveBeenCalledWith(value); 148 | expect(sut.model.hidden.forget.bh[i].update).toHaveBeenCalledWith(value); 149 | 150 | expect(sut.model.hidden.cell.Wx[i].update).toHaveBeenCalledWith(value); 151 | expect(sut.model.hidden.cell.Wh[i].update).toHaveBeenCalledWith(value); 152 | expect(sut.model.hidden.cell.bh[i].update).toHaveBeenCalledWith(value); 153 | } 154 | }; 155 | }); 156 | 157 | describe('Decoder Layer:', () => { 158 | 159 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer', () => { 160 | sut.update(0.01); 161 | 162 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 163 | }); 164 | 165 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer with given value', () => { 166 | sut.update(0.01); 167 | 168 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 169 | }); 170 | 171 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 172 | expect(sut.model.decoder.Wh.update).toHaveBeenCalled(); 173 | expect(sut.model.decoder.b.update).toHaveBeenCalled(); 174 | }; 175 | 176 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 177 | expect(sut.model.decoder.Wh.update).toHaveBeenCalledWith(value); 178 | expect(sut.model.decoder.b.update).toHaveBeenCalledWith(value); 179 | }; 180 | }); 181 | }); 182 | 183 | const spyOnUpdateMethods = () => { 184 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 185 | spyOn(sut.model.hidden.input.Wx[i], 'update'); 186 | spyOn(sut.model.hidden.input.Wh[i], 'update'); 187 | spyOn(sut.model.hidden.input.bh[i], 'update'); 188 | 189 | spyOn(sut.model.hidden.output.Wx[i], 'update'); 190 | spyOn(sut.model.hidden.output.Wh[i], 'update'); 191 | spyOn(sut.model.hidden.output.bh[i], 'update'); 192 | 193 | spyOn(sut.model.hidden.forget.Wx[i], 'update'); 194 | spyOn(sut.model.hidden.forget.Wh[i], 'update'); 195 | spyOn(sut.model.hidden.forget.bh[i], 'update'); 196 | 197 | spyOn(sut.model.hidden.cell.Wx[i], 'update'); 198 | spyOn(sut.model.hidden.cell.Wh[i], 'update'); 199 | spyOn(sut.model.hidden.cell.bh[i], 'update'); 200 | } 201 | 202 | spyOn(sut.model.decoder.Wh, 'update'); 203 | spyOn(sut.model.decoder.b, 'update'); 204 | }; 205 | }); 206 | }); 207 | -------------------------------------------------------------------------------- /src/rnn/lstm.ts: -------------------------------------------------------------------------------- 1 | import { RandMat, Mat, Graph, InnerState, NetOpts } from './..'; 2 | import { RNNModel } from './rnn-model'; 3 | 4 | export class LSTM extends RNNModel { 5 | /** 6 | * Generates a Neural Net instance from a pretrained Neural Net JSON. 7 | * @param {{ hidden: { input: { Wh, Wx, bh }, forget: { Wh, Wx, bh }, output: { Wh, Wx, bh }, cell: { Wh, Wx, bh } }, decoder: { Wh, b } }} opt Specs of the Neural Net. 8 | */ 9 | constructor(opt: { hidden: { input: { Wh, Wx, bh }, forget: { Wh, Wx, bh }, output: { Wh, Wx, bh }, cell: { Wh, Wx, bh } }, decoder: { Wh, b } }); 10 | /** 11 | * Generates a Neural Net with given specs. 12 | * @param {NetOpts} opt Specs of the Neural Net. [defaults to: needsBackprop = true, mu = 0, std = 0.01] 13 | */ 14 | constructor(opt: NetOpts); 15 | constructor(opt: any) { 16 | super(opt); 17 | } 18 | 19 | protected initializeNetworkModel(): { hidden: any; decoder: { Wh: Mat; b: Mat; }; } { 20 | return { 21 | hidden: { 22 | input: { 23 | Wx: new Array(this.architecture.hiddenUnits.length), 24 | Wh: new Array(this.architecture.hiddenUnits.length), 25 | bh: new Array(this.architecture.hiddenUnits.length) 26 | }, 27 | forget: { 28 | Wx: new Array(this.architecture.hiddenUnits.length), 29 | Wh: new Array(this.architecture.hiddenUnits.length), 30 | bh: new Array(this.architecture.hiddenUnits.length) 31 | }, 32 | output: { 33 | Wx: new Array(this.architecture.hiddenUnits.length), 34 | Wh: new Array(this.architecture.hiddenUnits.length), 35 | bh: new Array(this.architecture.hiddenUnits.length) 36 | }, 37 | cell: { 38 | Wx: new Array(this.architecture.hiddenUnits.length), 39 | Wh: new Array(this.architecture.hiddenUnits.length), 40 | bh: new Array(this.architecture.hiddenUnits.length) 41 | }, 42 | }, 43 | decoder: { 44 | Wh: null, 45 | b: null 46 | } 47 | }; 48 | } 49 | 50 | protected isFromJSON(opt: any) { 51 | return RNNModel.has(opt, ['hidden', 'decoder']) 52 | && RNNModel.has(opt.hidden, ['input', 'forget', 'output', 'cell']) 53 | && RNNModel.has(opt.input, ['Wh', 'Wx', 'bh']) 54 | && RNNModel.has(opt.forget, ['Wh', 'Wx', 'bh']) 55 | && RNNModel.has(opt.output, ['Wh', 'Wx', 'bh']) 56 | && RNNModel.has(opt.cell, ['Wh', 'Wx', 'bh']) 57 | && RNNModel.has(opt.decoder, ['Wh', 'b']); 58 | } 59 | 60 | protected initializeHiddenLayerFromJSON(opt: { hidden: any; decoder: { Wh: Mat; b: Mat; }; }): void { 61 | RNNModel.assert(opt.hidden.forget && opt.hidden.forget && opt.hidden.output && opt.hidden.cell, 'Wrong JSON Format to recreat Hidden Layer.'); 62 | this.isValid(opt.hidden.input); 63 | this.isValid(opt.hidden.forget); 64 | this.isValid(opt.hidden.output); 65 | this.isValid(opt.hidden.cell); 66 | 67 | for (let i = 0; i < opt.hidden.Wh.length; i++) { 68 | this.model.hidden.input.Wx = Mat.fromJSON(opt.hidden.input.Wx); 69 | this.model.hidden.input.Wh = Mat.fromJSON(opt.hidden.input.Wh); 70 | this.model.hidden.input.bh = Mat.fromJSON(opt.hidden.input.bh); 71 | this.model.hidden.forget.Wx = Mat.fromJSON(opt.hidden.Wx); 72 | this.model.hidden.forget.Wh = Mat.fromJSON(opt.hidden.Wh); 73 | this.model.hidden.forget.bh = Mat.fromJSON(opt.hidden.bh); 74 | this.model.hidden.output.Wx = Mat.fromJSON(opt.hidden.Wx); 75 | this.model.hidden.output.Wh = Mat.fromJSON(opt.hidden.Wh); 76 | this.model.hidden.output.bh = Mat.fromJSON(opt.hidden.bh); 77 | // cell write params 78 | this.model.hidden.cell.Wx = Mat.fromJSON(opt.hidden.Wx); 79 | this.model.hidden.cell.Wh = Mat.fromJSON(opt.hidden.Wh); 80 | this.model.hidden.cell.bh = Mat.fromJSON(opt.hidden.hb); 81 | } 82 | } 83 | 84 | private isValid(component: any): any { 85 | RNNModel.assert(component && !Array.isArray(component['Wx']), 'Wrong JSON Format to recreat Hidden Layer.'); 86 | RNNModel.assert(component && !Array.isArray(component['Wh']), 'Wrong JSON Format to recreat Hidden Layer.'); 87 | RNNModel.assert(component && !Array.isArray(component['bh']), 'Wrong JSON Format to recreat Hidden Layer.'); 88 | } 89 | 90 | protected initializeHiddenLayer() { 91 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 92 | // loop over hidden depths 93 | const prevSize = i === 0 ? this.architecture.inputSize : this.architecture.hiddenUnits[i - 1]; 94 | const hiddenSize = this.architecture.hiddenUnits[i]; 95 | // gate parameters 96 | this.model.hidden.input.Wx[i] = new RandMat(hiddenSize, prevSize, 0, 0.08); 97 | this.model.hidden.input.Wh[i] = new RandMat(hiddenSize, hiddenSize, 0, 0.08); 98 | this.model.hidden.input.bh[i] = new Mat(hiddenSize, 1); 99 | this.model.hidden.forget.Wx[i] = new RandMat(hiddenSize, prevSize, 0, 0.08); 100 | this.model.hidden.forget.Wh[i] = new RandMat(hiddenSize, hiddenSize, 0, 0.08); 101 | this.model.hidden.forget.bh[i] = new Mat(hiddenSize, 1); 102 | this.model.hidden.output.Wx[i] = new RandMat(hiddenSize, prevSize, 0, 0.08); 103 | this.model.hidden.output.Wh[i] = new RandMat(hiddenSize, hiddenSize, 0, 0.08); 104 | this.model.hidden.output.bh[i] = new Mat(hiddenSize, 1); 105 | // cell write params 106 | this.model.hidden.cell.Wx[i] = new RandMat(hiddenSize, prevSize, 0, 0.08); 107 | this.model.hidden.cell.Wh[i] = new RandMat(hiddenSize, hiddenSize, 0, 0.08); 108 | this.model.hidden.cell.bh[i] = new Mat(hiddenSize, 1); 109 | } 110 | } 111 | 112 | /** 113 | * Forward pass for a single tick of Neural Network 114 | * @param input 1D column vector with observations 115 | * @param previousActivationState Structure containing hidden representation ['h'] and cell memory ['c'] of type `Mat[]` from previous iteration 116 | * @param graph Optional: inject Graph to append Operations 117 | * @returns Structure containing hidden representation ['h'] and cell memory ['c'] of type `Mat[]` and output ['output'] of type `Mat` 118 | */ 119 | public forward(input: Mat, previousActivationState?: InnerState, graph?: Graph): InnerState { 120 | previousActivationState = previousActivationState ? previousActivationState : null; 121 | graph = graph ? graph : this.graph; 122 | 123 | const previousHiddenActivations = { cells: null, units: null }; 124 | previousHiddenActivations.cells = this.getPreviousCellActivationsFrom(previousActivationState); 125 | previousHiddenActivations.units = this.getPreviousHiddenUnitActivationsFrom(previousActivationState); 126 | 127 | const hiddenActivations: { units, cells } = this.computeHiddenActivations(input, previousHiddenActivations, graph); 128 | 129 | const output = this.computeOutput(hiddenActivations.units, graph); 130 | 131 | // return cell memory, hidden representation and output 132 | return { 'hiddenActivationState': hiddenActivations.units, 'cells': hiddenActivations.cells, 'output': output }; 133 | } 134 | 135 | private computeHiddenActivations(state: Mat, previousHiddenActivations: { units: Mat[], cells: Mat[] }, graph: Graph) { 136 | const hiddenActivations = { units: [], cells: [] }; 137 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 138 | const inputVector = (i === 0) ? state : hiddenActivations.units[i - 1]; // first iteration fill Observations 139 | const previousUnitActivations = previousHiddenActivations.units[i]; 140 | const previousCellActivations = previousHiddenActivations.cells[i]; 141 | // input gate 142 | const weightedStatelessInputPortion1 = graph.mul(this.model.hidden.input.Wx[i], inputVector); 143 | const weightedStatefulInputPortion1 = graph.mul(this.model.hidden.input.Wh[i], previousUnitActivations); 144 | const summedUpInput1 = graph.add(graph.add(weightedStatelessInputPortion1, weightedStatefulInputPortion1), this.model.hidden.input.bh[i]); 145 | const inputGateActivation = graph.sig(summedUpInput1); 146 | // forget gate 147 | const weightedStatelessInputPortion2 = graph.mul(this.model.hidden.forget.Wx[i], inputVector); 148 | const weightedStatefulInputPortion2 = graph.mul(this.model.hidden.forget.Wh[i], previousUnitActivations); 149 | const summedUpInput2 = graph.add(graph.add(weightedStatelessInputPortion2, weightedStatefulInputPortion2), this.model.hidden.forget.bh[i]); 150 | const forgetGateActivation = graph.sig(summedUpInput2); 151 | // output gate 152 | const weightedStatelessInputPortion3 = graph.mul(this.model.hidden.output.Wx[i], inputVector); 153 | const weightedStatefulInputPortion3 = graph.mul(this.model.hidden.output.Wh[i], previousUnitActivations); 154 | const summedUpInput3 = graph.add(graph.add(weightedStatelessInputPortion3, weightedStatefulInputPortion3), this.model.hidden.output.bh[i]); 155 | const outputGateActivation = graph.sig(summedUpInput3); 156 | // write operation on cells 157 | const weightedStatelessInputPortion4 = graph.mul(this.model.hidden.cell.Wx[i], inputVector); 158 | const weightedStatefulInputPortion4 = graph.mul(this.model.hidden.cell.Wh[i], previousUnitActivations); 159 | const summedUpInput4 = graph.add(graph.add(weightedStatelessInputPortion4, weightedStatefulInputPortion4), this.model.hidden.cell.bh[i]); 160 | const cellWriteActivation = graph.tanh(summedUpInput4); 161 | // compute new cell activation 162 | const retainCell = graph.eltmul(forgetGateActivation, previousCellActivations); // what do we keep from cell 163 | const writeCell = graph.eltmul(inputGateActivation, cellWriteActivation); // what do we write to cell 164 | const cellActivations = graph.add(retainCell, writeCell); // new cell contents 165 | // compute hidden state as gated, saturated cell activations 166 | const activations = graph.eltmul(outputGateActivation, graph.tanh(cellActivations)); 167 | hiddenActivations.cells.push(cellActivations); 168 | hiddenActivations.units.push(activations); 169 | } 170 | return hiddenActivations; 171 | } 172 | 173 | private getPreviousCellActivationsFrom(previousActivationState: InnerState): Mat[] { 174 | let previousCellsActivations; 175 | if (this.givenPreviousActivationState(previousActivationState)) { 176 | previousCellsActivations = previousActivationState.cells; 177 | } 178 | else { 179 | previousCellsActivations = new Array(); 180 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 181 | previousCellsActivations.push(new Mat(this.architecture.hiddenUnits[i], 1)); 182 | } 183 | } 184 | return previousCellsActivations; 185 | } 186 | 187 | private getPreviousHiddenUnitActivationsFrom(previousActivationState: InnerState): Mat[] { 188 | let previousHiddenActivations; 189 | if (this.givenPreviousActivationState(previousActivationState)) { 190 | previousHiddenActivations = previousActivationState.hiddenActivationState; 191 | } 192 | else { 193 | previousHiddenActivations = new Array(); 194 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 195 | previousHiddenActivations.push(new Mat(this.architecture.hiddenUnits[i], 1)); 196 | } 197 | } 198 | return previousHiddenActivations; 199 | } 200 | 201 | private givenPreviousActivationState(previousInnerState: InnerState) { 202 | return previousInnerState && typeof previousInnerState.hiddenActivationState !== 'undefined'; 203 | } 204 | 205 | protected updateHiddenUnits(alpha: number): void { 206 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 207 | this.model.hidden.input.Wx[i].update(alpha); 208 | this.model.hidden.input.Wh[i].update(alpha); 209 | this.model.hidden.input.bh[i].update(alpha); 210 | 211 | this.model.hidden.output.Wx[i].update(alpha); 212 | this.model.hidden.output.Wh[i].update(alpha); 213 | this.model.hidden.output.bh[i].update(alpha); 214 | 215 | this.model.hidden.forget.Wx[i].update(alpha); 216 | this.model.hidden.forget.Wh[i].update(alpha); 217 | this.model.hidden.forget.bh[i].update(alpha); 218 | 219 | this.model.hidden.cell.Wx[i].update(alpha); 220 | this.model.hidden.cell.Wh[i].update(alpha); 221 | this.model.hidden.cell.bh[i].update(alpha); 222 | } 223 | } 224 | 225 | protected updateDecoder(alpha: number): void { 226 | this.model.decoder.Wh.update(alpha); 227 | this.model.decoder.b.update(alpha); 228 | } 229 | } 230 | -------------------------------------------------------------------------------- /src/rnn/rnn-model.spec.ts: -------------------------------------------------------------------------------- 1 | import { RNN } from '../.'; 2 | 3 | /** 4 | * Tests are being executed on a RNN instance. 5 | * RNN Class fully delegates instantiation to FNNModel Class. 6 | */ 7 | describe('Recurrent Neural Network Model:', () => { 8 | 9 | let sut: RNN; 10 | 11 | describe('Instantiation:', () => { 12 | 13 | describe('Configuration with NetOpts:', () => { 14 | 15 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 16 | 17 | beforeEach(() => { 18 | sut = new RNN(config); 19 | }); 20 | 21 | describe('Hidden Layer:', () => { 22 | // Hidden Layer are responsibility of concrete RNN implementations 23 | }); 24 | 25 | describe('Decoder Layer:', () => { 26 | 27 | it('fresh instance >> on creation >> model should hold decoder layer containing weight matrix with given dimensions', () => { 28 | sut = new RNN(config); 29 | 30 | expectDecoderWeightMatrixToHaveDimensionsOf(3, 4); 31 | }); 32 | 33 | it('fresh instance >> on creation >> model should hold decoder layer containing bias matrix with given dimensions', () => { 34 | sut = new RNN(config); 35 | 36 | expectDecoderBiasMatrixToHaveDimensionsOf(3, 1); 37 | }); 38 | 39 | const expectDecoderWeightMatrixToHaveDimensionsOf = (expectedRows: number, expectedCols: number) => { 40 | expect(sut.model.decoder.Wh.rows).toBe(expectedRows); 41 | expect(sut.model.decoder.Wh.cols).toBe(expectedCols); 42 | }; 43 | 44 | const expectDecoderBiasMatrixToHaveDimensionsOf = (expectedRows: number, expectedCols: number) => { 45 | expect(sut.model.decoder.b.rows).toBe(expectedRows); 46 | expect(sut.model.decoder.b.cols).toBe(expectedCols); 47 | }; 48 | }); 49 | }); 50 | 51 | describe('Configuration with JSON Object', () => { 52 | 53 | }); 54 | }); 55 | }); 56 | -------------------------------------------------------------------------------- /src/rnn/rnn-model.ts: -------------------------------------------------------------------------------- 1 | import { Graph, Mat, RandMat, InnerState, NetOpts } from './..'; 2 | import { Assertable } from './../utils/assertable'; 3 | 4 | export abstract class RNNModel extends Assertable { 5 | 6 | protected architecture: { inputSize: number, hiddenUnits: Array, outputSize: number }; 7 | 8 | public model: { hidden: any, decoder: { Wh: Mat, b: Mat } }; 9 | 10 | protected graph: Graph; 11 | 12 | /** 13 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 14 | * @param {{ hidden: any, decoder: { Wh: Mat, b: Mat } }} opt Specs of the Neural Net. 15 | */ 16 | constructor(opt: { hidden: any, decoder: { Wh: Mat, b: Mat } }); 17 | /** 18 | * Generates a Neural Net with given specs. 19 | * @param {NetOpts} opt Specs of the Neural Net. 20 | */ 21 | constructor(opt: NetOpts); 22 | constructor(opt: any) { 23 | super(); 24 | const needsBackpropagation = opt && opt.needsBackpropagation ? opt.needsBackpropagation : true; 25 | 26 | this.graph = new Graph(); 27 | this.graph.memorizeOperationSequence(true); 28 | 29 | if (this.isFromJSON(opt)) { 30 | this.initializeModelFromJSONObject(opt); 31 | } else if (this.isFreshInstanceCall(opt)) { 32 | this.initializeModelAsFreshInstance(opt); 33 | } else { 34 | RNNModel.assert(false, 'Improper input for DNN.'); 35 | } 36 | } 37 | 38 | protected abstract isFromJSON(opt: any): boolean; 39 | 40 | protected initializeModelFromJSONObject(opt: { hidden: any, decoder: { Wh: Mat, b: Mat } }): void { 41 | this.initializeHiddenLayerFromJSON(opt); 42 | this.model.decoder.Wh = Mat.fromJSON(opt['decoder']['Wh']); 43 | this.model.decoder.b = Mat.fromJSON(opt['decoder']['b']); 44 | } 45 | 46 | protected abstract initializeHiddenLayerFromJSON(opt: { hidden: any, decoder: { Wh: Mat, b: Mat } }): void; 47 | 48 | private isFreshInstanceCall(opt: NetOpts): boolean { 49 | return RNNModel.has(opt, ['architecture']) && RNNModel.has(opt.architecture, ['inputSize', 'hiddenUnits', 'outputSize']); 50 | } 51 | 52 | private initializeModelAsFreshInstance(opt: NetOpts): void { 53 | this.architecture = opt.architecture; 54 | 55 | const mu = opt['mu'] ? opt['mu'] : 0; 56 | const std = opt['std'] ? opt['std'] : 0.01; 57 | 58 | this.model = this.initializeNetworkModel(); 59 | 60 | this.initializeHiddenLayer(mu, std); 61 | 62 | this.initializeDecoder(mu, std); 63 | } 64 | 65 | protected abstract initializeNetworkModel(): { hidden: any; decoder: { Wh: Mat; b: Mat; }; }; 66 | 67 | protected abstract initializeHiddenLayer(mu: number, std: number): void; 68 | 69 | protected initializeDecoder(mu: number, std: number): void { 70 | this.model.decoder.Wh = new RandMat(this.architecture.outputSize, this.architecture.hiddenUnits[this.architecture.hiddenUnits.length - 1], mu, std); 71 | this.model.decoder.b = new Mat(this.architecture.outputSize, 1); 72 | } 73 | 74 | public abstract forward(input: Mat, previousActivationState?: InnerState, graph?: Graph): InnerState; 75 | 76 | /** 77 | * Updates all weights depending on their specific gradients 78 | * @param alpha discount factor for weight updates 79 | * @returns {void} 80 | */ 81 | public update(alpha: number): void { 82 | this.updateHiddenUnits(alpha); 83 | this.updateDecoder(alpha); 84 | } 85 | 86 | protected abstract updateHiddenUnits(alpha: number): void; 87 | protected abstract updateDecoder(alpha: number): void; 88 | 89 | protected computeOutput(hiddenActivations: Mat[], graph: Graph): Mat { 90 | const precedingHiddenLayerActivations = hiddenActivations[hiddenActivations.length - 1]; 91 | const weightedInputs = graph.mul(this.model.decoder.Wh, precedingHiddenLayerActivations); 92 | return graph.add(weightedInputs, this.model.decoder.b); 93 | } 94 | 95 | protected static has(obj: any, keys: Array): boolean { 96 | RNNModel.assert(obj, '[class:rnn-model] improper input for instantiation'); 97 | for (const key of keys) { 98 | if (Object.hasOwnProperty.call(obj, key)) { continue; } 99 | return false; 100 | } 101 | return true; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/rnn/rnn.spec.ts: -------------------------------------------------------------------------------- 1 | import { RNN, Mat, NetOpts, Utils } from '..'; 2 | 3 | describe('Deep Recurrent Neural Network (RNN):', () => { 4 | 5 | let sut: RNN; 6 | 7 | describe('Instantiation:', () => { 8 | 9 | describe('Configuration with NetOpts:', () => { 10 | 11 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 12 | 13 | beforeEach(() => { 14 | sut = new RNN(config); 15 | }); 16 | 17 | it('fresh instance >> on creation >> should hold model with hidden layer, containing arrays of weight and bias matrices', () => { 18 | expect(sut.model).toBeDefined(); 19 | expect(sut.model.hidden).toBeDefined(); 20 | expect(sut.model.hidden.Wh).toBeDefined(); 21 | expect(sut.model.hidden.Wh.length).toBe(2); 22 | expect(sut.model.hidden.Wx).toBeDefined(); 23 | expect(sut.model.hidden.Wx.length).toBe(2); 24 | expect(sut.model.hidden.bh).toBeDefined(); 25 | expect(sut.model.hidden.bh.length).toBe(2); 26 | }); 27 | 28 | it('fresh instance >> on creation >> should hold model with decoder layer, containing weight and bias matrices', () => { 29 | expect(sut.model.decoder).toBeDefined(); 30 | expect(sut.model.decoder.Wh).toBeDefined(); 31 | expect(sut.model.decoder.b).toBeDefined(); 32 | }); 33 | 34 | describe('Hidden Layer:', () => { 35 | 36 | it('fresh instance >> on creation >> model should hold hidden layer containing weight matrices for stateless connections with expected dimensions', () => { 37 | expectHiddenStatelessWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength(2, [3, 4]); 38 | }); 39 | 40 | it('fresh instance >> on creation >> model should hold hidden layer containing weight matrices for stateful connections with expected dimensions', () => { 41 | expectHiddenStatefulWeightMatricesToHaveSquaredDimensions(2, [3, 4]); 42 | }); 43 | 44 | it('fresh instance >> on creation >> model should hold hidden layer containing bias matrices with expected dimensions', () => { 45 | expectHiddenBiasMatricesToBeVectorWithRowsOfSizeOfPrecedingLayer(2, [3, 4]); 46 | }); 47 | 48 | const expectHiddenStatelessWeightMatricesToHaveColsOfSizeOfPrecedingLayerAndRowsOfConfiguredLength = (inputSize: number, hiddenUnits: Array) => { 49 | let precedingLayerSize = inputSize; 50 | let expectedRows, expectedCols; 51 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 52 | expectedRows = hiddenUnits[i]; 53 | expectedCols = precedingLayerSize; 54 | expect(sut.model.hidden.Wx[i].rows).toBe(expectedRows); 55 | expect(sut.model.hidden.Wx[i].cols).toBe(expectedCols); 56 | precedingLayerSize = expectedRows; 57 | } 58 | }; 59 | 60 | const expectHiddenStatefulWeightMatricesToHaveSquaredDimensions = (inputSize: number, hiddenUnits: Array) => { 61 | let expectedRows, expectedCols; 62 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 63 | expectedRows = expectedCols = hiddenUnits[i]; 64 | expect(sut.model.hidden.Wh[i].rows).toBe(expectedRows); 65 | expect(sut.model.hidden.Wh[i].cols).toBe(expectedCols); 66 | } 67 | }; 68 | 69 | const expectHiddenBiasMatricesToBeVectorWithRowsOfSizeOfPrecedingLayer = (inputSize: number, hiddenUnits: Array) => { 70 | let expectedRows; 71 | const expectedCols = 1; 72 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 73 | expectedRows = hiddenUnits[i]; 74 | expect(sut.model.hidden.bh[i].rows).toBe(expectedRows); 75 | expect(sut.model.hidden.bh[i].cols).toBe(expectedCols); 76 | } 77 | }; 78 | }); 79 | }); 80 | }); 81 | 82 | describe('Backpropagation:', () => { 83 | 84 | const config = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 85 | 86 | beforeEach(() => { 87 | sut = new RNN(config); 88 | 89 | spyOnUpdateMethods(); 90 | }); 91 | 92 | describe('Update:', () => { 93 | 94 | describe('Hidden Layer:', () => { 95 | 96 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer', () => { 97 | sut.update(0.01); 98 | 99 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 100 | }); 101 | 102 | it('fresh instance >> update >> should call update methods of weight and bias matrices of all hidden layer with given value', () => { 103 | sut.update(0.01); 104 | 105 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 106 | }); 107 | 108 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 109 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 110 | expect(sut.model.hidden.Wx[i].update).toHaveBeenCalled(); 111 | expect(sut.model.hidden.Wh[i].update).toHaveBeenCalled(); 112 | expect(sut.model.hidden.bh[i].update).toHaveBeenCalled(); 113 | } 114 | }; 115 | 116 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 117 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 118 | expect(sut.model.hidden.Wx[i].update).toHaveBeenCalledWith(value); 119 | expect(sut.model.hidden.Wh[i].update).toHaveBeenCalledWith(value); 120 | expect(sut.model.hidden.bh[i].update).toHaveBeenCalledWith(value); 121 | } 122 | }; 123 | }); 124 | 125 | describe('Decoder Layer:', () => { 126 | 127 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer', () => { 128 | sut.update(0.01); 129 | 130 | expectUpdateOfLayersMethodsToHaveBeenCalled(); 131 | }); 132 | 133 | it('fresh instance >> update >> should call update methods of weight and bias matrices of decoder layer with given value', () => { 134 | sut.update(0.01); 135 | 136 | expectUpdateOfLayersMethodsToHaveBeenCalledWithValue(0.01); 137 | }); 138 | 139 | const expectUpdateOfLayersMethodsToHaveBeenCalled = () => { 140 | expect(sut.model.decoder.Wh.update).toHaveBeenCalled(); 141 | expect(sut.model.decoder.b.update).toHaveBeenCalled(); 142 | }; 143 | 144 | const expectUpdateOfLayersMethodsToHaveBeenCalledWithValue = (value: number) => { 145 | expect(sut.model.decoder.Wh.update).toHaveBeenCalledWith(value); 146 | expect(sut.model.decoder.b.update).toHaveBeenCalledWith(value); 147 | }; 148 | }); 149 | }); 150 | 151 | const spyOnUpdateMethods = () => { 152 | for (let i = 0; i < config.architecture.hiddenUnits.length; i++) { 153 | spyOn(sut.model.hidden.Wx[i], 'update'); 154 | spyOn(sut.model.hidden.Wh[i], 'update'); 155 | spyOn(sut.model.hidden.bh[i], 'update'); 156 | } 157 | 158 | spyOn(sut.model.decoder.Wh, 'update'); 159 | spyOn(sut.model.decoder.b, 'update'); 160 | }; 161 | }); 162 | 163 | describe('Forward Pass:', () => { 164 | 165 | const netOpts: NetOpts = { architecture: { inputSize: 2, hiddenUnits: [3, 4], outputSize: 3 } }; 166 | let sut: RNN; 167 | let input: Mat; 168 | 169 | beforeEach(()=> { 170 | patchFillRandn(); 171 | input = new Mat(2, 1); 172 | input.setFrom([1, 0]); 173 | sut = new RNN(netOpts); 174 | }); 175 | 176 | describe('Stateless:', () => { 177 | 178 | it('given fresh instance with some input vector and no previous inner state >> forward pass >> should return out.output with given dimensions', () => { 179 | 180 | const out = sut.forward(input); 181 | 182 | expect(out.output.rows).toBe(3); 183 | expect(out.output.cols).toBe(1); 184 | }); 185 | 186 | it('given fresh instance with some input vector and no previous inner state >> forward pass >> should return out.hiddenActivationState with given dimensions', () => { 187 | 188 | const out = sut.forward(input); 189 | 190 | expect(out.hiddenActivationState[0].rows).toBe(3); 191 | expect(out.hiddenActivationState[0].cols).toBe(1); 192 | expect(out.hiddenActivationState[1].rows).toBe(4); 193 | expect(out.hiddenActivationState[1].cols).toBe(1); 194 | }); 195 | 196 | it('given fresh instance with some input vector and no previous inner state >> forward pass >> should return out.output with expected results', () => { 197 | 198 | const out = sut.forward(input); 199 | 200 | expect(out.output.w[0]).toBe(12); 201 | expect(out.output.w[1]).toBe(12); 202 | expect(out.output.w[2]).toBe(12); 203 | }); 204 | }); 205 | 206 | const patchFillRandn = () => { 207 | spyOn(Utils, 'fillRandn').and.callFake(fakeFillRandn); 208 | }; 209 | 210 | const fakeFillRandn = (arr) => { 211 | Utils.fillConst(arr, 1); 212 | }; 213 | }); 214 | }); 215 | -------------------------------------------------------------------------------- /src/rnn/rnn.ts: -------------------------------------------------------------------------------- 1 | import { RandMat, Mat, Graph, InnerState, NetOpts } from './..'; 2 | import { RNNModel } from './rnn-model'; 3 | 4 | export class RNN extends RNNModel { 5 | /** 6 | * Generates a Neural Net instance from a pre-trained Neural Net JSON. 7 | * @param {{ hidden: { Wh, Wx, bh }, decoder: { Wh, b } }} opt Specs of the Neural Net. 8 | */ 9 | constructor(opt: { hidden: { Wh, Wx, bh }, decoder: { Wh, b } }); 10 | /** 11 | * Generates a Neural Net with given specs. 12 | * @param {NetOpts} opt Specs of the Neural Net. [defaults to: needsBackprop = true, mu = 0, std = 0.01] 13 | */ 14 | constructor(opt: NetOpts); 15 | constructor(opt: any) { 16 | super(opt); 17 | } 18 | 19 | protected isFromJSON(opt: any): boolean { 20 | return RNNModel.has(opt, ['hidden', 'decoder']) 21 | && RNNModel.has(opt.hidden, ['Wh', 'Wx', 'bh']) 22 | && RNNModel.has(opt.decoder, ['Wh', 'b']); 23 | } 24 | 25 | protected initializeHiddenLayerFromJSON(opt: { hidden: { Wh: Mat[], Wx: Mat[], bh: Mat[] }, decoder: { Wh: Mat, b: Mat } }): void { 26 | RNNModel.assert(!Array.isArray(opt['hidden']['Wh']), 'Wrong JSON Format to recreate Hidden Layer.'); 27 | RNNModel.assert(!Array.isArray(opt['hidden']['Wx']), 'Wrong JSON Format to recreate Hidden Layer.'); 28 | RNNModel.assert(!Array.isArray(opt['hidden']['bh']), 'Wrong JSON Format to recreate Hidden Layer.'); 29 | for (let i = 0; i < opt.hidden.Wh.length; i++) { 30 | this.model.hidden.Wx[i] = Mat.fromJSON(opt.hidden.Wx[i]); 31 | this.model.hidden.Wh[i] = Mat.fromJSON(opt.hidden.Wh[i]); 32 | this.model.hidden.bh[i] = Mat.fromJSON(opt.hidden.bh[i]); 33 | } 34 | } 35 | 36 | protected initializeNetworkModel(): { hidden: any; decoder: { Wh: Mat; b: Mat; }; } { 37 | return { 38 | hidden: { 39 | Wx: new Array(this.architecture.hiddenUnits.length), 40 | Wh: new Array(this.architecture.hiddenUnits.length), 41 | bh: new Array(this.architecture.hiddenUnits.length) 42 | }, 43 | decoder: { 44 | Wh: null, 45 | b: null 46 | } 47 | }; 48 | } 49 | 50 | protected initializeHiddenLayer(): void { 51 | let hiddenSize; 52 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 53 | const previousSize = i === 0 ? this.architecture.inputSize : this.architecture.hiddenUnits[i - 1]; 54 | hiddenSize = this.architecture.hiddenUnits[i]; 55 | this.model.hidden.Wx[i] = new RandMat(hiddenSize, previousSize, 0, 0.08); 56 | this.model.hidden.Wh[i] = new RandMat(hiddenSize, hiddenSize, 0, 0.08); 57 | this.model.hidden.bh[i] = new Mat(hiddenSize, 1); 58 | } 59 | } 60 | 61 | /** 62 | * Forward pass for a single tick of Neural Network 63 | * @param input 1D column vector with observations 64 | * @param previousActivationState Structure containing hidden representation ['h'] of type `Mat[]` from previous iteration 65 | * @param graph optional: inject Graph to append Operations 66 | * @returns Structure containing hidden representation ['h'] of type `Mat[]` and output ['output'] of type `Mat` 67 | */ 68 | forward(input: Mat, previousActivationState?: InnerState, graph?: Graph): InnerState { 69 | previousActivationState = previousActivationState ? previousActivationState : null; 70 | graph = graph ? graph : this.graph; 71 | 72 | const previousHiddenActivations = this.getPreviousHiddenActivationsFrom(previousActivationState); 73 | 74 | const hiddenActivations = this.computeHiddenActivations(input, previousHiddenActivations, graph); 75 | 76 | const output = this.computeOutput(hiddenActivations, graph); 77 | 78 | // return hidden representation and output 79 | return { 'hiddenActivationState': hiddenActivations, 'output': output }; 80 | } 81 | 82 | private getPreviousHiddenActivationsFrom(previousActivationState: InnerState): Mat[] { 83 | let previousHiddenActivations; 84 | if (this.givenPreviousActivationState(previousActivationState)) { 85 | previousHiddenActivations = previousActivationState.hiddenActivationState; 86 | } else { 87 | previousHiddenActivations = new Array(); 88 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 89 | previousHiddenActivations.push(new Mat(this.architecture.hiddenUnits[i], 1)); 90 | } 91 | } 92 | return previousHiddenActivations; 93 | } 94 | 95 | private givenPreviousActivationState(previousActivationState: InnerState) { 96 | return previousActivationState && typeof previousActivationState.hiddenActivationState !== 'undefined'; 97 | } 98 | 99 | private computeHiddenActivations(input: Mat, previousHiddenActivations: Mat[], graph: Graph): Mat[] { 100 | const hiddenActivations = new Array(); 101 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 102 | const inputVector = i === 0 ? input : hiddenActivations[i - 1]; 103 | const previousActivations = previousHiddenActivations[i]; 104 | const weightedStatelessInputPortion = graph.mul(this.model.hidden.Wx[i], inputVector); 105 | const weightedStatefulInputPortion = graph.mul(this.model.hidden.Wh[i], previousActivations); 106 | const activation = graph.relu(graph.add(graph.add(weightedStatelessInputPortion, weightedStatefulInputPortion), this.model.hidden.bh[i])); 107 | hiddenActivations.push(activation); 108 | } 109 | return hiddenActivations; 110 | } 111 | 112 | protected updateHiddenUnits(alpha: number): void { 113 | for (let i = 0; i < this.architecture.hiddenUnits.length; i++) { 114 | this.model.hidden.Wx[i].update(alpha); 115 | this.model.hidden.Wh[i].update(alpha); 116 | this.model.hidden.bh[i].update(alpha); 117 | } 118 | } 119 | 120 | protected updateDecoder(alpha: number): void { 121 | this.model.decoder.Wh.update(alpha); 122 | this.model.decoder.b.update(alpha); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/solver.ts: -------------------------------------------------------------------------------- 1 | import { Mat } from '.'; 2 | import { RNNModel } from './rnn/rnn-model'; 3 | 4 | export class Solver { 5 | protected readonly decayRate: number; 6 | protected readonly smoothEps: number; 7 | protected readonly stepCache: {}; 8 | 9 | protected stepTotalNumber: number; 10 | protected stepNumberOfClippings: number; 11 | 12 | constructor(decayRate: number = 0.999, smoothEps: number = 1e-8) { 13 | this.decayRate = decayRate; 14 | this.smoothEps = smoothEps; 15 | this.stepCache = {}; 16 | } 17 | 18 | private reset(): void { 19 | this.stepNumberOfClippings = 0; 20 | this.stepTotalNumber = 0; 21 | } 22 | 23 | /** 24 | * Performs a RMSprop parameter update of Model. 25 | * @param stepSize 26 | * @param l2Regularization 27 | * @param clippingValue Gradient clipping 28 | * @returns an Object containing the Clipping Ratio 29 | */ 30 | public step(model: RNNModel, stepSize: number, l2Regularization: number, clippingValue: number): { 'ratioClipped': number } { 31 | this.reset(); 32 | const solverStats = { ratioClipped: 0 }; 33 | 34 | for (const key in model) { 35 | if (model.hasOwnProperty(key)) { 36 | this.iterateModelLayer(model, key, clippingValue, l2Regularization, stepSize); 37 | } 38 | } 39 | 40 | solverStats.ratioClipped = this.stepNumberOfClippings * 1.0 / this.stepTotalNumber; 41 | return solverStats; 42 | } 43 | 44 | private iterateModelLayer(model: RNNModel, key: any, clipval: number, regc: number, stepSize: number): void { 45 | const currentModelLayer = model[key]; 46 | if (!(this.stepCache.hasOwnProperty(key))) { 47 | this.stepCache[key] = new Mat(currentModelLayer.n, currentModelLayer.d); 48 | } 49 | 50 | const currentStepCache = this.stepCache[key]; 51 | for (let i = 0; i < currentModelLayer.w.length; i++) { 52 | let mdwi = this.RMSprop(currentModelLayer, i, currentStepCache); 53 | mdwi = this.gradientClipping(mdwi, clipval); 54 | this.update(currentModelLayer, i, stepSize, mdwi, currentStepCache, regc); 55 | this.resetGradients(currentModelLayer, i); 56 | } 57 | } 58 | 59 | /** 60 | * rmsprop with adaptive learning rates 61 | * RMSprop decay the past accumulated gradient, 62 | * so only a portion of past gradients are considered. 63 | * Now, instead of considering all of the past gradients, 64 | * RMSprop behaves like moving average. 65 | * (https://wiseodd.github.io/techblog/2016/06/22/nn-optimization/) 66 | */ 67 | private RMSprop(modelLayer: Mat, i: number, stepCache: Mat): number { 68 | const mdwi = modelLayer.dw[i]; 69 | stepCache.w[i] = stepCache.w[i] * this.decayRate + (1.0 - this.decayRate) * mdwi * mdwi; 70 | return mdwi; 71 | } 72 | 73 | /** 74 | * 75 | * @param mdwi 76 | * @param clipval 77 | */ 78 | private gradientClipping(mdwi: number, clipval: number): number { 79 | if (mdwi > clipval) { 80 | mdwi = clipval; 81 | this.stepNumberOfClippings++; 82 | } 83 | else if (mdwi < -clipval) { 84 | mdwi = -clipval; 85 | this.stepNumberOfClippings++; 86 | } 87 | this.stepTotalNumber++; 88 | return mdwi; 89 | } 90 | 91 | /** 92 | * updates and regularizes 93 | * @param m 94 | * @param i 95 | * @param stepSize 96 | * @param mdwi 97 | * @param s 98 | * @param regc 99 | */ 100 | private update(m: Mat, i: number, stepSize: number, mdwi: number, stepCache: Mat, regc: number): void { 101 | m.w[i] += -stepSize * mdwi / Math.sqrt(stepCache.w[i] + this.smoothEps) - regc * m.w[i]; 102 | } 103 | 104 | /** 105 | * resets the gradients for the next iteration 106 | * @param currentModelLayer 107 | * @param i 108 | */ 109 | private resetGradients(currentModelLayer: any, i: number) { 110 | currentModelLayer.dw[i] = 0; 111 | } 112 | } 113 | -------------------------------------------------------------------------------- /src/utils.spec.ts: -------------------------------------------------------------------------------- 1 | import { Utils } from '.'; 2 | 3 | describe('Utils:', () => { 4 | 5 | const sut = Utils; 6 | 7 | describe('Public Methods:', () => { 8 | 9 | describe('Random Number Functions:', () => { 10 | 11 | describe('Gaussian Random Number Generator (gaussRandom):', () => { 12 | 13 | let actual = { min: null, max: null, std: null, var: null, mean: null }; 14 | 15 | it('Unpatched random number generator >> 100000 iterations of gaussRandom (private function) >> should comply to given statistical restrictions', () => { 16 | const actualSamples = []; 17 | 18 | for (let i = 0; i < 100000; i++) { 19 | actualSamples.push(sut['gaussRandom']()); 20 | } 21 | 22 | actual = determineBasicStatistics(actualSamples); 23 | 24 | expect(actual.min).toBeGreaterThan(-5, 'Min'); 25 | expect(actual.min).toBeLessThan(-3.5, 'Min'); 26 | expect(actual.max).toBeGreaterThan(3.5, 'Max'); 27 | expect(actual.max).toBeLessThan(5, 'Max'); 28 | expect(actual.mean).toBeGreaterThan(-0.1, 'Mean'); 29 | expect(actual.mean).toBeLessThan(0.1, 'Mean'); 30 | expect(actual.std).toBeGreaterThan(0.9, 'Std'); 31 | expect(actual.std).toBeLessThan(1.1, 'Std'); 32 | expect(actual.var).toBeGreaterThan(0.9, 'Var'); 33 | expect(actual.var).toBeLessThan(1.1, 'Var'); 34 | }); 35 | 36 | it('Unpatched random number generator >> box_muller (private function) >> should comply to given statistical restrictions', () => { 37 | const actualSamples = []; 38 | 39 | for (let i = 0; i < 100000; i++) { 40 | actualSamples.push(sut['box_muller']()); 41 | } 42 | 43 | actual = determineBasicStatistics(actualSamples); 44 | 45 | expect(actual.min).toBeGreaterThan(0, 'Min'); 46 | expect(actual.min).toBeLessThan(0.12, 'Min'); 47 | expect(actual.max).toBeGreaterThan(0.87, 'Max'); 48 | expect(actual.max).toBeLessThan(1, 'Max'); 49 | expect(actual.mean).toBeGreaterThan(0.49, 'Mean'); 50 | expect(actual.mean).toBeLessThan(0.51, 'Mean'); 51 | expect(actual.std).toBeGreaterThan(0.09, 'Std'); 52 | expect(actual.std).toBeLessThan(0.11, 'Std'); 53 | expect(actual.var).toBeGreaterThan(0.009, 'Var'); 54 | expect(actual.var).toBeLessThan(0.011, 'Var'); 55 | }); 56 | }); 57 | 58 | describe('With Unpatched Random Number Generator:', () => { 59 | 60 | let actual = { min: null, max: null, std: null, var: null, mean: null }; 61 | 62 | it('no arrangement >> randn >> should comply to given statistical restrictions', () => { 63 | const actualSamples = []; 64 | 65 | for (let i = 0; i < 100000; i++) { 66 | actualSamples.push(sut.randn(0, 1)); 67 | } 68 | actual = determineBasicStatistics(actualSamples); 69 | 70 | expect(actual.min).toBeGreaterThan(-5, 'Min'); 71 | expect(actual.min).toBeLessThan(-3.5, 'Min'); 72 | expect(actual.max).toBeGreaterThan(3.5, 'Max'); 73 | expect(actual.max).toBeLessThan(5, 'Max'); 74 | expect(actual.mean).toBeGreaterThan(-0.1, 'Mean'); 75 | expect(actual.mean).toBeLessThan(0.1, 'Mean'); 76 | expect(actual.std).toBeGreaterThan(0.9, 'Std'); 77 | expect(actual.std).toBeLessThan(1.1, 'Std'); 78 | expect(actual.var).toBeGreaterThan(0.9, 'Var'); 79 | expect(actual.var).toBeLessThan(1.1, 'Var'); 80 | }); 81 | 82 | it('no arrangement >> randf >> should comply to given statistical restrictions', () => { 83 | const actualSamples = []; 84 | 85 | for (let i = 0; i < 100000; i++) { 86 | actualSamples.push(sut.randf(0, 100)); 87 | } 88 | actual = determineBasicStatistics(actualSamples); 89 | 90 | expect(actual.min).toBeGreaterThan(0, 'Min'); 91 | expect(actual.min).toBeLessThan(0.01, 'Min'); 92 | expect(actual.max).toBeGreaterThan(99.9, 'Max'); 93 | expect(actual.max).toBeLessThan(100, 'Max'); 94 | expect(actual.mean).toBeGreaterThan(49.7, 'Mean'); 95 | expect(actual.mean).toBeLessThan(50.3, 'Mean'); 96 | }); 97 | 98 | it('no arrangement >> randi >> should comply to given statistical restrictions', () => { 99 | const actualSamples = []; 100 | 101 | for (let i = 0; i < 100000; i++) { 102 | actualSamples.push(sut.randi(0, 100)); 103 | } 104 | actual = determineBasicStatistics(actualSamples); 105 | 106 | expect(actual.min).toBeGreaterThanOrEqual(0, 'Min'); 107 | expect(actual.min).toBeLessThanOrEqual(1, 'Min'); 108 | expect(actual.max).toBeGreaterThanOrEqual(98, 'Max'); 109 | expect(actual.max).toBeLessThanOrEqual(99, 'Max'); 110 | expect(actual.mean).toBeGreaterThan(49, 'Mean'); 111 | expect(actual.mean).toBeLessThan(51, 'Mean'); 112 | }); 113 | 114 | it('no arrangement >> skewedRandn (skewness factor 1) >> should comply to given statistical restrictions', () => { 115 | const actualSamples = []; 116 | 117 | for (let i = 0; i < 100000; i++) { 118 | actualSamples.push(sut.skewedRandn(0, 1, 1)); 119 | } 120 | actual = determineBasicStatistics(actualSamples); 121 | 122 | expect(actual.min).toBeGreaterThan(-5, 'Min'); 123 | expect(actual.min).toBeLessThan(-3.5, 'Min'); 124 | expect(actual.max).toBeGreaterThan(3.5, 'Max'); 125 | expect(actual.max).toBeLessThan(5, 'Max'); 126 | expect(actual.mean).toBeGreaterThan(-0.05, 'Mean'); 127 | expect(actual.mean).toBeLessThan(0.05, 'Mean'); 128 | expect(actual.std).toBeGreaterThan(0.9, 'Std'); 129 | expect(actual.std).toBeLessThan(1.1, 'Std'); 130 | expect(actual.var).toBeGreaterThan(0.9, 'Var'); 131 | expect(actual.var).toBeLessThan(1.1, 'Var'); 132 | }); 133 | 134 | xit('no arrangement >> skewedRandn (skewness factor 0.5 & 2) >> should be provide a mirrored distributional picture', () => { 135 | /* 136 | * skewness should not interfere with `std` & `var` 137 | * ==> `std` & `var` should be ~1 138 | * current implementation distorts the distribution when shifting to positive side. 139 | * TODO: Algorithm needs a revision. 140 | */ 141 | 142 | // skewness factor 0.5 143 | let actualSamples = []; 144 | 145 | for (let i = 0; i < 100000; i++) { 146 | actualSamples.push(sut.skewedRandn(0, 1, 0.5)); 147 | } 148 | actual = determineBasicStatistics(actualSamples); 149 | 150 | expect(actual.min).toBeGreaterThan(-5, '[skewness factor 0.5] Min'); 151 | expect(actual.min).toBeLessThan(1.5, '[skewness factor 0.5] Min'); 152 | expect(actual.max).toBeGreaterThan(3.5, '[skewness factor 0.5] Max'); 153 | expect(actual.max).toBeLessThan(5, '[skewness factor 0.5] Max'); 154 | expect(actual.mean).toBeGreaterThan(1.9, '[skewness factor 0.5] Mean'); 155 | expect(actual.mean).toBeLessThan(2.1, '[skewness factor 0.5] Mean'); 156 | expect(actual.std).toBeGreaterThan(0.7, '[skewness factor 0.5] Std'); 157 | expect(actual.std).toBeLessThan(0.8, '[skewness factor 0.5] Std'); 158 | expect(actual.var).toBeGreaterThan(0.45, '[skewness factor 0.5] Var'); 159 | expect(actual.var).toBeLessThan(0.55, '[skewness factor 0.5] Var'); 160 | 161 | actualSamples = []; 162 | 163 | for (let i = 0; i < 100000; i++) { 164 | actualSamples.push(sut.skewedRandn(0, 1, 2)); 165 | } 166 | actual = determineBasicStatistics(actualSamples); 167 | 168 | expect(actual.min).toBeGreaterThanOrEqual(-5, '[skewness factor 2] Min'); 169 | expect(actual.min).toBeLessThan(-3.5, '[skewness factor 2] Min'); 170 | expect(actual.max).toBeGreaterThan(1.5, '[skewness factor 2] Max'); 171 | expect(actual.max).toBeLessThan(5, '[skewness factor 2] Max'); 172 | expect(actual.mean).toBeGreaterThan(-2.45, '[skewness factor 2] Mean'); 173 | expect(actual.mean).toBeLessThan(-2.35, '[skewness factor 2] Mean'); 174 | expect(actual.std).toBeGreaterThan(0.9, '[skewness factor 2] Std'); 175 | expect(actual.std).toBeLessThan(1.1, '[skewness factor 2] Std'); 176 | expect(actual.var).toBeGreaterThan(0.9, '[skewness factor 2] Var'); 177 | expect(actual.var).toBeLessThan(1.1, '[skewness factor 2] Var'); 178 | }); 179 | }); 180 | 181 | describe('With Patched Random Number Generator:', () => { 182 | 183 | beforeEach(() => { 184 | spyOn(Utils, 'gaussRandom' as any).and.callFake(() => { return 1; }); 185 | spyOn(Math, 'random').and.callFake(() => { return 1; }); 186 | }); 187 | 188 | it('patched random number generator >> randn >> should give back 2.123', () => { 189 | const actual = sut.randn(1, 1.123); 190 | 191 | expect(actual).toBe(2.123); 192 | }); 193 | 194 | it('patched random number generator >> randi >> should give back ', () => { 195 | const actual = sut.randi(1, 1.123); 196 | 197 | expect(actual).toBe(1); 198 | }); 199 | 200 | it('patched random number generator >> randf >> should give back ', () => { 201 | const actual = sut.randf(1, 1.123); 202 | 203 | expect(actual).toBe(1.123); 204 | }); 205 | }); 206 | 207 | const determineBasicStatistics = (sample: Array): { min, max, std, var, mean } => { 208 | const out = { min: null, max: null, std: null, var: null, mean: null }; 209 | // RANGE: MIN & MAX 210 | out.min = Number.POSITIVE_INFINITY; 211 | out.max = Number.NEGATIVE_INFINITY; 212 | for (let i = 0; i < sample.length; i++) { 213 | if (sample[i] > out.max) { out.max = sample[i]; } 214 | if (sample[i] < out.min) { out.min = sample[i]; } 215 | } 216 | // MEAN, MEDIAN & MODE 217 | out.std = Utils.std(sample); 218 | out.var = Utils.var(sample); 219 | out.mean = Utils.mean(sample); 220 | return out; 221 | }; 222 | }); 223 | 224 | describe('Array Filler:', () => { 225 | 226 | let actual: Array; 227 | 228 | beforeEach(() => { 229 | spyOn(sut, 'randn').and.callFake(() => { return 1; }); 230 | spyOn(sut, 'randf').and.callFake(() => { return 1; }); 231 | actual = new Array(5); 232 | }); 233 | 234 | it('Unpopulated Array of size 5 >> fillRandn >> should call Utils.randn 5 times', () => { 235 | sut.fillRandn(actual, 0, 5); 236 | 237 | expect(sut.randn).toHaveBeenCalled(); 238 | expect(sut.randn).toHaveBeenCalledTimes(5); 239 | expect(sut.randn).toHaveBeenCalledWith(0, 5); 240 | }); 241 | 242 | it('Unpopulated Array of size 5 >> fillRandn >> should be filled with values from Utils.randn', () => { 243 | sut.fillRandn(actual, 0, 5); 244 | 245 | expectSutToBeFilledWith(1); 246 | }); 247 | 248 | it('Unpopulated Array of size 5 >> fillRand >> should call Utils.randf 5 times', () => { 249 | sut.fillRand(actual, 0, 5); 250 | 251 | expect(sut.randf).toHaveBeenCalled(); 252 | expect(sut.randf).toHaveBeenCalledTimes(5); 253 | expect(sut.randf).toHaveBeenCalledWith(0, 5); 254 | }); 255 | 256 | it('Unpopulated Array of size 5 >> fillRand >> should be filled with values from Utils.randf', () => { 257 | sut.fillRand(actual, 0, 5); 258 | 259 | expectSutToBeFilledWith(1); 260 | }); 261 | 262 | it('Unpopulated Array of size 5 >> fillConst >> should be filled with constant value 2', () => { 263 | sut.fillConst(actual, 2); 264 | 265 | expectSutToBeFilledWith(2); 266 | }); 267 | 268 | const expectSutToBeFilledWith = (expected: number) => { 269 | for (let i = 0; i < sut.length; i++) { 270 | expect(actual[i]).toBe(expected); 271 | } 272 | }; 273 | }); 274 | 275 | describe('Statistic Tools:', () => { 276 | 277 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> sum >> should return 23', () => { 278 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 279 | 280 | const actual = sut.sum(input); 281 | 282 | expect(actual).toBe(23); 283 | }); 284 | 285 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> mean >> should return 2.875', () => { 286 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 287 | 288 | const actual = sut.mean(input); 289 | 290 | expect(actual).toBe(2.875); 291 | }); 292 | 293 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> median >> should return 3', () => { 294 | const evenInput = [3, 5, 4, 4, 1, 1, 2, 3]; 295 | 296 | const actual = sut.median(evenInput); 297 | 298 | expect(actual).toBe(3); 299 | }); 300 | 301 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3, 9] >> median >> should return 3', () => { 302 | const oddInput = [3, 5, 4, 4, 1, 1, 2, 3, 9]; 303 | 304 | const actual = sut.median(oddInput); 305 | 306 | expect(actual).toBe(3); 307 | }); 308 | 309 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> var (unbiased) >> should return 2.125', () => { 310 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 311 | 312 | const actual = sut.var(input, 'unbiased'); 313 | 314 | expect(actual).toBe(2.125); 315 | }); 316 | 317 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> var >> should return 2.125 and thus default to `unbiased` form', () => { 318 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 319 | 320 | const actual = sut.var(input); 321 | 322 | expect(actual).toBe(2.125); 323 | }); 324 | 325 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> var (uncorrected) >> should return 1.859375', () => { 326 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 327 | 328 | const actual = sut.var(input, 'uncorrected'); 329 | 330 | expect(actual).toBe(1.859375); 331 | }); 332 | 333 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> var (biased) >> should return 1.6527777777777777', () => { 334 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 335 | 336 | const actual = sut.var(input, 'biased'); 337 | 338 | expect(actual).toBe(1.6527777777777777); 339 | }); 340 | 341 | it('some Array [3] with one element >> var (unbiased) >> should return 0 (as fallback)', () => { 342 | const input = [3]; 343 | 344 | const actual = sut.var(input, 'unbiased'); 345 | 346 | expect(actual).toBe(0); 347 | }); 348 | 349 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> std >> should return 1.45722 and thus default to `unbiased` form', () => { 350 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 351 | 352 | const actual = sut.std(input); 353 | 354 | expect(actual).toBeCloseTo(1.45722); 355 | }); 356 | 357 | it('some Array [3, 5, 4, 4, 1, 1, 2, 3] >> mode >> should return [1, 3, 4]', () => { 358 | const input = [3, 5, 4, 4, 1, 1, 2, 3]; 359 | 360 | const actual = sut.mode(input); 361 | 362 | expect(actual).toEqual([1, 3, 4]); 363 | }); 364 | 365 | }); 366 | 367 | describe('Array Creation Functions:', () => { 368 | 369 | let actual: Array | Float64Array; 370 | 371 | it('no instance >> zeros >> should be of expected length', () => { 372 | actual = sut.zeros(4); 373 | 374 | expect(actual.length).toBe(4); 375 | }); 376 | 377 | it('no instance >> zeros >> should be filled with constant zeros', () => { 378 | actual = sut.zeros(4); 379 | 380 | expectSutToBeFilledWith(0); 381 | }); 382 | 383 | it('no instance >> ones >> should be of expected length', () => { 384 | actual = sut.ones(4); 385 | 386 | expect(actual.length).toBe(4); 387 | }); 388 | 389 | it('no instance >> ones >> should be filled with constant ones', () => { 390 | actual = sut.ones(4); 391 | 392 | expectSutToBeFilledWith(1); 393 | }); 394 | 395 | const expectSutToBeFilledWith = (expected: number) => { 396 | for (let i = 0; i < sut.length; i++) { 397 | expect(actual[i]).toBe(expected); 398 | } 399 | }; 400 | }); 401 | 402 | describe('Output Functions:', () => { 403 | 404 | describe('Softmax:', () => { 405 | 406 | it('Populated Array >> softmax >> returning array values should sum up to 1', () => { 407 | const actual = sut.softmax([0, 1, 10, 3, 4]); 408 | 409 | expect(sut.sum(actual)).toBe(1); 410 | }); 411 | 412 | it('Populated Array >> softmax >> should contain values that are exponentially scaled and then normalized', () => { 413 | const input = [0, 1, 10, 3, 4]; 414 | const actual = sut.softmax(input); 415 | 416 | expectValuesToBeExponentiallyNormalized(input, actual); 417 | }); 418 | 419 | const expectValuesToBeExponentiallyNormalized = (input: Array, actual: Array | Float64Array): void => { 420 | const expSum = getExponentialSumOf(input); 421 | 422 | for (let i = 0; i < input.length; i++) { 423 | const expected = Math.exp(input[i]) / expSum; 424 | expect(actual[i]).toBe(expected); 425 | } 426 | }; 427 | 428 | const getExponentialSumOf = (arr: Array): number => { 429 | let expSum = 0; 430 | for (let i = 0; i < arr.length; i++) { 431 | expSum += Math.exp(arr[i]); 432 | } 433 | return expSum; 434 | }; 435 | }); 436 | 437 | describe('Argmax:', () => { 438 | 439 | it('Populated Array >> argmax >> should return index of field with max value', () => { 440 | const actual = sut.argmax([0, 1, 10, 3, 4]); 441 | 442 | expect(actual).toBe(2); 443 | }); 444 | }); 445 | }); 446 | 447 | describe('Weighted Sampling:', () => { 448 | 449 | beforeEach(() => { 450 | spyOn(Math, 'random').and.callFake(() => { return 1.2; }); 451 | }); 452 | 453 | it('Populated Array >> patched sampleWeighted >> should return index of field with accumulated value greater than the given output of patch', () => { 454 | const actual = sut.sampleWeighted([0, 1, 10, 3, 4]); 455 | 456 | // 0 + 1 + 10 > 1.2 >> 10 is in field 2 457 | expect(actual).toBe(2); 458 | }); 459 | 460 | it('Populated Array of zeros >> patched sampleWeighted >> should return 0 (fallback)', () => { 461 | const actual = Utils.sampleWeighted([0, 0, 0, 0]); 462 | 463 | expect(actual).toBe(0); 464 | }); 465 | }); 466 | }); 467 | }); 468 | -------------------------------------------------------------------------------- /src/utils.ts: -------------------------------------------------------------------------------- 1 | export class Utils { 2 | /** 3 | * Returns a random floating point number of a uniform distribution between `min` and `max` 4 | * @param {number} min lower bound 5 | * @param {number} max upper bound 6 | * @returns {number} random float value 7 | */ 8 | public static randf(min: number, max: number): number { 9 | return Math.random() * (max - min) + min; 10 | } 11 | 12 | /** 13 | * Returns a random integer number of a uniform distribution between [`min`, `max`) 14 | * @param {number} min lower bound 15 | * @param {number} max upper bound 16 | * @returns {number} random integer value 17 | */ 18 | public static randi(min: number, max: number): number { 19 | return Math.floor(Utils.randf(min, max)); 20 | } 21 | 22 | /** 23 | * Returns a sample of a normal distribution 24 | * @param {number} mu mean 25 | * @param {number} std standard deviation 26 | * @returns {number} random value 27 | */ 28 | public static randn(mu: number, std: number): number { 29 | return mu + Utils.gaussRandom() * std; 30 | } 31 | 32 | /** 33 | * Returns a random sample number from a normal distributed set 34 | * @param {number} min lower bound 35 | * @param {number} max upper bound 36 | * @param {number} skew factor of skewness; < 1 shifts to the right; > 1 shifts to the left 37 | * @returns {number} random value 38 | */ 39 | public static skewedRandn(mu: number, std: number, skew: number): number { 40 | let sample = Utils.box_muller(); 41 | sample = Math.pow(sample, skew); 42 | sample = (sample - 0.5) * 10; 43 | return mu + sample * std; 44 | } 45 | 46 | /** 47 | * Gaussian-distributed sample from a normal distributed set. 48 | */ 49 | private static gaussRandom(): number { 50 | return (Utils.box_muller() - 0.5) * 10; 51 | } 52 | 53 | /** 54 | * Box-Muller Transform, to transform uniform random values into standard gaussian distributed random values. 55 | * @returns random value between of interval (0,1) 56 | */ 57 | private static box_muller(): number { 58 | // Based on: 59 | // https://en.wikipedia.org/wiki/Box%E2%80%93Muller_transform 60 | // and 61 | // https://stackoverflow.com/questions/25582882/javascript-math-random-normal-distribution-gaussian-bell-curve 62 | let z0 = 0, u1 = 0, u2 = 0; 63 | do { 64 | u1 = u2 = 0; 65 | // Convert interval from [0,1) to (0,1) 66 | do { u1 = Math.random(); } while (u1 === 0); 67 | do { u2 = Math.random(); } while (u2 === 0); 68 | z0 = Math.sqrt(-2.0 * Math.log(u1)) * Math.cos(2.0 * Math.PI * u2); 69 | z0 = z0 / 10.0 + 0.5; 70 | } while (z0 > 1 || z0 < 0); // resample c 71 | return z0; 72 | } 73 | 74 | /** 75 | * Calculates the sum of a given set 76 | * @param arr randomly populated array of numbers 77 | */ 78 | public static sum(arr: Array | Float64Array): number { 79 | let sum = 0; 80 | for (let i = 0; i < arr.length; i++) { 81 | sum += arr[i]; 82 | } 83 | return sum; 84 | } 85 | 86 | /** 87 | * Calculates the mean of a given set 88 | * @param arr set of values 89 | */ 90 | public static mean(arr: Array | Float64Array): number { 91 | // mean of [3, 5, 4, 4, 1, 1, 2, 3] is 2.875 92 | const count = arr.length; 93 | const sum = Utils.sum(arr); 94 | return sum / count; 95 | } 96 | 97 | /** 98 | * Calculates the median of a given set 99 | * @param arr set of values 100 | */ 101 | public static median(arr: Array | Float64Array): number { 102 | // median of [3, 5, 4, 4, 1, 1, 2, 3] = 3 103 | let median = 0; 104 | const count = arr.length; 105 | arr.sort(); 106 | if (count % 2 === 0) { // is even 107 | // average of two middle numbers 108 | median = (arr[count / 2 - 1] + arr[count / 2]) / 2; 109 | } else { // is odd 110 | // middle number only 111 | median = arr[(count - 1) / 2]; 112 | } 113 | return median; 114 | } 115 | 116 | /** 117 | * Calculates the standard deviation of a given set 118 | * @param arr set of values 119 | * @param precision the floating point precision for grouping results, e.g. 1e3 [defaults to 1e6] 120 | */ 121 | public static mode(arr: Array | Float64Array, precision?: number): Array | Float64Array { 122 | // as result can be bimodal or multimodal, 123 | // the returned result is provided as an array 124 | // mode of [3, 5, 4, 4, 1, 1, 2, 3] = [1, 3, 4] 125 | precision = precision ? precision : 1e6; 126 | const modes = []; 127 | const count = []; 128 | let num = 0; 129 | let maxCount = 0; 130 | // populate array with number counts 131 | for (let i = 0; i < arr.length; i++) { 132 | num = Math.round(arr[i] * precision) / precision; 133 | count[num] = (count[num] || 0) + 1; // initialize or increment for number 134 | if (count[num] > maxCount) { 135 | maxCount = count[num]; // memorize count value of max index 136 | } 137 | } 138 | // memorize numbers equal with maxCount 139 | for (const i in count) { 140 | if (count.hasOwnProperty(i)) { 141 | if (count[i] === maxCount) { 142 | modes.push(Number(i)); 143 | } 144 | } 145 | } 146 | return modes; 147 | } 148 | 149 | /** 150 | * Calculates the population variance (uncorrected), the sample variance (unbiased) or biased variance of a given set 151 | * @param arr set of values 152 | * @param normalization defaults to sample variance ('unbiased') 153 | */ 154 | public static var(arr: Array | Float64Array, normalization?: 'uncorrected' | 'biased' | 'unbiased'): number { 155 | normalization = normalization ? normalization : 'unbiased'; 156 | const count = arr.length; 157 | 158 | // calculate the variance 159 | const mean = Utils.mean(arr); 160 | let sum = 0; 161 | let diff = 0; 162 | for (let i = 0; i < arr.length; i++) { 163 | diff = arr[i] - mean; 164 | sum += diff * diff; 165 | } 166 | 167 | switch (normalization) { 168 | case 'uncorrected': 169 | return sum / count; 170 | 171 | case 'biased': 172 | return sum / (count + 1); 173 | 174 | case 'unbiased': 175 | return (count === 1) ? 0 : sum / (count - 1); 176 | 177 | default: 178 | return ; 179 | } 180 | } 181 | 182 | /** 183 | * Calculates the standard deviation of a given set 184 | * @param arr set of values 185 | * @param normalization defaults to sample variance ('unbiased') 186 | */ 187 | public static std = (arr: Array | Float64Array, normalization?: 'uncorrected' | 'biased' | 'unbiased'): number => { 188 | return Math.sqrt(Utils.var(arr, normalization)); 189 | } 190 | 191 | /** 192 | * Fills the given array with normal distributed random values. 193 | * @param arr Array to be filled 194 | * @param mu mean 195 | * @param std standard deviation 196 | * @returns {void} void 197 | */ 198 | public static fillRandn(arr: Array | Float64Array, mu: number, std: number): void { 199 | for (let i = 0; i < arr.length; i++) { arr[i] = Utils.randn(mu, std); } 200 | } 201 | 202 | /** 203 | * Fills the given array with uniformly distributed random values between `min` and `max`. 204 | * @param arr Array to be filled 205 | * @param min lower bound 206 | * @param max upper bound 207 | * @returns {void} void 208 | */ 209 | public static fillRand(arr: Array | Float64Array, min: number, max: number): void { 210 | for (let i = 0; i < arr.length; i++) { arr[i] = Utils.randf(min, max); } 211 | } 212 | 213 | /** 214 | * Fills the pointed array with constant values. 215 | * @param {Array | Float64Array} arr Array to be filled 216 | * @param {number} c value 217 | * @returns {void} void 218 | */ 219 | public static fillConst(arr: Array | Float64Array, c: number): void { 220 | for (let i = 0; i < arr.length; i++) { arr[i] = c; } 221 | } 222 | 223 | /** 224 | * returns array populated with ones of length n and uses typed arrays if available 225 | * @param {number} n length of Array 226 | * @returns {Array | Float64Array} Array 227 | */ 228 | public static ones(n: number): Array | Float64Array { 229 | return Utils.fillArray(n, 1); 230 | } 231 | 232 | /** 233 | * returns array of zeros of length n and uses typed arrays if available 234 | * @param {number} n length of Array 235 | * @returns {Array | Float64Array} Array 236 | */ 237 | public static zeros(n: number): Array | Float64Array { 238 | return Utils.fillArray(n, 0); 239 | } 240 | 241 | private static fillArray(n: number, val: number): Array | Float64Array { 242 | if (typeof n === 'undefined' || isNaN(n)) { return []; } 243 | if (typeof ArrayBuffer === 'undefined') { 244 | const arr = new Array(n); 245 | Utils.fillConst(arr, val); 246 | return arr; 247 | } else { 248 | const arr = new Float64Array(n); 249 | Utils.fillConst(arr, val); 250 | return arr; 251 | } 252 | } 253 | 254 | /** 255 | * Softmax of a given set of values 256 | * @param {Array | Float64Array} arr set of values 257 | */ 258 | public static softmax(arr: Array | Float64Array): Array | Float64Array { 259 | const output = []; 260 | let expSum = 0; 261 | for(let i = 0; i < arr.length; i++) { 262 | expSum += Math.exp(arr[i]); 263 | } 264 | for(let i = 0; i < arr.length; i++) { 265 | output[i] = Math.exp(arr[i]) / expSum; 266 | } 267 | 268 | return output; 269 | } 270 | 271 | /** 272 | * Argmax of a given set of values 273 | * @param {Array | Float64Array} arr set of values 274 | * @returns {number} Index of Argmax Operation 275 | */ 276 | public static argmax(arr: Array | Float64Array): number { 277 | let maxValue = arr[0]; 278 | let maxIndex = 0; 279 | for (let i = 1; i < arr.length; i++) { 280 | const v = arr[i]; 281 | if (v > maxValue) { 282 | maxIndex = i; 283 | maxValue = v; 284 | } 285 | } 286 | return maxIndex; 287 | } 288 | 289 | /** 290 | * Returns an index of the weighted sample of Array `arr` 291 | * @param {Array | Float64Array} arr Array to be sampled 292 | * @returns {number} 293 | */ 294 | public static sampleWeighted(arr: Array | Float64Array): number { 295 | const r = Math.random(); 296 | let c = 0.0; 297 | for (let i = 0; i < arr.length; i++) { 298 | c += arr[i]; 299 | if (c >= r) { return i; } 300 | } 301 | 302 | return 0; 303 | } 304 | 305 | } 306 | -------------------------------------------------------------------------------- /src/utils/assertable.spec.ts: -------------------------------------------------------------------------------- 1 | import { Assertable } from "./assertable"; 2 | 3 | 4 | describe('Assertable', () => { 5 | 6 | const sut = Assertable['assert']; 7 | 8 | it('no instance >> assert true statement >> should not throw an error', () => { 9 | const actual = () => { sut(true, 'Don\'t expect to throw.'); }; 10 | 11 | expect(actual).not.toThrowError(); 12 | }); 13 | 14 | it('no instance >> assert true statement >> should throw an error', () => { 15 | const actual = () => { sut(false, 'Expect to throw.'); }; 16 | 17 | expect(actual).toThrowError(/Expect to throw\./); 18 | }); 19 | }); 20 | -------------------------------------------------------------------------------- /src/utils/assertable.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * Adds static assertability to Classes 3 | */ 4 | export abstract class Assertable { 5 | /** 6 | * Asserts a condition and throws Error if not assertion fails 7 | * @param {boolean} condition 8 | * @param {string} message 9 | */ 10 | protected static assert(condition: boolean, message: string = '') { 11 | // from http://stackoverflow.com/questions/15313418/javascript-assert 12 | if (!condition) { 13 | message = message || "Assertion failed"; 14 | if (typeof Error !== "undefined") { 15 | throw new Error(message); 16 | } 17 | throw message; // Fallback 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/utils/inner-state.ts: -------------------------------------------------------------------------------- 1 | import { Mat } from './..'; 2 | 3 | /** 4 | * State of inner activations 5 | */ 6 | export interface InnerState { 7 | hiddenActivationState: Array; 8 | output: Mat; 9 | cells?: Array; 10 | } 11 | -------------------------------------------------------------------------------- /src/utils/mat-ops.spec.ts: -------------------------------------------------------------------------------- 1 | import { Mat, Utils } from '..'; 2 | import { MatOps } from './mat-ops'; 3 | 4 | describe('MatOps:', () => { 5 | 6 | const sut = MatOps; 7 | let actual: Mat; 8 | let expected: Mat; 9 | let mat1: Mat; 10 | 11 | beforeEach(() => { 12 | mat1 = new Mat(2, 4); 13 | mat1.setFrom([1, 4, 6, 10, 2, 7, 5, 3]); 14 | }); 15 | 16 | describe('Single Matrix Operations:', () => { 17 | 18 | describe('Row Pluck:', () => { 19 | 20 | let rowIndex: number; 21 | 22 | beforeEach(() => { 23 | rowIndex = 0; 24 | }); 25 | 26 | it('given a matrix >> rowPluck >> should return new instance of matrix-object (reference)', () => { 27 | actual = sut.rowPluck(mat1, rowIndex); 28 | 29 | expectOperationHasReturnedNewInstance(); 30 | }); 31 | 32 | it('given a matrix with dimensions (2,4) >> rowPluck >> should return matrix with dimensions (4,1)', () => { 33 | actual = sut.rowPluck(mat1, rowIndex); 34 | 35 | expectOperationHasReturnedMatrixWithDimensions(4, 1); 36 | }); 37 | 38 | it('given a matrix with dimensions (2,4) and incompatible rowIndex >> rowPluck >> should throw error', () => { 39 | const incompatible = 2; 40 | 41 | const callFunction = () => { sut.rowPluck(mat1, incompatible); }; 42 | 43 | expect(callFunction).toThrowError('[class:MatOps] rowPluck: dimensions misaligned'); 44 | }); 45 | 46 | it('given a matrix >> rowPluck >> should return matrix with expected content', () => { 47 | actual = sut.rowPluck(mat1, rowIndex); 48 | 49 | expected = new Mat(4, 1); 50 | expectRowpluckHasReturnedMatrixWithContent([1, 4, 6, 10]); 51 | }); 52 | 53 | const expectRowpluckHasReturnedMatrixWithContent = (content: Array) => { 54 | expected.setFrom(content); 55 | 56 | for (let i = 0; i < actual.w.length; i++) { 57 | expect(actual.w[i]).toBe(expected.w[i]); 58 | } 59 | }; 60 | }); 61 | 62 | describe('Gauss noise-addition:', () => { 63 | 64 | let std: Mat; 65 | 66 | beforeEach(() => { 67 | std = new Mat(2, 4); 68 | std.setFrom([0.1, 0.2, 0.02, 0.5, 1, 0.01, 0, 1]); 69 | spyOn(Utils, 'randn').and.callFake((mu: number, std: number) => { return mu + std; }); 70 | }); 71 | 72 | it('given a matrix >> gauss >> should return new instance of matrix-object (reference)', () => { 73 | actual = sut.gauss(mat1, std); 74 | 75 | expectOperationHasReturnedNewInstance(); 76 | }); 77 | 78 | it('given a matrix with dimensions (2,4) >> gauss >> should return matrix with dimensions (2,4)', () => { 79 | actual = sut.gauss(mat1, std); 80 | 81 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 82 | }); 83 | 84 | it('given a matrix with dimensions (2,4) and (3,3) >> gauss >> should throw error', () => { 85 | const incompatible = new Mat(3, 3); 86 | 87 | const callFunction = () => { sut.gauss(mat1, incompatible); }; 88 | 89 | expect(callFunction).toThrowError('[class:MatOps] gauss: dimensions misaligned'); 90 | }); 91 | 92 | it('given a matrix >> gauss >> should return matrix with expected content', () => { 93 | actual = sut.gauss(mat1, std); 94 | 95 | expected = new Mat(2, 4); 96 | expectGaussHasReturnedMatrixWithGaussianDistributedContent([1, 4, 6, 10, 2, 7, 5, 3], [0.1, 0.2, 0.02, 0.5, 1, 0.01, 0, 1]); 97 | }); 98 | 99 | const expectGaussHasReturnedMatrixWithGaussianDistributedContent = (content: Array, std: Array) => { 100 | expected.setFrom(content); 101 | 102 | for (let i = 0; i < actual.w.length; i++) { 103 | expect(actual.w[i]).toBe(expected.w[i] + std[i]); 104 | } 105 | }; 106 | }); 107 | 108 | describe('Monadic Operations', () => { 109 | 110 | describe('Hyperbolic Tangens', () => { 111 | 112 | it('given a matrix >> tanh >> should return new instance of matrix-object (reference)', () => { 113 | actual = sut.tanh(mat1); 114 | 115 | expectOperationHasReturnedNewInstance(); 116 | }); 117 | 118 | it('given a matrix with dimensions (2,4) >> tanh >> should return matrix with dimensions (2,4)', () => { 119 | actual = sut.tanh(mat1); 120 | 121 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 122 | }); 123 | 124 | it('given a matrix with dimensions (2,4) >> tanh >> should return matrix with dimensions (2,4)', () => { 125 | actual = sut.tanh(mat1); 126 | 127 | expectMonadicOperationHasReturnedMatrixWithContent([0.761594, 0.999329, 0.999987, 0.999999, 0.964027, 0.999998, 0.999909, 0.995054]); 128 | }); 129 | }); 130 | 131 | describe('Sigmoid', () => { 132 | 133 | it('given a matrix >> sig >> should return new instance of matrix-object (reference)', () => { 134 | actual = sut.sig(mat1); 135 | 136 | expectOperationHasReturnedNewInstance(); 137 | }); 138 | 139 | it('given a matrix with dimensions (2,4) >> sig >> should return matrix with dimensions (2,4)', () => { 140 | actual = sut.sig(mat1); 141 | 142 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 143 | }); 144 | 145 | it('given a matrix with dimensions (2,4) >> sig >> should return matrix with dimensions (2,4)', () => { 146 | actual = sut.sig(mat1); 147 | 148 | expectMonadicOperationHasReturnedMatrixWithContent([0.731058, 0.982013, 0.997527, 0.999954, 0.880797, 0.999088, 0.993307, 0.952574]); 149 | }); 150 | }); 151 | 152 | describe('Rectified Linear Units (ReLU)', () => { 153 | 154 | beforeEach(() => { 155 | // Mat with some negative values 156 | mat1.setFrom([1, -4, 6, 10, 2, -7, 5, 3]); 157 | }); 158 | 159 | it('given a matrix >> relu >> should return new instance of matrix-object (reference)', () => { 160 | actual = sut.relu(mat1); 161 | 162 | expectOperationHasReturnedNewInstance(); 163 | }); 164 | 165 | it('given a matrix with dimensions (2,4) >> relu >> should return matrix with dimensions (2,4)', () => { 166 | actual = sut.relu(mat1); 167 | 168 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 169 | }); 170 | 171 | it('given a matrix with dimensions (2,4) >> relu >> should return matrix with dimensions (2,4)', () => { 172 | actual = sut.relu(mat1); 173 | 174 | expectMonadicOperationHasReturnedMatrixWithContent([1, 0, 6, 10, 2, 0, 5, 3]); 175 | }); 176 | }); 177 | 178 | const expectMonadicOperationHasReturnedMatrixWithContent = (content: Array) => { 179 | expected.setFrom(content); 180 | 181 | for (let i = 0; i < actual.w.length; i++) { 182 | expect(actual.w[i]).toBeCloseTo(expected.w[i], 5); 183 | } 184 | }; 185 | }); 186 | }); 187 | 188 | describe('Dual Matrix Operations:', () => { 189 | 190 | let mat2: Mat; 191 | 192 | describe('Multiplication:', () => { 193 | 194 | beforeEach(() => { 195 | mat2 = new Mat(4, 3); 196 | mat2.setFrom([1, 4, 6, 2, 7, 5, 9, 0, 11, 3, 1, 0]); 197 | }); 198 | 199 | it('given two matrices >> multiply >> should return new instance of matrix-object (reference)', () => { 200 | actual = sut.mul(mat1, mat2); 201 | 202 | expectDualMatrixOperationHasReturnedNewInstance(); 203 | }); 204 | 205 | it('given two matrices with dimensions (2,4)*(4,3) >> multiply >> should return matrix with dimensions (2,3)', () => { 206 | actual = sut.mul(mat1, mat2); 207 | 208 | expectOperationHasReturnedMatrixWithDimensions(2, 3); 209 | }); 210 | 211 | it('given two matrices with incompatible dimensions (2,4)*(3,3) >> multiply >> should throw error', () => { 212 | const incompatible = new Mat(3,3); 213 | 214 | const callFunction = () => { sut.mul(mat1, incompatible); }; 215 | 216 | expect(callFunction).toThrowError('[class:MatOps] mul: dimensions misaligned'); 217 | }); 218 | 219 | it('given two matrices >> multiply >> should return matrix with expected content', () => { 220 | actual = sut.mul(mat1, mat2); 221 | 222 | expected = new Mat(2, 3); 223 | expectMultiplicationHasReturnedMatrixWithContent([93, 42, 92, 70, 60, 102]); 224 | }); 225 | 226 | const expectMultiplicationHasReturnedMatrixWithContent = (content: Array) => { 227 | expected.setFrom(content); 228 | 229 | for (let i = 0; i < actual.w.length; i++) { 230 | expect(actual.w[i]).toBe(expected.w[i]); 231 | } 232 | }; 233 | }); 234 | 235 | describe('Addition:', () => { 236 | 237 | beforeEach(() => { 238 | mat2 = new Mat(2, 4); 239 | mat2.setFrom([1, 4, 6, 10, 2, 7, 5, 3]); 240 | }); 241 | 242 | it('given two matrices >> add >> should return new instance of matrix-object (reference)', () => { 243 | actual = sut.add(mat1, mat2); 244 | 245 | expectDualMatrixOperationHasReturnedNewInstance(); 246 | }); 247 | 248 | it('given two matrices with dimensions (2,4)*(2,4) >> add >> should return matrix with dimensions (2,4)', () => { 249 | actual = sut.add(mat1, mat2); 250 | 251 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 252 | }); 253 | 254 | it('given two matrices with incompatible dimensions (2,4)*(3,3) >> add >> should throw error', () => { 255 | const incompatible = new Mat(3, 3); 256 | 257 | const callFunction = () => { sut.add(mat1, incompatible); }; 258 | 259 | expect(callFunction).toThrowError('[class:MatOps] add: dimensions misaligned'); 260 | }); 261 | 262 | it('given two matrices >> add >> should return matrix with expected content', () => { 263 | actual = sut.add(mat1, mat2); 264 | 265 | expected = new Mat(2, 4); 266 | expectAdditionHasReturnedMatrixWithContent([2, 8, 12, 20, 4, 14, 10, 6]); 267 | }); 268 | 269 | const expectAdditionHasReturnedMatrixWithContent = (content: Array) => { 270 | expected.setFrom(content); 271 | 272 | for (let i = 0; i < actual.w.length; i++) { 273 | expect(actual.w[i]).toBe(expected.w[i]); 274 | } 275 | }; 276 | }); 277 | 278 | describe('Dot Product:', () => { 279 | 280 | beforeEach(() => { 281 | mat2 = new Mat(2, 4); 282 | mat2.setFrom([1, 4, 6, 10, 2, 7, 5, 3]); 283 | }); 284 | 285 | it('given two matrices >> dot >> should return new instance of matrix-object (reference)', () => { 286 | actual = sut.dot(mat1, mat2); 287 | 288 | expectDualMatrixOperationHasReturnedNewInstance(); 289 | }); 290 | 291 | it('given two matrices with dimensions (2,4)*(2,4) >> dot >> should return matrix with dimensions (1,1)', () => { 292 | actual = sut.dot(mat1, mat2); 293 | 294 | expectOperationHasReturnedMatrixWithDimensions(1, 1); 295 | }); 296 | 297 | it('given two matrices with incompatible dimensions (2,4)*(3,3) >> dot >> should throw error', () => { 298 | const incompatible = new Mat(3, 3); 299 | 300 | const callFunction = () => { sut.dot(mat1, incompatible); }; 301 | 302 | expect(callFunction).toThrowError('[class:MatOps] dot: dimensions misaligned'); 303 | }); 304 | 305 | it('given two matrices >> dot >> should return matrix with expected content', () => { 306 | actual = sut.dot(mat1, mat2); 307 | 308 | expected = new Mat(1, 1); 309 | expectDotHasReturnedMatrixWithContent([1 + 16 + 36 + 100 + 4 + 49 + 25 + 9]); 310 | }); 311 | 312 | const expectDotHasReturnedMatrixWithContent = (content: Array) => { 313 | expected.setFrom(content); 314 | 315 | for (let i = 0; i < actual.w.length; i++) { 316 | expect(actual.w[i]).toBe(expected.w[i]); 317 | } 318 | }; 319 | }); 320 | 321 | describe('Elementwise Multiplication:', () => { 322 | 323 | beforeEach(() => { 324 | mat2 = new Mat(2, 4); 325 | mat2.setFrom([1, 4, 6, 10, 2, 7, 5, 3]); 326 | }); 327 | 328 | it('given two matrices >> eltmul >> should return new instance of matrix-object (reference)', () => { 329 | actual = sut.eltmul(mat1, mat2); 330 | 331 | expectDualMatrixOperationHasReturnedNewInstance(); 332 | }); 333 | 334 | it('given two matrices with dimensions (2,4)*(2,4) >> eltmul >> should return matrix with dimensions (2,4)', () => { 335 | actual = sut.eltmul(mat1, mat2); 336 | 337 | expectOperationHasReturnedMatrixWithDimensions(2, 4); 338 | }); 339 | 340 | it('given two matrices with incompatible dimensions (2,4)*(3,3) >> eltmul >> should throw error', () => { 341 | const incompatible = new Mat(3, 3); 342 | 343 | const callFunction = () => { sut.eltmul(mat1, incompatible); }; 344 | 345 | expect(callFunction).toThrowError('[class:MatOps] eltmul: dimensions misaligned'); 346 | }); 347 | 348 | it('given two matrices >> eltmul >> should return matrix with expected content', () => { 349 | actual = sut.eltmul(mat1, mat2); 350 | 351 | expected = new Mat(2, 4); 352 | expectEltmulHasReturnedMatrixWithContent([1, 16, 36, 100, 4, 49, 25, 9]); 353 | }); 354 | 355 | const expectEltmulHasReturnedMatrixWithContent = (content: Array) => { 356 | expected.setFrom(content); 357 | 358 | for (let i = 0; i < actual.w.length; i++) { 359 | expect(actual.w[i]).toBe(expected.w[i]); 360 | } 361 | }; 362 | }); 363 | 364 | const expectDualMatrixOperationHasReturnedNewInstance = (): void => { 365 | expectOperationHasReturnedNewInstance(); 366 | expect(actual === mat2).toBe(false); 367 | }; 368 | }); 369 | 370 | const expectOperationHasReturnedNewInstance = (): void => { 371 | expect(actual === mat1).toBe(false); 372 | }; 373 | 374 | const expectOperationHasReturnedMatrixWithDimensions = (rows: number, cols: number): void => { 375 | expected = new Mat(rows, cols); 376 | 377 | expect(actual.rows).toBe(expected.rows); 378 | expect(actual.cols).toBe(expected.cols); 379 | }; 380 | }); 381 | 382 | -------------------------------------------------------------------------------- /src/utils/mat-ops.ts: -------------------------------------------------------------------------------- 1 | import { Mat, Utils } from '..'; 2 | import { Assertable } from './assertable'; 3 | 4 | export class MatOps extends Assertable { 5 | 6 | /** 7 | * Non-destructively pluck a row of m with rowIndex 8 | * @param m 9 | * @param rowIndex index of row 10 | * @returns a column Vector [cols, 1] 11 | */ 12 | public static rowPluck(m: Mat, rowIndex: number): Mat { 13 | Mat.assert(rowIndex >= 0 && rowIndex < m.rows, '[class:MatOps] rowPluck: dimensions misaligned'); 14 | const out = new Mat(m.cols, 1); 15 | for (let i = 0; i < m.cols; i++) { 16 | out.w[i] = m.w[m.cols * rowIndex + i]; 17 | } 18 | return out; 19 | } 20 | 21 | public static getRowPluckBackprop(m: Mat, rowIndex: number, out: Mat): Function { 22 | return () => { 23 | for (let i = 0; i < m.cols; i++) { 24 | m.dw[m.cols * rowIndex + i] += out.dw[i]; 25 | } 26 | }; 27 | } 28 | 29 | /** 30 | * Non-destructive elementwise gaussian-distributed noise-addition. 31 | * @param {Mat} m 32 | * @param {number} std Matrix with STD values 33 | * @returns {Mat} Matrix with results 34 | */ 35 | public static gauss(m: Mat, std: Mat): Mat { 36 | Mat.assert(m.w.length === std.w.length, '[class:MatOps] gauss: dimensions misaligned'); 37 | const out = new Mat(m.rows, m.cols); 38 | for (let i = 0; i < m.w.length; i++) { 39 | out.w[i] = Utils.randn(m.w[i], std.w[i]); 40 | } 41 | return out; 42 | } 43 | 44 | /** 45 | * Non-destructive elementwise tanh. 46 | * @param {Mat} m 47 | * @returns {Mat} Matrix with results 48 | */ 49 | public static tanh(m: Mat): Mat { 50 | const out = new Mat(m.rows, m.cols); 51 | for (let i = 0; i < m.w.length; i++) { 52 | out.w[i] = Math.tanh(m.w[i]); 53 | } 54 | return out; 55 | } 56 | 57 | public static getTanhBackprop(m: Mat, out: Mat): Function { 58 | return () => { 59 | for (let i = 0; i < m.w.length; i++) { 60 | // grad for z = tanh(x) is (1 - z^2) 61 | const mwi = out.w[i]; 62 | m.dw[i] += (1.0 - mwi * mwi) * out.dw[i]; 63 | } 64 | }; 65 | } 66 | 67 | /** 68 | * Non-destructive elementwise sigmoid. 69 | * @param m 70 | * @returns Mat with results 71 | */ 72 | public static sig(m: Mat): Mat { 73 | const out = new Mat(m.rows, m.cols); 74 | for (let i = 0; i < m.w.length; i++) { 75 | out.w[i] = MatOps.sigmoid(m.w[i]); 76 | } 77 | return out; 78 | } 79 | 80 | private static sigmoid(x: number): number { 81 | // helper function for computing sigmoid 82 | return 1.0 / (1 + Math.exp(-x)); 83 | } 84 | 85 | public static getSigmoidBackprop(m: Mat, out: Mat): Function { 86 | return () => { 87 | for (let i = 0; i < m.w.length; i++) { 88 | // grad for z = tanh(x) is (1 - z^2) 89 | const mwi = out.w[i]; 90 | m.dw[i] += mwi * (1.0 - mwi) * out.dw[i]; 91 | } 92 | }; 93 | } 94 | 95 | /** 96 | * Non-destructive elementwise ReLu. 97 | * @returns Mat with results 98 | */ 99 | public static relu(m: Mat): Mat { 100 | const out = new Mat(m.rows, m.cols); 101 | for (let i = 0; i < m.w.length; i++) { 102 | out.w[i] = Math.max(0, m.w[i]); // relu 103 | } 104 | return out; 105 | } 106 | 107 | public static getReluBackprop(m: Mat, out: Mat): Function { 108 | return () => { 109 | for (let i = 0; i < m.w.length; i++) { 110 | m.dw[i] += m.w[i] > 0 ? out.dw[i] : 0.0; 111 | } 112 | }; 113 | } 114 | 115 | /** 116 | * Non-destructive elementwise add. 117 | * @param {Mat} m1 118 | * @param {Mat} m2 119 | */ 120 | public static add(m1: Mat, m2: Mat): Mat { 121 | Mat.assert(m1.w.length === m2.w.length && m1.rows === m2.rows, '[class:MatOps] add: dimensions misaligned'); 122 | const out = new Mat(m1.rows, m1.cols); 123 | for (let i = 0; i < m1.w.length; i++) { 124 | out.w[i] = m1.w[i] + m2.w[i]; 125 | } 126 | return out; 127 | } 128 | 129 | public static getAddBackprop(m1: Mat, m2: Mat, out: Mat): Function { 130 | return () => { 131 | for (let i = 0; i < m1.w.length; i++) { 132 | m1.dw[i] += out.dw[i]; 133 | m2.dw[i] += out.dw[i]; 134 | } 135 | }; 136 | } 137 | 138 | /** 139 | * Non-destructive Matrix multiplication. 140 | * @param m1 141 | * @param m2 142 | * @returns Mat with results 143 | */ 144 | public static mul(m1: Mat, m2: Mat): Mat { 145 | Mat.assert(m1.cols === m2.rows, '[class:MatOps] mul: dimensions misaligned'); 146 | const out = new Mat(m1.rows, m2.cols); 147 | for (let row = 0; row < m1.rows; row++) { // loop over rows of m1 148 | for (let col = 0; col < m2.cols; col++) { // loop over cols of m2 149 | let dot = 0.0; 150 | for (let k = 0; k < m1.cols; k++) { // dot product loop 151 | dot += m1.w[m1.cols * row + k] * m2.w[m2.cols * k + col]; 152 | } 153 | out.w[m2.cols * row + col] = dot; 154 | } 155 | } 156 | return out; 157 | } 158 | 159 | public static getMulBackprop(m1: Mat, m2: Mat, out: Mat): Function { 160 | return () => { 161 | for (let i = 0; i < m1.rows; i++) { 162 | for (let j = 0; j < m2.cols; j++) { 163 | for (let k = 0; k < m1.cols; k++) { 164 | const b = out.dw[m2.cols * i + j]; 165 | m1.dw[m1.cols * i + k] += m2.w[m2.cols * k + j] * b; 166 | m2.dw[m2.cols * k + j] += m1.w[m1.cols * i + k] * b; 167 | } 168 | } 169 | } 170 | }; 171 | } 172 | 173 | /** 174 | * Non-destructive dot Product. 175 | * @param m1 176 | * @param m2 177 | * @return {Mat} Matrix of dimension 1x1 178 | */ 179 | public static dot(m1: Mat, m2: Mat): Mat { 180 | Mat.assert(m1.w.length === m2.w.length && m1.rows === m2.rows, '[class:MatOps] dot: dimensions misaligned'); 181 | const out = new Mat(1, 1); 182 | let dot = 0.0; 183 | for (let i = 0; i < m1.w.length; i++) { 184 | dot += m1.w[i] * m2.w[i]; 185 | } 186 | out.w[0] = dot; 187 | return out; 188 | } 189 | 190 | public static getDotBackprop(m1: Mat, m2: Mat, out: Mat): Function { 191 | return () => { 192 | for (let i = 0; i < m1.w.length; i++) { 193 | m1.dw[i] += m2.w[i] * out.dw[0]; 194 | m2.dw[i] += m1.w[i] * out.dw[0]; 195 | } 196 | }; 197 | } 198 | 199 | /** 200 | * Non-destructive elementwise Matrix multiplication. 201 | * @param m1 202 | * @param m2 203 | * @return {Mat} Matrix with results 204 | */ 205 | public static eltmul(m1: Mat, m2: Mat): Mat { 206 | Mat.assert(m1.w.length === m2.w.length && m1.rows === m2.rows, '[class:MatOps] eltmul: dimensions misaligned'); 207 | const out = new Mat(m1.rows, m1.cols); 208 | for (let i = 0; i < m1.w.length; i++) { 209 | out.w[i] = m1.w[i] * m2.w[i]; 210 | } 211 | return out; 212 | } 213 | 214 | public static getEltmulBackprop(m1: Mat, m2: Mat, out: Mat): Function { 215 | return () => { 216 | for (let i = 0; i < m1.w.length; i++) { 217 | m1.dw[i] += m2.w[i] * out.dw[i]; 218 | m2.dw[i] += m1.w[i] * out.dw[i]; 219 | } 220 | }; 221 | } 222 | } 223 | -------------------------------------------------------------------------------- /src/utils/net-opts.ts: -------------------------------------------------------------------------------- 1 | export interface NetOpts { 2 | architecture: { 3 | inputSize: number, 4 | hiddenUnits: Array, 5 | outputSize: number 6 | }; 7 | training?: { 8 | alpha?: number, 9 | lossClamp?: number, 10 | loss?: number 11 | }; 12 | other?: { 13 | mu?: number; 14 | std?: number; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "lib": [ 4 | "es6" 5 | ], 6 | "target": "es6", 7 | "module": "commonjs", 8 | "moduleResolution": "node", 9 | "sourceMap": true, 10 | "declaration": true, 11 | "removeComments": true, 12 | "rootDir": "./src/", 13 | "outDir": "./dist/", 14 | "noImplicitAny": false, 15 | "strictNullChecks": false, 16 | "allowUnusedLabels": false, 17 | "typeRoots": [ 18 | "node_modules/@types" 19 | ] 20 | }, 21 | "include": [ 22 | "src/*.ts", 23 | "src/**/*.ts" 24 | ], 25 | "exclude": [ 26 | "node_modules", 27 | "dist" 28 | ] 29 | } -------------------------------------------------------------------------------- /tslint-google.json: -------------------------------------------------------------------------------- 1 | { 2 | "repository": { 3 | "author": "Google (unofficial)", 4 | "url": "https://github.com/google/ts-style" 5 | }, 6 | "rules": { 7 | "array-type": [ 8 | true, 9 | "array-simple" 10 | ], 11 | "arrow-return-shorthand": true, 12 | "ban": [ 13 | true, 14 | { 15 | "name": "parseInt", 16 | "message": "tsstyle#type-coercion" 17 | }, 18 | { 19 | "name": "parseFloat", 20 | "message": "tsstyle#type-coercion" 21 | }, 22 | { 23 | "name": "Array", 24 | "message": "tsstyle#array-constructor" 25 | } 26 | ], 27 | "ban-types": [ 28 | true, 29 | [ 30 | "Object", 31 | "Use {} instead." 32 | ], 33 | [ 34 | "String", 35 | "Use 'string' instead." 36 | ], 37 | [ 38 | "Number", 39 | "Use 'number' instead." 40 | ], 41 | [ 42 | "Boolean", 43 | "Use 'boolean' instead." 44 | ] 45 | ], 46 | "class-name": true, 47 | "curly": [ 48 | true, 49 | "ignore-same-line" 50 | ], 51 | "forin": true, 52 | "interface-name": [ 53 | true, 54 | "never-prefix" 55 | ], 56 | "jsdoc-format": true, 57 | "label-position": true, 58 | "member-access": [ 59 | true, 60 | "no-public" 61 | ], 62 | "new-parens": true, 63 | "no-angle-bracket-type-assertion": true, 64 | "no-any": true, 65 | "no-arg": true, 66 | "no-conditional-assignment": true, 67 | "no-construct": true, 68 | "no-debugger": true, 69 | "no-default-export": true, 70 | "no-duplicate-variable": true, 71 | "no-inferrable-types": true, 72 | "no-namespace": [ 73 | true, 74 | "allow-declarations" 75 | ], 76 | "no-reference": true, 77 | "no-string-throw": true, 78 | "no-unused-expression": true, 79 | "no-var-keyword": true, 80 | "object-literal-shorthand": true, 81 | "only-arrow-functions": [ 82 | true, 83 | "allow-declarations", 84 | "allow-named-functions" 85 | ], 86 | "prefer-const": true, 87 | "radix": true, 88 | "semicolon": [ 89 | true, 90 | "always", 91 | "ignore-bound-class-methods" 92 | ], 93 | "switch-default": true, 94 | "triple-equals": [ 95 | true, 96 | "allow-null-check" 97 | ], 98 | "use-isnan": true, 99 | "variable-name": [ 100 | true, 101 | "check-format", 102 | "ban-keywords", 103 | "allow-leading-underscore", 104 | "allow-trailing-underscore" 105 | ] 106 | } 107 | } -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./tslint-google.json", 3 | "rules": { 4 | "array-type": [ 5 | false 6 | ], 7 | "arrow-return-shorthand": false, 8 | "eofline": true, 9 | "member-access": false, 10 | "no-any": false, 11 | "no-inferrable-types": false 12 | } 13 | } --------------------------------------------------------------------------------