├── .vscode
└── settings.json
├── README.md
├── assets
├── logo.png
└── logo.svg
├── examples
├── nn.ts
└── perceptron.ts
├── mod.ts
└── src
├── Connection.ts
├── Cost.ts
├── Layer.js
├── LayerConnection.ts
├── Network.js
├── Neuron.js
├── Squash.ts
├── Trainer.js
├── architect.ts
└── architectures
├── Hopfield.js
├── LSTM.js
├── Liquid.js
└── Perceptron.js
/.vscode/settings.json:
--------------------------------------------------------------------------------
1 | {
2 | "deno.enable": true,
3 | "deno.lint": true,
4 | "deno.unstable": true
5 | }
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 | 
4 |
5 |
6 |
7 |
8 | dependency-less neural network library
9 | Synaptic.js ported to Deno
10 |
11 | ## Usage:
12 |
13 | ```ts
14 |
15 | import { Layer, Network, Trainer } from "./mod.ts";
16 |
17 | class Perceptron extends Network {
18 | public constructor(input: any, hidden: any, output: any) {
19 | super();
20 | let inputLayer = new Layer(input);
21 | let hiddenLayer = new Layer(hidden);
22 | let outputLayer = new Layer(output);
23 |
24 | inputLayer.project(hiddenLayer);
25 | hiddenLayer.project(outputLayer);
26 |
27 | this.set({
28 | input: inputLayer,
29 | hidden: [hiddenLayer],
30 | output: outputLayer,
31 | });
32 | }
33 | }
34 |
35 |
36 | let myPerceptron = new Perceptron(2,3,1);
37 |
38 | let myTrainer = new Trainer(myPerceptron);
39 |
40 | myTrainer.XOR();
41 |
42 | myPerceptron.activate([0,0]); // 0.0268581547421616
43 | myPerceptron.activate([1,0]); // 0.9829673642853368
44 | myPerceptron.activate([0,1]); // 0.9831714267395621
45 | myPerceptron.activate([1,1]); // 0.02128894618097928
46 | ```
47 |
48 | ```
49 | deno run -A ./test.ts
50 | ```
--------------------------------------------------------------------------------
/assets/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/load1n9/synaptic/18b5007f36667063e13345ab09c7dad8839e1a93/assets/logo.png
--------------------------------------------------------------------------------
/assets/logo.svg:
--------------------------------------------------------------------------------
1 |
2 |
166 |
--------------------------------------------------------------------------------
/examples/nn.ts:
--------------------------------------------------------------------------------
1 | import { Layer, Network } from "../mod.ts";
2 |
3 | const inputLayer: Layer = new Layer(2);
4 | const hiddenLayer: Layer = new Layer(3);
5 | const outputLayer: Layer = new Layer(1);
6 |
7 | inputLayer.project(hiddenLayer);
8 | hiddenLayer.project(outputLayer);
9 |
10 | const myNetwork: Network = new Network({
11 | input: inputLayer,
12 | hidden: [hiddenLayer],
13 | output: outputLayer
14 | });
15 |
16 | const learningRate = .3;
17 |
18 | for (let i:number = 0; i < 20000; i++) {
19 |
20 | myNetwork.activate([0,0]);
21 | myNetwork.propagate(learningRate, [0]);
22 |
23 | myNetwork.activate([0,1]);
24 | myNetwork.propagate(learningRate, [1]);
25 |
26 | myNetwork.activate([1,0]);
27 | myNetwork.propagate(learningRate, [1]);
28 |
29 | myNetwork.activate([1,1]);
30 | myNetwork.propagate(learningRate, [0]);
31 | }
32 |
33 | console.log(myNetwork.activate([0,0])); // [0.015020775950893527]
34 | console.log(myNetwork.activate([0,1])); // [0.9815816381088985]
35 | console.log(myNetwork.activate([1,0])); // [0.9871822457132193]
36 | console.log(myNetwork.activate([1,1])); // [0.012950087641929467]
37 |
--------------------------------------------------------------------------------
/examples/perceptron.ts:
--------------------------------------------------------------------------------
1 | import { Layer, Network, Trainer } from "../mod.ts";
2 |
3 | class Perceptron extends Network {
4 | public constructor(input: number, hidden: number, output: number) {
5 | super();
6 | const inputLayer = new Layer(input);
7 | const hiddenLayer = new Layer(hidden);
8 | const outputLayer = new Layer(output);
9 |
10 | inputLayer.project(hiddenLayer);
11 | hiddenLayer.project(outputLayer);
12 |
13 | this.set({
14 | input: inputLayer,
15 | hidden: [hiddenLayer],
16 | output: outputLayer,
17 | });
18 | }
19 | }
20 |
21 | const myPerceptron = new Perceptron(2, 3, 1);
22 |
23 | const myTrainer = new Trainer(myPerceptron);
24 |
25 | myTrainer.XOR();
26 |
27 | console.log(
28 | myPerceptron.activate([0, 0]), // 0.0268581547421616
29 | myPerceptron.activate([1, 0]), // 0.9829673642853368
30 | myPerceptron.activate([0, 1]), // 0.9831714267395621
31 | myPerceptron.activate([1, 1]), // 0.02128894618097928
32 | );
33 |
--------------------------------------------------------------------------------
/mod.ts:
--------------------------------------------------------------------------------
1 | export { Neuron } from "./src/Neuron.js";
2 | export { Layer } from "./src/Layer.js";
3 | export { Trainer } from "./src/Trainer.js";
4 | export { Network } from "./src/Network.js";
5 | import * as Architect from "./src/architect.ts";
6 | export { Architect };
7 |
--------------------------------------------------------------------------------
/src/Connection.ts:
--------------------------------------------------------------------------------
1 | export let connections = 0;
2 |
3 | export class Connection {
4 | public ID = Connection.uid();
5 | public gain = 1;
6 | public gater: any = null;
7 | public weight: any;
8 | public constructor(
9 | public from: any,
10 | public to: any,
11 | weight?: any,
12 | ) {
13 | this.weight = weight || Math.random() * .2 - .1;
14 | }
15 |
16 | static uid() {
17 | return connections++;
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/src/Cost.ts:
--------------------------------------------------------------------------------
1 | export const CROSS_ENTROPY = (target: number[], output: number[]) => {
2 | let crossentropy = 0;
3 | for (const i in output) {
4 | crossentropy -= (target[i] * Math.log(output[i] + 1e-15)) +
5 | ((1 - target[i]) * Math.log((1 + 1e-15) - output[i]));
6 | }
7 | return crossentropy;
8 | };
9 |
10 | export const MSE = (target: number[], output: number[]) => {
11 | let mse = 0;
12 | for (let i = 0; i < output.length; i++) {
13 | mse += Math.pow(target[i] - output[i], 2);
14 | }
15 | return mse / output.length;
16 | };
17 |
18 | export const BINARY = (target: number[], output: number[]) => {
19 | let misses = 0;
20 | for (let i = 0; i < output.length; i++) {
21 | misses += Number(Math.round(target[i] * 2) != Math.round(output[i] * 2));
22 | }
23 | return misses;
24 | };
25 |
--------------------------------------------------------------------------------
/src/Layer.js:
--------------------------------------------------------------------------------
1 | import { LayerConnection } from "./LayerConnection.ts";
2 | import { Neuron } from "./Neuron.js";
3 | import { Network } from "./Network.js";
4 |
5 | const connectionType = {
6 | ALL_TO_ALL: "ALL TO ALL",
7 | ONE_TO_ONE: "ONE TO ONE",
8 | ALL_TO_ELSE: "ALL TO ELSE",
9 | };
10 |
11 | const gateType = {
12 | INPUT: "INPUT",
13 | OUTPUT: "OUTPUT",
14 | ONE_TO_ONE: "ONE TO ONE",
15 | };
16 |
17 | export class Layer {
18 | static connectionType = connectionType;
19 | static gateType = gateType;
20 |
21 | constructor(size) {
22 | this.size = size | 0;
23 | this.list = [];
24 |
25 | this.connectedTo = [];
26 |
27 | while (size--) {
28 | let neuron = new Neuron();
29 | this.list.push(neuron);
30 | }
31 | }
32 |
33 | activate(input) {
34 | let activations = [];
35 |
36 | if (typeof input != "undefined") {
37 | if (input.length != this.size) {
38 | throw new Error(
39 | "INPUT size and LAYER size must be the same to activate!",
40 | );
41 | }
42 |
43 | for (let id in this.list) {
44 | let neuron = this.list[id];
45 | let activation = neuron.activate(input[id]);
46 | activations.push(activation);
47 | }
48 | } else {
49 | for (let id in this.list) {
50 | let neuron = this.list[id];
51 | let activation = neuron.activate();
52 | activations.push(activation);
53 | }
54 | }
55 | return activations;
56 | }
57 |
58 | propagate(rate, target) {
59 | if (typeof target != "undefined") {
60 | if (target.length != this.size) {
61 | throw new Error(
62 | "TARGET size and LAYER size must be the same to propagate!",
63 | );
64 | }
65 |
66 | for (let id = this.list.length - 1; id >= 0; id--) {
67 | let neuron = this.list[id];
68 | neuron.propagate(rate, target[id]);
69 | }
70 | } else {
71 | for (let id = this.list.length - 1; id >= 0; id--) {
72 | let neuron = this.list[id];
73 | neuron.propagate(rate);
74 | }
75 | }
76 | }
77 |
78 | project(layer, type, weights) {
79 | if (layer instanceof Network) {
80 | layer = layer.layers.input;
81 | }
82 |
83 | if (layer instanceof Layer) {
84 | if (!this.connected(layer)) {
85 | return new LayerConnection(this, layer, type, weights);
86 | }
87 | } else {
88 | throw new Error(
89 | "Invalid argument, you can only project connections to LAYERS and NETWORKS!",
90 | );
91 | }
92 | }
93 |
94 | gate(connection, type) {
95 | if (type == Layer.gateType.INPUT) {
96 | if (connection.to.size != this.size) {
97 | throw new Error(
98 | "GATER layer and CONNECTION.TO layer must be the same size in order to gate!",
99 | );
100 | }
101 |
102 | for (let id in connection.to.list) {
103 | let neuron = connection.to.list[id];
104 | let gater = this.list[id];
105 | for (let input in neuron.connections.inputs) {
106 | let gated = neuron.connections.inputs[input];
107 | if (gated.ID in connection.connections) {
108 | gater.gate(gated);
109 | }
110 | }
111 | }
112 | } else if (type == Layer.gateType.OUTPUT) {
113 | if (connection.from.size != this.size) {
114 | throw new Error(
115 | "GATER layer and CONNECTION.FROM layer must be the same size in order to gate!",
116 | );
117 | }
118 |
119 | for (var id in connection.from.list) {
120 | let neuron = connection.from.list[id];
121 | let gater = this.list[id];
122 | for (let projected in neuron.connections.projected) {
123 | let gated = neuron.connections.projected[projected];
124 | if (gated.ID in connection.connections) {
125 | gater.gate(gated);
126 | }
127 | }
128 | }
129 | } else if (type == Layer.gateType.ONE_TO_ONE) {
130 | if (connection.size != this.size) {
131 | throw new Error(
132 | "The number of GATER UNITS must be the same as the number of CONNECTIONS to gate!",
133 | );
134 | }
135 |
136 | for (let id in connection.list) {
137 | let gater = this.list[id];
138 | let gated = connection.list[id];
139 | gater.gate(gated);
140 | }
141 | }
142 | connection.gatedfrom.push({ layer: this, type: type });
143 | }
144 |
145 | selfconnected() {
146 | for (let id in this.list) {
147 | let neuron = this.list[id];
148 | if (!neuron.selfconnected()) {
149 | return false;
150 | }
151 | }
152 | return true;
153 | }
154 |
155 | connected(layer) {
156 | let connections = 0;
157 | for (let here in this.list) {
158 | for (let there in layer.list) {
159 | let from = this.list[here];
160 | let to = layer.list[there];
161 | let connected = from.connected(to);
162 | if (connected.type == "projected") {
163 | connections++;
164 | }
165 | }
166 | }
167 | if (connections == this.size * layer.size) {
168 | return Layer.connectionType.ALL_TO_ALL;
169 | }
170 |
171 | connections = 0;
172 | for (let neuron in this.list) {
173 | let from = this.list[neuron];
174 | let to = layer.list[neuron];
175 | let connected = from.connected(to);
176 | if (connected.type == "projected") {
177 | connections++;
178 | }
179 | }
180 | if (connections == this.size) {
181 | return Layer.connectionType.ONE_TO_ONE;
182 | }
183 | }
184 |
185 | clear() {
186 | for (let id in this.list) {
187 | let neuron = this.list[id];
188 | neuron.clear();
189 | }
190 | }
191 |
192 | reset() {
193 | for (let id in this.list) {
194 | let neuron = this.list[id];
195 | neuron.reset();
196 | }
197 | }
198 |
199 | neurons() {
200 | return this.list;
201 | }
202 |
203 | add(neuron) {
204 | neuron = neuron || new Neuron();
205 | this.list.push(neuron);
206 | this.size++;
207 | }
208 |
209 | set(options) {
210 | options = options || {};
211 |
212 | for (let i in this.list) {
213 | let neuron = this.list[i];
214 | if (options.label) {
215 | neuron.label = options.label + "_" + neuron.ID;
216 | }
217 | if (options.squash) {
218 | neuron.squash = options.squash;
219 | }
220 | if (options.bias) {
221 | neuron.bias = options.bias;
222 | }
223 | }
224 | return this;
225 | }
226 | }
227 |
--------------------------------------------------------------------------------
/src/LayerConnection.ts:
--------------------------------------------------------------------------------
1 | import { Layer } from "./Layer.js";
2 |
3 | export let connections = 0;
4 |
5 | export class LayerConnection {
6 | public ID: number;
7 | public from: any;
8 | public to: any;
9 | public selfconnection: any;
10 | public connections: any;
11 | public list: any;
12 | public size: number;
13 | public gatedfrom: any;
14 |
15 | public constructor(
16 | fromLayer: any,
17 | toLayer: any,
18 | public type?: any,
19 | weights?: any,
20 | ) {
21 | this.ID = LayerConnection.uid();
22 | this.from = fromLayer;
23 | this.to = toLayer;
24 | this.selfconnection = toLayer == fromLayer;
25 | this.connections = {};
26 | this.list = [];
27 | this.size = 0;
28 | this.gatedfrom = [];
29 |
30 | if (typeof this.type == "undefined") {
31 | if (fromLayer == toLayer) {
32 | this.type = Layer.connectionType.ONE_TO_ONE;
33 | } else {
34 | this.type = Layer.connectionType.ALL_TO_ALL;
35 | }
36 | }
37 |
38 | if (
39 | this.type == Layer.connectionType.ALL_TO_ALL ||
40 | this.type == Layer.connectionType.ALL_TO_ELSE
41 | ) {
42 | for (let here in this.from.list) {
43 | for (let there in this.to.list) {
44 | let from = this.from.list[here];
45 | let to = this.to.list[there];
46 | if (this.type == Layer.connectionType.ALL_TO_ELSE && from == to) {
47 | continue;
48 | }
49 | let connection = from.project(to, weights);
50 |
51 | this.connections[connection.ID] = connection;
52 | this.size = this.list.push(connection);
53 | }
54 | }
55 | } else if (this.type == Layer.connectionType.ONE_TO_ONE) {
56 | for (let neuron in this.from.list) {
57 | let from = this.from.list[neuron];
58 | let to = this.to.list[neuron];
59 | let connection = from.project(to, weights);
60 |
61 | this.connections[connection.ID] = connection;
62 | this.size = this.list.push(connection);
63 | }
64 | }
65 |
66 | fromLayer.connectedTo.push(this);
67 | }
68 |
69 | static uid() {
70 | return connections++;
71 | }
72 | }
73 |
--------------------------------------------------------------------------------
/src/Network.js:
--------------------------------------------------------------------------------
1 | import { Neuron } from "./Neuron.js";
2 | import { Layer } from "./Layer.js";
3 | import { Trainer } from "./Trainer.js";
4 |
5 | export class Network {
6 | constructor(layers) {
7 | if (typeof layers != "undefined") {
8 | this.layers = {
9 | input: layers.input || null,
10 | hidden: layers.hidden || [],
11 | output: layers.output || null,
12 | };
13 | this.optimized = null;
14 | }
15 | }
16 |
17 | activate(input) {
18 | if (this.optimized === false) {
19 | this.layers.input.activate(input);
20 | for (var i = 0; i < this.layers.hidden.length; i++) {
21 | this.layers.hidden[i].activate();
22 | }
23 | return this.layers.output.activate();
24 | } else {
25 | if (this.optimized == null) {
26 | this.optimize();
27 | }
28 | return this.optimized.activate(input);
29 | }
30 | }
31 |
32 | propagate(rate, target) {
33 | if (this.optimized === false) {
34 | this.layers.output.propagate(rate, target);
35 | for (let i = this.layers.hidden.length - 1; i >= 0; i--) {
36 | this.layers.hidden[i].propagate(rate);
37 | }
38 | } else {
39 | if (this.optimized == null) {
40 | this.optimize();
41 | }
42 | this.optimized.propagate(rate, target);
43 | }
44 | }
45 |
46 | project(unit, type, weights) {
47 | if (this.optimized) {
48 | this.optimized.reset();
49 | }
50 |
51 | if (unit instanceof Network) {
52 | return this.layers.output.project(unit.layers.input, type, weights);
53 | }
54 |
55 | if (unit instanceof Layer) {
56 | return this.layers.output.project(unit, type, weights);
57 | }
58 |
59 | throw new Error(
60 | "Invalid argument, you can only project connections to LAYERS and NETWORKS!",
61 | );
62 | }
63 |
64 | gate(connection, type) {
65 | if (this.optimized) {
66 | this.optimized.reset();
67 | }
68 | this.layers.output.gate(connection, type);
69 | }
70 |
71 | clear() {
72 | this.restore();
73 |
74 | let inputLayer = this.layers.input,
75 | outputLayer = this.layers.output;
76 |
77 | inputLayer.clear();
78 | for (let i = 0; i < this.layers.hidden.length; i++) {
79 | this.layers.hidden[i].clear();
80 | }
81 | outputLayer.clear();
82 |
83 | if (this.optimized) {
84 | this.optimized.reset();
85 | }
86 | }
87 |
88 | reset() {
89 | this.restore();
90 |
91 | let inputLayer = this.layers.input,
92 | outputLayer = this.layers.output;
93 |
94 | inputLayer.reset();
95 | for (let i = 0; i < this.layers.hidden.length; i++) {
96 | this.layers.hidden[i].reset();
97 | }
98 | outputLayer.reset();
99 |
100 | if (this.optimized) {
101 | this.optimized.reset();
102 | }
103 | }
104 |
105 | optimize() {
106 | let that = this;
107 | let optimized = {};
108 | let neurons = this.neurons();
109 |
110 | for (let i = 0; i < neurons.length; i++) {
111 | let neuron = neurons[i].neuron;
112 | let layer = neurons[i].layer;
113 | while (neuron.neuron) {
114 | neuron = neuron.neuron;
115 | }
116 | optimized = neuron.optimize(optimized, layer);
117 | }
118 |
119 | for (let i = 0; i < optimized.propagation_sentences.length; i++) {
120 | optimized.propagation_sentences[i].reverse();
121 | }
122 | optimized.propagation_sentences.reverse();
123 |
124 | let hardcode = "";
125 | hardcode += "var F = Float64Array ? new Float64Array(" + optimized.memory +
126 | ") : []; ";
127 | for (let i in optimized.variables) {
128 | hardcode += "F[" + optimized.variables[i].id + "] = " +
129 | (optimized.variables[
130 | i
131 | ].value || 0) + "; ";
132 | }
133 | hardcode += "var activate = function(input){\n";
134 | for (let i = 0; i < optimized.inputs.length; i++) {
135 | hardcode += "F[" + optimized.inputs[i] + "] = input[" + i + "]; ";
136 | }
137 | for (let i = 0; i < optimized.activation_sentences.length; i++) {
138 | if (optimized.activation_sentences[i].length > 0) {
139 | for (let j = 0; j < optimized.activation_sentences[i].length; j++) {
140 | hardcode += optimized.activation_sentences[i][j].join(" ");
141 | hardcode += optimized.trace_sentences[i][j].join(" ");
142 | }
143 | }
144 | }
145 | hardcode += " var output = []; ";
146 | for (let i = 0; i < optimized.outputs.length; i++) {
147 | hardcode += "output[" + i + "] = F[" + optimized.outputs[i] + "]; ";
148 | }
149 | hardcode += "return output; }; ";
150 | hardcode += "var propagate = function(rate, target){\n";
151 | hardcode += "F[" + optimized.variables.rate.id + "] = rate; ";
152 | for (let i = 0; i < optimized.targets.length; i++) {
153 | hardcode += "F[" + optimized.targets[i] + "] = target[" + i + "]; ";
154 | }
155 | for (let i = 0; i < optimized.propagation_sentences.length; i++) {
156 | for (let j = 0; j < optimized.propagation_sentences[i].length; j++) {
157 | hardcode += optimized.propagation_sentences[i][j].join(" ") + " ";
158 | }
159 | }
160 | hardcode += " };\n";
161 | hardcode +=
162 | "var ownership = function(memoryBuffer){\nF = memoryBuffer;\nthis.memory = F;\n};\n";
163 | hardcode +=
164 | "return {\nmemory: F,\nactivate: activate,\npropagate: propagate,\nownership: ownership\n};";
165 | hardcode = hardcode.split(";").join(";\n");
166 |
167 | let constructor = new Function(hardcode);
168 |
169 | let network = constructor();
170 | network.data = {
171 | variables: optimized.variables,
172 | activate: optimized.activation_sentences,
173 | propagate: optimized.propagation_sentences,
174 | trace: optimized.trace_sentences,
175 | inputs: optimized.inputs,
176 | outputs: optimized.outputs,
177 | check_activation: this.activate,
178 | check_propagation: this.propagate,
179 | };
180 |
181 | network.reset = () => {
182 | if (that.optimized) {
183 | that.optimized = null;
184 | that.activate = network.data.check_activation;
185 | that.propagate = network.data.check_propagation;
186 | }
187 | };
188 |
189 | this.optimized = network;
190 | this.activate = network.activate;
191 | this.propagate = network.propagate;
192 | }
193 |
194 | restore() {
195 | if (!this.optimized) {
196 | return;
197 | }
198 |
199 | let optimized = this.optimized;
200 |
201 | let getValue = () => {
202 | let args = Array.prototype.slice.call(arguments);
203 |
204 | let unit = args.shift();
205 | let prop = args.pop();
206 |
207 | let id = prop + "_";
208 | for (let property in args) {
209 | id += args[property] + "_";
210 | }
211 | id += unit.ID;
212 |
213 | let memory = optimized.memory;
214 | let variables = optimized.data.variables;
215 |
216 | if (id in variables) {
217 | return memory[variables[id].id];
218 | }
219 | return 0;
220 | };
221 |
222 | let list = this.neurons();
223 |
224 | for (let i = 0; i < list.length; i++) {
225 | let neuron = list[i].neuron;
226 | while (neuron.neuron) {
227 | neuron = neuron.neuron;
228 | }
229 |
230 | neuron.state = getValue(neuron, "state");
231 | neuron.old = getValue(neuron, "old");
232 | neuron.activation = getValue(neuron, "activation");
233 | neuron.bias = getValue(neuron, "bias");
234 |
235 | for (let input in neuron.trace.elegibility) {
236 | neuron.trace.elegibility[input] = getValue(
237 | neuron,
238 | "trace",
239 | "elegibility",
240 | input,
241 | );
242 | }
243 |
244 | for (let gated in neuron.trace.extended) {
245 | for (let input in neuron.trace.extended[gated]) {
246 | neuron.trace.extended[gated][input] = getValue(
247 | neuron,
248 | "trace",
249 | "extended",
250 | gated,
251 | input,
252 | );
253 | }
254 | }
255 |
256 | for (let j in neuron.connections.projected) {
257 | let connection = neuron.connections.projected[j];
258 | connection.weight = getValue(connection, "weight");
259 | connection.gain = getValue(connection, "gain");
260 | }
261 | }
262 | }
263 |
264 | neurons() {
265 | let neurons = [];
266 |
267 | let inputLayer = this.layers.input.neurons(),
268 | outputLayer = this.layers.output.neurons();
269 |
270 | for (let i = 0; i < inputLayer.length; i++) {
271 | neurons.push({
272 | neuron: inputLayer[i],
273 | layer: "input",
274 | });
275 | }
276 |
277 | for (let i = 0; i < this.layers.hidden.length; i++) {
278 | let hiddenLayer = this.layers.hidden[i].neurons();
279 | for (let j = 0; j < hiddenLayer.length; j++) {
280 | neurons.push({
281 | neuron: hiddenLayer[j],
282 | layer: i,
283 | });
284 | }
285 | }
286 |
287 | for (let i = 0; i < outputLayer.length; i++) {
288 | neurons.push({
289 | neuron: outputLayer[i],
290 | layer: "output",
291 | });
292 | }
293 |
294 | return neurons;
295 | }
296 |
297 | inputs() {
298 | return this.layers.input.size;
299 | }
300 |
301 | outputs() {
302 | return this.layers.output.size;
303 | }
304 |
305 | set(layers) {
306 | this.layers = {
307 | input: layers.input || null,
308 | hidden: layers.hidden || [],
309 | output: layers.output || null,
310 | };
311 | if (this.optimized) {
312 | this.optimized.reset();
313 | }
314 | }
315 |
316 | setOptimize(bool) {
317 | this.restore();
318 | if (this.optimized) {
319 | this.optimized.reset();
320 | }
321 | this.optimized = bool ? null : false;
322 | }
323 |
324 | toJSON(ignoreTraces) {
325 | this.restore();
326 |
327 | let list = this.neurons();
328 | let neurons = [];
329 | let connections = [];
330 |
331 | let ids = {};
332 | for (let i = 0; i < list.length; i++) {
333 | let neuron = list[i].neuron;
334 | while (neuron.neuron) {
335 | neuron = neuron.neuron;
336 | }
337 | ids[neuron.ID] = i;
338 |
339 | let copy = {
340 | trace: {
341 | elegibility: {},
342 | extended: {},
343 | },
344 | state: neuron.state,
345 | old: neuron.old,
346 | activation: neuron.activation,
347 | bias: neuron.bias,
348 | layer: list[i].layer,
349 | };
350 |
351 | copy.squash = neuron.squash == Neuron.squash.LOGISTIC
352 | ? "LOGISTIC"
353 | : neuron.squash == Neuron.squash.TANH
354 | ? "TANH"
355 | : neuron.squash == Neuron.squash.IDENTITY
356 | ? "IDENTITY"
357 | : neuron.squash == Neuron.squash.HLIM
358 | ? "HLIM"
359 | : neuron.squash == Neuron.squash.RELU
360 | ? "RELU"
361 | : null;
362 |
363 | neurons.push(copy);
364 | }
365 |
366 | for (let i = 0; i < list.length; i++) {
367 | let neuron = list[i].neuron;
368 | while (neuron.neuron) {
369 | neuron = neuron.neuron;
370 | }
371 |
372 | for (let j in neuron.connections.projected) {
373 | let connection = neuron.connections.projected[j];
374 | connections.push({
375 | from: ids[connection.from.ID],
376 | to: ids[connection.to.ID],
377 | weight: connection.weight,
378 | gater: connection.gater ? ids[connection.gater.ID] : null,
379 | });
380 | }
381 | if (neuron.selfconnected()) {
382 | connections.push({
383 | from: ids[neuron.ID],
384 | to: ids[neuron.ID],
385 | weight: neuron.selfconnection.weight,
386 | gater: neuron.selfconnection.gater
387 | ? ids[neuron.selfconnection.gater.ID]
388 | : null,
389 | });
390 | }
391 | }
392 |
393 | return {
394 | neurons: neurons,
395 | connections: connections,
396 | };
397 | }
398 |
399 | toDot(edgeConnection) {
400 | if (!typeof edgeConnection) {
401 | edgeConnection = false;
402 | }
403 | let code = "digraph nn {\n rankdir = BT\n";
404 | let layers = [this.layers.input].concat(
405 | this.layers.hidden,
406 | this.layers.output,
407 | );
408 | for (let i = 0; i < layers.length; i++) {
409 | for (let j = 0; j < layers[i].connectedTo.length; j++) {
410 | let connection = layers[i].connectedTo[j];
411 | let layerTo = connection.to;
412 | let size = connection.size;
413 | let layerID = layers.indexOf(layers[i]);
414 | let layerToID = layers.indexOf(layerTo);
415 | if (edgeConnection) {
416 | if (connection.gatedfrom.length) {
417 | let fakeNode = "fake" + layerID + "_" + layerToID;
418 | code += " " + fakeNode +
419 | ' [label = "", shape = point, width = 0.01, height = 0.01]\n';
420 | code += " " + layerID + " -> " + fakeNode + " [label = " + size +
421 | ", arrowhead = none]\n";
422 | code += " " + fakeNode + " -> " + layerToID + "\n";
423 | } else {
424 | code += " " + layerID + " -> " + layerToID + " [label = " +
425 | size + "]\n";
426 | }
427 | for (let from in connection.gatedfrom) { // gatings
428 | let layerfrom = connection.gatedfrom[from].layer;
429 | let layerfromID = layers.indexOf(layerfrom);
430 | code += " " + layerfromID + " -> " + fakeNode +
431 | " [color = blue]\n";
432 | }
433 | } else {
434 | code += " " + layerID + " -> " + layerToID + " [label = " + size +
435 | "]\n";
436 | for (let from in connection.gatedfrom) { // gatings
437 | let layerfrom = connection.gatedfrom[from].layer;
438 | let layerfromID = layers.indexOf(layerfrom);
439 | code += " " + layerfromID + " -> " + layerToID +
440 | " [color = blue]\n";
441 | }
442 | }
443 | }
444 | }
445 | code += "}\n";
446 | return {
447 | code: code,
448 | link: "https://chart.googleapis.com/chart?chl=" +
449 | escape(code.replace("/ /g", "+")) + "&cht=gv",
450 | };
451 | }
452 |
453 | standalone() {
454 | if (!this.optimized) {
455 | this.optimize();
456 | }
457 |
458 | let data = this.optimized.data;
459 |
460 | let activation = "function (input) {\n";
461 |
462 | for (let i = 0; i < data.inputs.length; i++) {
463 | activation += "F[" + data.inputs[i] + "] = input[" + i + "];\n";
464 | }
465 |
466 | for (let i = 0; i < data.activate.length; i++) {
467 | for (let j = 0; j < data.activate[i].length; j++) {
468 | activation += data.activate[i][j].join("") + "\n";
469 | }
470 | }
471 |
472 | activation += "var output = [];\n";
473 | for (let i = 0; i < data.outputs.length; i++) {
474 | activation += "output[" + i + "] = F[" + data.outputs[i] + "];\n";
475 | }
476 | activation += "return output;\n}";
477 |
478 | let memory = activation.match(/F\[(\d+)\]/g);
479 | let dimension = 0;
480 | let ids = {};
481 |
482 | for (let i = 0; i < memory.length; i++) {
483 | let tmp = memory[i].match(/\d+/)[0];
484 | if (!(tmp in ids)) {
485 | ids[tmp] = dimension++;
486 | }
487 | }
488 | let hardcode = "F = {\n";
489 |
490 | for (let i in ids) {
491 | hardcode += ids[i] + ": " + this.optimized.memory[i] + ",\n";
492 | }
493 | hardcode = hardcode.substring(0, hardcode.length - 2) + "\n};\n";
494 | hardcode = "var run = " +
495 | activation.replace(/F\[(\d+)]/g, function (index) {
496 | return "F[" + ids[index.match(/\d+/)[0]] + "]";
497 | }).replace("{\n", "{\n" + hardcode + "") + ";\n";
498 | hardcode += "return run";
499 |
500 | return new Function(hardcode)();
501 | }
502 |
503 | worker(memory, set, options) {
504 | let workerOptions = {};
505 | if (options) workerOptions = options;
506 | workerOptions.rate = workerOptions.rate || .2;
507 | workerOptions.iterations = workerOptions.iterations || 100000;
508 | workerOptions.error = workerOptions.error || .005;
509 | workerOptions.cost = workerOptions.cost || null;
510 | workerOptions.crossValidate = workerOptions.crossValidate || null;
511 |
512 | let costFunction = "// REPLACED BY WORKER\nvar cost = " +
513 | (options && options.cost || this.cost || Trainer.cost.MSE) + ";\n";
514 | let workerFunction = Network.getWorkerSharedFunctions();
515 | workerFunction = workerFunction.replace(
516 | /var cost = options && options\.cost \|\| this\.cost \|\| Trainer\.cost\.MSE;/g,
517 | costFunction,
518 | );
519 |
520 | workerFunction = workerFunction.replace(
521 | "return results;",
522 | 'postMessage({action: "done", message: results, memoryBuffer: F}, [F.buffer]);',
523 | );
524 |
525 | workerFunction = workerFunction.replace(
526 | "console.log('iterations', iterations, 'error', error, 'rate', currentRate)",
527 | "postMessage({action: 'log', message: {\n" +
528 | "iterations: iterations,\n" +
529 | "error: error,\n" +
530 | "rate: currentRate\n" +
531 | "}\n" +
532 | "})",
533 | );
534 |
535 | workerFunction = workerFunction.replace(
536 | "abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate })",
537 | "postMessage({action: 'schedule', message: {\n" +
538 | "iterations: iterations,\n" +
539 | "error: error,\n" +
540 | "rate: currentRate\n" +
541 | "}\n" +
542 | "})",
543 | );
544 |
545 | if (!this.optimized) {
546 | this.optimize();
547 | }
548 |
549 | let hardcode = "var inputs = " + this.optimized.data.inputs.length + ";\n";
550 | hardcode += "var outputs = " + this.optimized.data.outputs.length + ";\n";
551 | hardcode += "var F = new Float64Array([" +
552 | this.optimized.memory.toString() + "]);\n";
553 | hardcode += "var activate = " + this.optimized.activate.toString() + ";\n";
554 | hardcode += "var propagate = " + this.optimized.propagate.toString() +
555 | ";\n";
556 | hardcode += "onmessage = function(e) {\n" +
557 | "if (e.data.action == 'startTraining') {\n" +
558 | "train(" + JSON.stringify(set) + "," + JSON.stringify(workerOptions) +
559 | ");\n" +
560 | "}\n" +
561 | "}";
562 |
563 | let workerSourceCode = workerFunction + "\n" + hardcode;
564 | let blob = new Blob([workerSourceCode]);
565 | let blobURL = window.URL.createObjectURL(blob);
566 |
567 | return new Worker(blobURL);
568 | }
569 |
570 | clone() {
571 | return Network.fromJSON(this.toJSON());
572 | }
573 |
574 | static getWorkerSharedFunctions() {
575 | if (typeof Network._SHARED_WORKER_FUNCTIONS !== "undefined") {
576 | return Network._SHARED_WORKER_FUNCTIONS;
577 | }
578 | let train_f = Trainer.prototype.train.toString();
579 | train_f = train_f.replace(/this._trainSet/g, "_trainSet");
580 | train_f = train_f.replace(/this.test/g, "test");
581 | train_f = train_f.replace(/this.crossValidate/g, "crossValidate");
582 | train_f = train_f.replace("crossValidate = true", "// REMOVED BY WORKER");
583 |
584 | let _trainSet_f = Trainer.prototype._trainSet.toString().replace(
585 | /this.network./g,
586 | "",
587 | );
588 |
589 | let test_f = Trainer.prototype.test.toString().replace(
590 | /this.network./g,
591 | "",
592 | );
593 |
594 | return Network._SHARED_WORKER_FUNCTIONS = train_f + "\n" + _trainSet_f +
595 | "\n" + test_f;
596 | }
597 |
598 | static fromJSON(json) {
599 | let neurons = [];
600 |
601 | let layers = {
602 | input: new Layer(),
603 | hidden: [],
604 | output: new Layer(),
605 | };
606 |
607 | for (let i = 0; i < json.neurons.length; i++) {
608 | let config = json.neurons[i];
609 |
610 | let neuron = new Neuron();
611 | neuron.trace.elegibility = {};
612 | neuron.trace.extended = {};
613 | neuron.state = config.state;
614 | neuron.old = config.old;
615 | neuron.activation = config.activation;
616 | neuron.bias = config.bias;
617 | neuron.squash = config.squash in Neuron.squash
618 | ? Neuron.squash[config.squash]
619 | : Neuron.squash.LOGISTIC;
620 | neurons.push(neuron);
621 |
622 | if (config.layer == "input") {
623 | layers.input.add(neuron);
624 | } else if (config.layer == "output") {
625 | layers.output.add(neuron);
626 | } else {
627 | if (typeof layers.hidden[config.layer] == "undefined") {
628 | layers.hidden[config.layer] = new Layer();
629 | }
630 | layers.hidden[config.layer].add(neuron);
631 | }
632 | }
633 |
634 | for (let i = 0; i < json.connections.length; i++) {
635 | let config = json.connections[i];
636 | let from = neurons[config.from];
637 | let to = neurons[config.to];
638 | let weight = config.weight;
639 | let gater = neurons[config.gater];
640 |
641 | let connection = from.project(to, weight);
642 | if (gater) {
643 | gater.gate(connection);
644 | }
645 | }
646 |
647 | return new Network(layers);
648 | }
649 | }
650 |
--------------------------------------------------------------------------------
/src/Neuron.js:
--------------------------------------------------------------------------------
1 | import { Connection, connections } from './Connection.ts';
2 | import * as squash from './Squash.ts';
3 | let neurons = 0;
4 |
5 |
6 |
7 | export class Neuron {
8 | static squash = squash;
9 |
10 | constructor() {
11 | this.ID = Neuron.uid();
12 |
13 | this.connections = {
14 | inputs: {},
15 | projected: {},
16 | gated: {}
17 | };
18 | this.error = {
19 | responsibility: 0,
20 | projected: 0,
21 | gated: 0
22 | };
23 | this.trace = {
24 | elegibility: {},
25 | extended: {},
26 | influences: {}
27 | };
28 | this.state = 0;
29 | this.old = 0;
30 | this.activation = 0;
31 | this.selfconnection = new Connection(this, this, 0);
32 | this.squash = Neuron.squash.LOGISTIC;
33 | this.neighboors = {};
34 | this.bias = Math.random() * .2 - .1;
35 | }
36 |
37 | activate(input) {
38 | if (typeof input != 'undefined') {
39 | this.activation = input;
40 | this.derivative = 0;
41 | this.bias = 0;
42 | return this.activation;
43 | }
44 |
45 | this.old = this.state;
46 |
47 | this.state = this.selfconnection.gain * this.selfconnection.weight *
48 | this.state + this.bias;
49 |
50 | for (var i in this.connections.inputs) {
51 | var input = this.connections.inputs[i];
52 | this.state += input.from.activation * input.weight * input.gain;
53 | }
54 |
55 | this.activation = this.squash(this.state);
56 |
57 | this.derivative = this.squash(this.state, true);
58 |
59 | var influences = [];
60 | for (var id in this.trace.extended) {
61 | var neuron = this.neighboors[id];
62 |
63 | var influence = neuron.selfconnection.gater == this ? neuron.old : 0;
64 |
65 | for (var incoming in this.trace.influences[neuron.ID]) {
66 | influence += this.trace.influences[neuron.ID][incoming].weight *
67 | this.trace.influences[neuron.ID][incoming].from.activation;
68 | }
69 | influences[neuron.ID] = influence;
70 | }
71 |
72 | for (var i in this.connections.inputs) {
73 | var input = this.connections.inputs[i];
74 |
75 | this.trace.elegibility[input.ID] = this.selfconnection.gain * this.selfconnection
76 | .weight * this.trace.elegibility[input.ID] + input.gain * input.from
77 | .activation;
78 |
79 | for (var id in this.trace.extended) {
80 | var xtrace = this.trace.extended[id];
81 | var neuron = this.neighboors[id];
82 | var influence = influences[neuron.ID];
83 |
84 | xtrace[input.ID] = neuron.selfconnection.gain * neuron.selfconnection
85 | .weight * xtrace[input.ID] + this.derivative * this.trace.elegibility[
86 | input.ID] * influence;
87 | }
88 | }
89 |
90 | for (var connection in this.connections.gated) {
91 | this.connections.gated[connection].gain = this.activation;
92 | }
93 |
94 | return this.activation;
95 | }
96 |
97 | propagate(rate, target) {
98 | var error = 0;
99 |
100 | var isOutput = typeof target != 'undefined';
101 |
102 | if (isOutput)
103 | this.error.responsibility = this.error.projected = target - this.activation;
104 |
105 | else {
106 | for (var id in this.connections.projected) {
107 | var connection = this.connections.projected[id];
108 | var neuron = connection.to;
109 | error += neuron.error.responsibility * connection.gain * connection.weight;
110 | }
111 |
112 | this.error.projected = this.derivative * error;
113 |
114 | error = 0;
115 | for (var id in this.trace.extended) {
116 | var neuron = this.neighboors[id];
117 | var influence = neuron.selfconnection.gater == this ? neuron.old : 0;
118 |
119 | for (var input in this.trace.influences[id]) {
120 | influence += this.trace.influences[id][input].weight * this.trace.influences[
121 | neuron.ID][input].from.activation;
122 | }
123 | error += neuron.error.responsibility * influence;
124 | }
125 |
126 | this.error.gated = this.derivative * error;
127 |
128 | this.error.responsibility = this.error.projected + this.error.gated;
129 | }
130 |
131 | rate = rate || .1;
132 |
133 | for (var id in this.connections.inputs) {
134 | var input = this.connections.inputs[id];
135 |
136 | var gradient = this.error.projected * this.trace.elegibility[input.ID];
137 | for (var id in this.trace.extended) {
138 | var neuron = this.neighboors[id];
139 | gradient += neuron.error.responsibility * this.trace.extended[
140 | neuron.ID][input.ID];
141 | }
142 | input.weight += rate * gradient;
143 | }
144 |
145 | this.bias += rate * this.error.responsibility;
146 | }
147 |
148 | project(neuron, weight) {
149 | if (neuron == this) {
150 | this.selfconnection.weight = 1;
151 | return this.selfconnection;
152 | }
153 |
154 | var connected = this.connected(neuron);
155 | if (connected && connected.type == 'projected') {
156 | if (typeof weight != 'undefined')
157 | connected.connection.weight = weight;
158 | return connected.connection;
159 | } else {
160 | var connection = new Connection(this, neuron, weight);
161 | }
162 |
163 | this.connections.projected[connection.ID] = connection;
164 | this.neighboors[neuron.ID] = neuron;
165 | neuron.connections.inputs[connection.ID] = connection;
166 | neuron.trace.elegibility[connection.ID] = 0;
167 |
168 | for (var id in neuron.trace.extended) {
169 | var trace = neuron.trace.extended[id];
170 | trace[connection.ID] = 0;
171 | }
172 |
173 | return connection;
174 | }
175 |
176 | gate(connection) {
177 | this.connections.gated[connection.ID] = connection;
178 |
179 | var neuron = connection.to;
180 | if (!(neuron.ID in this.trace.extended)) {
181 | this.neighboors[neuron.ID] = neuron;
182 | var xtrace = this.trace.extended[neuron.ID] = {};
183 | for (var id in this.connections.inputs) {
184 | var input = this.connections.inputs[id];
185 | xtrace[input.ID] = 0;
186 | }
187 | }
188 |
189 | if (neuron.ID in this.trace.influences)
190 | this.trace.influences[neuron.ID].push(connection);
191 | else
192 | this.trace.influences[neuron.ID] = [connection];
193 |
194 | connection.gater = this;
195 | }
196 |
197 | selfconnected() {
198 | return this.selfconnection.weight !== 0;
199 | }
200 |
201 | connected(neuron) {
202 | var result = {
203 | type: null,
204 | connection: false
205 | };
206 |
207 | if (this == neuron) {
208 | if (this.selfconnected()) {
209 | result.type = 'selfconnection';
210 | result.connection = this.selfconnection;
211 | return result;
212 | } else
213 | return false;
214 | }
215 |
216 | for (var type in this.connections) {
217 | for (var connection in this.connections[type]) {
218 | var connection = this.connections[type][connection];
219 | if (connection.to == neuron) {
220 | result.type = type;
221 | result.connection = connection;
222 | return result;
223 | } else if (connection.from == neuron) {
224 | result.type = type;
225 | result.connection = connection;
226 | return result;
227 | }
228 | }
229 | }
230 |
231 | return false;
232 | }
233 |
234 | clear() {
235 | for (const trace in this.trace.elegibility) {
236 | this.trace.elegibility[trace] = 0;
237 | }
238 |
239 | for (const trace in this.trace.extended) {
240 | for (const extended in this.trace.extended[trace]) {
241 | this.trace.extended[trace][extended] = 0;
242 | }
243 | }
244 |
245 | this.error.responsibility = this.error.projected = this.error.gated = 0;
246 | }
247 |
248 | reset() {
249 | this.clear();
250 |
251 | for (var type in this.connections) {
252 | for (var connection in this.connections[type]) {
253 | this.connections[type][connection].weight = Math.random() * .2 - .1;
254 | }
255 | }
256 |
257 | this.bias = Math.random() * .2 - .1;
258 | this.old = this.state = this.activation = 0;
259 | }
260 |
261 | optimize(optimized, layer) {
262 |
263 | optimized = optimized || {};
264 | var store_activation = [];
265 | var store_trace = [];
266 | var store_propagation = [];
267 | var varID = optimized.memory || 0;
268 | var neurons = optimized.neurons || 1;
269 | var inputs = optimized.inputs || [];
270 | var targets = optimized.targets || [];
271 | var outputs = optimized.outputs || [];
272 | var variables = optimized.variables || {};
273 | var activation_sentences = optimized.activation_sentences || [];
274 | var trace_sentences = optimized.trace_sentences || [];
275 | var propagation_sentences = optimized.propagation_sentences || [];
276 | var layers = optimized.layers || { __count: 0, __neuron: 0 };
277 |
278 | var allocate = function (store) {
279 | var allocated = layer in layers && store[layers.__count];
280 | if (!allocated) {
281 | layers.__count = store.push([]) - 1;
282 | layers[layer] = layers.__count;
283 | }
284 | };
285 | allocate(activation_sentences);
286 | allocate(trace_sentences);
287 | allocate(propagation_sentences);
288 | var currentLayer = layers.__count;
289 |
290 | var getVar = function () {
291 | var args = Array.prototype.slice.call(arguments);
292 |
293 | if (args.length == 1) {
294 | if (args[0] == 'target') {
295 | var id = 'target_' + targets.length;
296 | targets.push(varID);
297 | } else
298 | var id = args[0];
299 | if (id in variables)
300 | return variables[id];
301 | return variables[id] = {
302 | value: 0,
303 | id: varID++
304 | };
305 | } else {
306 | var extended = args.length > 2;
307 | if (extended)
308 | var value = args.pop();
309 |
310 | var unit = args.shift();
311 | var prop = args.pop();
312 |
313 | if (!extended)
314 | var value = unit[prop];
315 |
316 | var id = prop + '_';
317 | for (var i = 0; i < args.length; i++)
318 | id += args[i] + '_';
319 | id += unit.ID;
320 | if (id in variables)
321 | return variables[id];
322 |
323 | return variables[id] = {
324 | value: value,
325 | id: varID++
326 | };
327 | }
328 | };
329 |
330 | var buildSentence = function () {
331 | var args = Array.prototype.slice.call(arguments);
332 | var store = args.pop();
333 | var sentence = '';
334 | for (var i = 0; i < args.length; i++)
335 | if (typeof args[i] == 'string')
336 | sentence += args[i];
337 | else
338 | sentence += 'F[' + args[i].id + ']';
339 |
340 | store.push(sentence + ';');
341 | };
342 |
343 | var isEmpty = function (obj) {
344 | for (var prop in obj) {
345 | if (obj.hasOwnProperty(prop))
346 | return false;
347 | }
348 | return true;
349 | };
350 |
351 | var noProjections = isEmpty(this.connections.projected);
352 | var noGates = isEmpty(this.connections.gated);
353 | var isInput = layer == 'input' ? true : isEmpty(this.connections.inputs);
354 | var isOutput = layer == 'output' ? true : noProjections && noGates;
355 |
356 | var rate = getVar('rate');
357 | var activation = getVar(this, 'activation');
358 | if (isInput)
359 | inputs.push(activation.id);
360 | else {
361 | activation_sentences[currentLayer].push(store_activation);
362 | trace_sentences[currentLayer].push(store_trace);
363 | propagation_sentences[currentLayer].push(store_propagation);
364 | var old = getVar(this, 'old');
365 | var state = getVar(this, 'state');
366 | var bias = getVar(this, 'bias');
367 | if (this.selfconnection.gater)
368 | var self_gain = getVar(this.selfconnection, 'gain');
369 | if (this.selfconnected())
370 | var self_weight = getVar(this.selfconnection, 'weight');
371 | buildSentence(old, ' = ', state, store_activation);
372 | if (this.selfconnected())
373 | if (this.selfconnection.gater)
374 | buildSentence(state, ' = ', self_gain, ' * ', self_weight, ' * ',
375 | state, ' + ', bias, store_activation);
376 | else
377 | buildSentence(state, ' = ', self_weight, ' * ', state, ' + ',
378 | bias, store_activation);
379 | else
380 | buildSentence(state, ' = ', bias, store_activation);
381 | for (var i in this.connections.inputs) {
382 | var input = this.connections.inputs[i];
383 | var input_activation = getVar(input.from, 'activation');
384 | var input_weight = getVar(input, 'weight');
385 | if (input.gater)
386 | var input_gain = getVar(input, 'gain');
387 | if (this.connections.inputs[i].gater)
388 | buildSentence(state, ' += ', input_activation, ' * ',
389 | input_weight, ' * ', input_gain, store_activation);
390 | else
391 | buildSentence(state, ' += ', input_activation, ' * ',
392 | input_weight, store_activation);
393 | }
394 | var derivative = getVar(this, 'derivative');
395 | switch (this.squash) {
396 | case Neuron.squash.LOGISTIC:
397 | buildSentence(activation, ' = (1 / (1 + Math.exp(-', state, ')))',
398 | store_activation);
399 | buildSentence(derivative, ' = ', activation, ' * (1 - ',
400 | activation, ')', store_activation);
401 | break;
402 | case Neuron.squash.TANH:
403 | var eP = getVar('aux');
404 | var eN = getVar('aux_2');
405 | buildSentence(eP, ' = Math.exp(', state, ')', store_activation);
406 | buildSentence(eN, ' = 1 / ', eP, store_activation);
407 | buildSentence(activation, ' = (', eP, ' - ', eN, ') / (', eP, ' + ', eN, ')', store_activation);
408 | buildSentence(derivative, ' = 1 - (', activation, ' * ', activation, ')', store_activation);
409 | break;
410 | case Neuron.squash.IDENTITY:
411 | buildSentence(activation, ' = ', state, store_activation);
412 | buildSentence(derivative, ' = 1', store_activation);
413 | break;
414 | case Neuron.squash.HLIM:
415 | buildSentence(activation, ' = +(', state, ' > 0)', store_activation);
416 | buildSentence(derivative, ' = 1', store_activation);
417 | break;
418 | case Neuron.squash.RELU:
419 | buildSentence(activation, ' = ', state, ' > 0 ? ', state, ' : 0', store_activation);
420 | buildSentence(derivative, ' = ', state, ' > 0 ? 1 : 0', store_activation);
421 | break;
422 | }
423 |
424 | for (var id in this.trace.extended) {
425 | var neuron = this.neighboors[id];
426 | var influence = getVar('influences[' + neuron.ID + ']');
427 | var neuron_old = getVar(neuron, 'old');
428 | var initialized = false;
429 | if (neuron.selfconnection.gater == this) {
430 | buildSentence(influence, ' = ', neuron_old, store_trace);
431 | initialized = true;
432 | }
433 | for (var incoming in this.trace.influences[neuron.ID]) {
434 | var incoming_weight = getVar(this.trace.influences[neuron.ID]
435 | [incoming], 'weight');
436 | var incoming_activation = getVar(this.trace.influences[neuron.ID]
437 | [incoming].from, 'activation');
438 |
439 | if (initialized)
440 | buildSentence(influence, ' += ', incoming_weight, ' * ', incoming_activation, store_trace);
441 | else {
442 | buildSentence(influence, ' = ', incoming_weight, ' * ', incoming_activation, store_trace);
443 | initialized = true;
444 | }
445 | }
446 | }
447 |
448 | for (var i in this.connections.inputs) {
449 | var input = this.connections.inputs[i];
450 | if (input.gater)
451 | var input_gain = getVar(input, 'gain');
452 | var input_activation = getVar(input.from, 'activation');
453 | var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
454 | .elegibility[input.ID]);
455 | if (this.selfconnected()) {
456 | if (this.selfconnection.gater) {
457 | if (input.gater)
458 | buildSentence(trace, ' = ', self_gain, ' * ', self_weight,
459 | ' * ', trace, ' + ', input_gain, ' * ', input_activation,
460 | store_trace);
461 | else
462 | buildSentence(trace, ' = ', self_gain, ' * ', self_weight,
463 | ' * ', trace, ' + ', input_activation, store_trace);
464 | } else {
465 | if (input.gater)
466 | buildSentence(trace, ' = ', self_weight, ' * ', trace, ' + ',
467 | input_gain, ' * ', input_activation, store_trace);
468 | else
469 | buildSentence(trace, ' = ', self_weight, ' * ', trace, ' + ',
470 | input_activation, store_trace);
471 | }
472 | } else {
473 | if (input.gater)
474 | buildSentence(trace, ' = ', input_gain, ' * ', input_activation,
475 | store_trace);
476 | else
477 | buildSentence(trace, ' = ', input_activation, store_trace);
478 | }
479 | for (var id in this.trace.extended) {
480 | var neuron = this.neighboors[id];
481 | var influence = getVar('influences[' + neuron.ID + ']');
482 |
483 | var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
484 | .elegibility[input.ID]);
485 | var xtrace = getVar(this, 'trace', 'extended', neuron.ID, input.ID,
486 | this.trace.extended[neuron.ID][input.ID]);
487 | if (neuron.selfconnected())
488 | var neuron_self_weight = getVar(neuron.selfconnection, 'weight');
489 | if (neuron.selfconnection.gater)
490 | var neuron_self_gain = getVar(neuron.selfconnection, 'gain');
491 | if (neuron.selfconnected())
492 | if (neuron.selfconnection.gater)
493 | buildSentence(xtrace, ' = ', neuron_self_gain, ' * ',
494 | neuron_self_weight, ' * ', xtrace, ' + ', derivative, ' * ',
495 | trace, ' * ', influence, store_trace);
496 | else
497 | buildSentence(xtrace, ' = ', neuron_self_weight, ' * ',
498 | xtrace, ' + ', derivative, ' * ', trace, ' * ',
499 | influence, store_trace);
500 | else
501 | buildSentence(xtrace, ' = ', derivative, ' * ', trace, ' * ',
502 | influence, store_trace);
503 | }
504 | }
505 | for (var connection in this.connections.gated) {
506 | var gated_gain = getVar(this.connections.gated[connection], 'gain');
507 | buildSentence(gated_gain, ' = ', activation, store_activation);
508 | }
509 | }
510 | if (!isInput) {
511 | var responsibility = getVar(this, 'error', 'responsibility', this.error
512 | .responsibility);
513 | if (isOutput) {
514 | var target = getVar('target');
515 | buildSentence(responsibility, ' = ', target, ' - ', activation,
516 | store_propagation);
517 | for (var id in this.connections.inputs) {
518 | var input = this.connections.inputs[id];
519 | var trace = getVar(this, 'trace', 'elegibility', input.ID, this.trace
520 | .elegibility[input.ID]);
521 | var input_weight = getVar(input, 'weight');
522 | buildSentence(input_weight, ' += ', rate, ' * (', responsibility,
523 | ' * ', trace, ')', store_propagation);
524 | }
525 | outputs.push(activation.id);
526 | } else {
527 | if (!noProjections && !noGates) {
528 | var error = getVar('aux');
529 | for (var id in this.connections.projected) {
530 | var connection = this.connections.projected[id];
531 | var neuron = connection.to;
532 | var connection_weight = getVar(connection, 'weight');
533 | var neuron_responsibility = getVar(neuron, 'error',
534 | 'responsibility', neuron.error.responsibility);
535 | if (connection.gater) {
536 | var connection_gain = getVar(connection, 'gain');
537 | buildSentence(error, ' += ', neuron_responsibility, ' * ',
538 | connection_gain, ' * ', connection_weight,
539 | store_propagation);
540 | } else
541 | buildSentence(error, ' += ', neuron_responsibility, ' * ',
542 | connection_weight, store_propagation);
543 | }
544 | var projected = getVar(this, 'error', 'projected', this.error.projected);
545 | buildSentence(projected, ' = ', derivative, ' * ', error,
546 | store_propagation);
547 | buildSentence(error, ' = 0', store_propagation);
548 | for (var id in this.trace.extended) {
549 | var neuron = this.neighboors[id];
550 | var influence = getVar('aux_2');
551 | var neuron_old = getVar(neuron, 'old');
552 | if (neuron.selfconnection.gater == this)
553 | buildSentence(influence, ' = ', neuron_old, store_propagation);
554 | else
555 | buildSentence(influence, ' = 0', store_propagation);
556 | for (var input in this.trace.influences[neuron.ID]) {
557 | var connection = this.trace.influences[neuron.ID][input];
558 | var connection_weight = getVar(connection, 'weight');
559 | var neuron_activation = getVar(connection.from, 'activation');
560 | buildSentence(influence, ' += ', connection_weight, ' * ',
561 | neuron_activation, store_propagation);
562 | }
563 | var neuron_responsibility = getVar(neuron, 'error',
564 | 'responsibility', neuron.error.responsibility);
565 | buildSentence(error, ' += ', neuron_responsibility, ' * ',
566 | influence, store_propagation);
567 | }
568 | var gated = getVar(this, 'error', 'gated', this.error.gated);
569 | buildSentence(gated, ' = ', derivative, ' * ', error,
570 | store_propagation);
571 | buildSentence(responsibility, ' = ', projected, ' + ', gated,
572 | store_propagation);
573 | for (var id in this.connections.inputs) {
574 | var input = this.connections.inputs[id];
575 | var gradient = getVar('aux');
576 | var trace = getVar(this, 'trace', 'elegibility', input.ID, this
577 | .trace.elegibility[input.ID]);
578 | buildSentence(gradient, ' = ', projected, ' * ', trace,
579 | store_propagation);
580 | for (var id in this.trace.extended) {
581 | var neuron = this.neighboors[id];
582 | var neuron_responsibility = getVar(neuron, 'error',
583 | 'responsibility', neuron.error.responsibility);
584 | var xtrace = getVar(this, 'trace', 'extended', neuron.ID,
585 | input.ID, this.trace.extended[neuron.ID][input.ID]);
586 | buildSentence(gradient, ' += ', neuron_responsibility, ' * ',
587 | xtrace, store_propagation);
588 | }
589 | var input_weight = getVar(input, 'weight');
590 | buildSentence(input_weight, ' += ', rate, ' * ', gradient,
591 | store_propagation);
592 | }
593 |
594 | } else if (noGates) {
595 | buildSentence(responsibility, ' = 0', store_propagation);
596 | for (var id in this.connections.projected) {
597 | var connection = this.connections.projected[id];
598 | var neuron = connection.to;
599 | var connection_weight = getVar(connection, 'weight');
600 | var neuron_responsibility = getVar(neuron, 'error',
601 | 'responsibility', neuron.error.responsibility);
602 | if (connection.gater) {
603 | var connection_gain = getVar(connection, 'gain');
604 | buildSentence(responsibility, ' += ', neuron_responsibility,
605 | ' * ', connection_gain, ' * ', connection_weight,
606 | store_propagation);
607 | } else
608 | buildSentence(responsibility, ' += ', neuron_responsibility,
609 | ' * ', connection_weight, store_propagation);
610 | }
611 | buildSentence(responsibility, ' *= ', derivative,
612 | store_propagation);
613 | for (var id in this.connections.inputs) {
614 | var input = this.connections.inputs[id];
615 | var trace = getVar(this, 'trace', 'elegibility', input.ID, this
616 | .trace.elegibility[input.ID]);
617 | var input_weight = getVar(input, 'weight');
618 | buildSentence(input_weight, ' += ', rate, ' * (',
619 | responsibility, ' * ', trace, ')', store_propagation);
620 | }
621 | } else if (noProjections) {
622 | buildSentence(responsibility, ' = 0', store_propagation);
623 | for (var id in this.trace.extended) {
624 | var neuron = this.neighboors[id];
625 | var influence = getVar('aux');
626 | var neuron_old = getVar(neuron, 'old');
627 | if (neuron.selfconnection.gater == this)
628 | buildSentence(influence, ' = ', neuron_old, store_propagation);
629 | else
630 | buildSentence(influence, ' = 0', store_propagation);
631 | for (var input in this.trace.influences[neuron.ID]) {
632 | var connection = this.trace.influences[neuron.ID][input];
633 | var connection_weight = getVar(connection, 'weight');
634 | var neuron_activation = getVar(connection.from, 'activation');
635 | buildSentence(influence, ' += ', connection_weight, ' * ',
636 | neuron_activation, store_propagation);
637 | }
638 | var neuron_responsibility = getVar(neuron, 'error',
639 | 'responsibility', neuron.error.responsibility);
640 | buildSentence(responsibility, ' += ', neuron_responsibility,
641 | ' * ', influence, store_propagation);
642 | }
643 | buildSentence(responsibility, ' *= ', derivative,
644 | store_propagation);
645 | for (var id in this.connections.inputs) {
646 | var input = this.connections.inputs[id];
647 | var gradient = getVar('aux');
648 | buildSentence(gradient, ' = 0', store_propagation);
649 | for (var id in this.trace.extended) {
650 | var neuron = this.neighboors[id];
651 | var neuron_responsibility = getVar(neuron, 'error',
652 | 'responsibility', neuron.error.responsibility);
653 | var xtrace = getVar(this, 'trace', 'extended', neuron.ID,
654 | input.ID, this.trace.extended[neuron.ID][input.ID]);
655 | buildSentence(gradient, ' += ', neuron_responsibility, ' * ',
656 | xtrace, store_propagation);
657 | }
658 | var input_weight = getVar(input, 'weight');
659 | buildSentence(input_weight, ' += ', rate, ' * ', gradient,
660 | store_propagation);
661 | }
662 | }
663 | }
664 | buildSentence(bias, ' += ', rate, ' * ', responsibility,
665 | store_propagation);
666 | }
667 | return {
668 | memory: varID,
669 | neurons: neurons + 1,
670 | inputs: inputs,
671 | outputs: outputs,
672 | targets: targets,
673 | variables: variables,
674 | activation_sentences: activation_sentences,
675 | trace_sentences: trace_sentences,
676 | propagation_sentences: propagation_sentences,
677 | layers: layers
678 | }
679 | }
680 |
681 | static uid() {
682 | return neurons++;
683 | }
684 |
685 | static quantity() {
686 | return {
687 | neurons: neurons,
688 | connections: connections
689 | }
690 | }
691 | }
--------------------------------------------------------------------------------
/src/Squash.ts:
--------------------------------------------------------------------------------
1 | export const LOGISTIC = (x: number, derivate?: boolean) => {
2 | const fx = 1 / (1 + Math.exp(-x));
3 | if (!derivate) {
4 | return fx;
5 | }
6 | return fx * (1 - fx);
7 | };
8 |
9 | export const TANH = (x: number, derivate?: boolean) => {
10 | if (derivate) {
11 | return 1 - Math.pow(Math.tanh(x), 2);
12 | }
13 | return Math.tanh(x);
14 | };
15 |
16 | export const IDENTITY = (x: number, derivate?: boolean) => {
17 | return derivate ? 1 : x;
18 | };
19 |
20 | export const HLIM = (x: number, derivate?: boolean) => {
21 | return derivate ? 1 : x > 0 ? 1 : 0;
22 | };
23 |
24 | export const RELU = (x: number, derivate?: boolean) => {
25 | if (derivate) {
26 | return x > 0 ? 1 : 0;
27 | }
28 | return x > 0 ? x : 0;
29 | };
30 |
--------------------------------------------------------------------------------
/src/Trainer.js:
--------------------------------------------------------------------------------
1 | import * as cost from './Cost.ts';
2 |
3 | const shuffleInplace = (o) => {
4 | for (let j, x, i = o.length; i; j = Math.floor(Math.random() * i), x = o[--i], o[i] = o[j], o[j] = x);
5 | return o;
6 | };
7 |
8 |
9 |
10 | export class Trainer {
11 | static cost = cost;
12 |
13 | constructor(network, options) {
14 | options = options || {};
15 | this.network = network;
16 | this.rate = options.rate || .2;
17 | this.iterations = options.iterations || 100000;
18 | this.error = options.error || .005;
19 | this.cost = options.cost || null;
20 | this.crossValidate = options.crossValidate || null;
21 | }
22 |
23 | train(set, options) {
24 | var error = 1;
25 | var iterations = bucketSize = 0;
26 | var abort = false;
27 | var currentRate;
28 | var cost = options && options.cost || this.cost || Trainer.cost.MSE;
29 | var crossValidate = false, testSet, trainSet;
30 |
31 | var start = Date.now();
32 |
33 | if (options) {
34 | if (options.iterations)
35 | this.iterations = options.iterations;
36 | if (options.error)
37 | this.error = options.error;
38 | if (options.rate)
39 | this.rate = options.rate;
40 | if (options.cost)
41 | this.cost = options.cost;
42 | if (options.schedule)
43 | this.schedule = options.schedule;
44 | if (options.customLog) {
45 | console.log('Deprecated: use schedule instead of customLog')
46 | this.schedule = options.customLog;
47 | }
48 | if (this.crossValidate || options.crossValidate) {
49 | if (!this.crossValidate) this.crossValidate = {};
50 | crossValidate = true;
51 | if (options.crossValidate.testSize)
52 | this.crossValidate.testSize = options.crossValidate.testSize;
53 | if (options.crossValidate.testError)
54 | this.crossValidate.testError = options.crossValidate.testError;
55 | }
56 | }
57 |
58 | currentRate = this.rate;
59 | if (Array.isArray(this.rate)) {
60 | var bucketSize = Math.floor(this.iterations / this.rate.length);
61 | }
62 |
63 | if (crossValidate) {
64 | var numTrain = Math.ceil((1 - this.crossValidate.testSize) * set.length);
65 | trainSet = set.slice(0, numTrain);
66 | testSet = set.slice(numTrain);
67 | }
68 |
69 | var lastError = 0;
70 | while ((!abort && iterations < this.iterations && error > this.error)) {
71 | if (crossValidate && error <= this.crossValidate.testError) {
72 | break;
73 | }
74 |
75 | var currentSetSize = set.length;
76 | error = 0;
77 | iterations++;
78 |
79 | if (bucketSize > 0) {
80 | var currentBucket = Math.floor(iterations / bucketSize);
81 | currentRate = this.rate[currentBucket] || currentRate;
82 | }
83 |
84 | if (typeof this.rate === 'function') {
85 | currentRate = this.rate(iterations, lastError);
86 | }
87 |
88 | if (crossValidate) {
89 | this._trainSet(trainSet, currentRate, cost);
90 | error += this.test(testSet).error;
91 | currentSetSize = 1;
92 | } else {
93 | error += this._trainSet(set, currentRate, cost);
94 | currentSetSize = set.length;
95 | }
96 |
97 | error /= currentSetSize;
98 | lastError = error;
99 |
100 | if (options) {
101 | if (this.schedule && this.schedule.every && iterations %
102 | this.schedule.every == 0)
103 | abort = this.schedule.do({ error: error, iterations: iterations, rate: currentRate });
104 | else if (options.log && iterations % options.log == 0) {
105 | console.log('iterations', iterations, 'error', error, 'rate', currentRate);
106 | }
107 | if (options.shuffle)
108 | shuffleInplace(set);
109 | }
110 | }
111 |
112 | const results = {
113 | error: error,
114 | iterations: iterations,
115 | time: Date.now() - start
116 | };
117 |
118 | return results;
119 | }
120 |
121 | trainAsync(set, options) {
122 | var train = this.workerTrain.bind(this);
123 | return new Promise(function (resolve, reject) {
124 | try {
125 | train(set, resolve, options, true)
126 | } catch (e) {
127 | reject(e)
128 | }
129 | })
130 | }
131 |
132 | _trainSet(set, currentRate, costFunction) {
133 | var errorSum = 0;
134 | for (let i = 0; i < set.length; i++) {
135 | const input = set[i].input;
136 | const target = set[i].output;
137 |
138 | const output = this.network.activate(input);
139 | this.network.propagate(currentRate, target);
140 |
141 | errorSum += costFunction(target, output);
142 | }
143 | return errorSum;
144 | }
145 |
146 | test(set, options) {
147 | var error = 0;
148 | var input, output, target;
149 | var cost = options && options.cost || this.cost || Trainer.cost.MSE;
150 |
151 | var start = Date.now();
152 |
153 | for (let i = 0; i < set.length; i++) {
154 | input = set[i].input;
155 | target = set[i].output;
156 | output = this.network.activate(input);
157 | error += cost(target, output);
158 | }
159 |
160 | error /= set.length;
161 |
162 | var results = {
163 | error: error,
164 | time: Date.now() - start
165 | };
166 |
167 | return results;
168 | }
169 |
170 | workerTrain(set, callback, options, suppressWarning) {
171 | if (!suppressWarning) {
172 | console.warn('Deprecated: do not use `workerTrain`, use `trainAsync` instead.')
173 | }
174 | var that = this;
175 |
176 | if (!this.network.optimized)
177 | this.network.optimize();
178 |
179 | var worker = this.network.worker(this.network.optimized.memory, set, options);
180 |
181 | worker.onmessage = function (e) {
182 | switch (e.data.action) {
183 | case 'done':
184 | var iterations = e.data.message.iterations;
185 | var error = e.data.message.error;
186 | var time = e.data.message.time;
187 |
188 | that.network.optimized.ownership(e.data.memoryBuffer);
189 |
190 | callback({
191 | error: error,
192 | iterations: iterations,
193 | time: time
194 | });
195 |
196 | worker.terminate();
197 | break;
198 |
199 | case 'log':
200 | console.log(e.data.message);
201 |
202 | case 'schedule':
203 | if (options && options.schedule && typeof options.schedule.do === 'function') {
204 | var scheduled = options.schedule.do
205 | scheduled(e.data.message)
206 | }
207 | break;
208 | }
209 | };
210 |
211 | worker.postMessage({ action: 'startTraining' });
212 | }
213 |
214 | XOR(options) {
215 | if (this.network.inputs() != 2 || this.network.outputs() != 1)
216 | throw new Error('Incompatible network (2 inputs, 1 output)');
217 |
218 | var defaults = {
219 | iterations: 100000,
220 | log: false,
221 | shuffle: true,
222 | cost: Trainer.cost.MSE
223 | };
224 |
225 | if (options)
226 | for (var i in options)
227 | defaults[i] = options[i];
228 |
229 | return this.train([{
230 | input: [0, 0],
231 | output: [0]
232 | }, {
233 | input: [1, 0],
234 | output: [1]
235 | }, {
236 | input: [0, 1],
237 | output: [1]
238 | }, {
239 | input: [1, 1],
240 | output: [0]
241 | }], defaults);
242 | }
243 |
244 | DSR(options) {
245 | options = options || {};
246 |
247 | var targets = options.targets || [2, 4, 7, 8];
248 | var distractors = options.distractors || [3, 5, 6, 9];
249 | var prompts = options.prompts || [0, 1];
250 | var length = options.length || 24;
251 | var criterion = options.success || 0.95;
252 | var iterations = options.iterations || 100000;
253 | var rate = options.rate || .1;
254 | var log = options.log || 0;
255 | var schedule = options.schedule || {};
256 | var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
257 |
258 | var trial, correct, i, j, success;
259 | trial = correct = i = j = success = 0;
260 | var error = 1,
261 | symbols = targets.length + distractors.length + prompts.length;
262 |
263 | var noRepeat = function (range, avoid) {
264 | var number = Math.random() * range | 0;
265 | var used = false;
266 | for (var i in avoid)
267 | if (number == avoid[i])
268 | used = true;
269 | return used ? noRepeat(range, avoid) : number;
270 | };
271 |
272 | var equal = function (prediction, output) {
273 | for (var i in prediction)
274 | if (Math.round(prediction[i]) != output[i])
275 | return false;
276 | return true;
277 | };
278 |
279 | var start = Date.now();
280 |
281 | while (trial < iterations && (success < criterion || trial % 1000 != 0)) {
282 | var sequence = [],
283 | sequenceLength = length - prompts.length;
284 | for (i = 0; i < sequenceLength; i++) {
285 | var any = Math.random() * distractors.length | 0;
286 | sequence.push(distractors[any]);
287 | }
288 | var indexes = [],
289 | positions = [];
290 | for (i = 0; i < prompts.length; i++) {
291 | indexes.push(Math.random() * targets.length | 0);
292 | positions.push(noRepeat(sequenceLength, positions));
293 | }
294 | positions = positions.sort();
295 | for (i = 0; i < prompts.length; i++) {
296 | sequence[positions[i]] = targets[indexes[i]];
297 | sequence.push(prompts[i]);
298 | }
299 |
300 | var distractorsCorrect;
301 | var targetsCorrect = distractorsCorrect = 0;
302 | error = 0;
303 | for (i = 0; i < length; i++) {
304 | var input = [];
305 | for (j = 0; j < symbols; j++)
306 | input[j] = 0;
307 | input[sequence[i]] = 1;
308 |
309 | var output = [];
310 | for (j = 0; j < targets.length; j++)
311 | output[j] = 0;
312 |
313 | if (i >= sequenceLength) {
314 | var index = i - sequenceLength;
315 | output[indexes[index]] = 1;
316 | }
317 |
318 | var prediction = this.network.activate(input);
319 |
320 | if (equal(prediction, output))
321 | if (i < sequenceLength)
322 | distractorsCorrect++;
323 | else
324 | targetsCorrect++;
325 | else {
326 | this.network.propagate(rate, output);
327 | }
328 |
329 | error += cost(output, prediction);
330 |
331 | if (distractorsCorrect + targetsCorrect == length)
332 | correct++;
333 | }
334 |
335 | if (trial % 1000 == 0)
336 | correct = 0;
337 | trial++;
338 | var divideError = trial % 1000;
339 | divideError = divideError == 0 ? 1000 : divideError;
340 | success = correct / divideError;
341 | error /= length;
342 |
343 | if (log && trial % log == 0)
344 | console.log('iterations:', trial, ' success:', success, ' correct:',
345 | correct, ' time:', Date.now() - start, ' error:', error);
346 | if (schedule.do && schedule.every && trial % schedule.every == 0)
347 | schedule.do({
348 | iterations: trial,
349 | success: success,
350 | error: error,
351 | time: Date.now() - start,
352 | correct: correct
353 | });
354 | }
355 |
356 | return {
357 | iterations: trial,
358 | success: success,
359 | error: error,
360 | time: Date.now() - start
361 | }
362 | }
363 |
364 | ERG(options) {
365 |
366 | options = options || {};
367 | var iterations = options.iterations || 150000;
368 | var criterion = options.error || .05;
369 | var rate = options.rate || .1;
370 | var log = options.log || 500;
371 | var cost = options.cost || this.cost || Trainer.cost.CROSS_ENTROPY;
372 |
373 | var Node = function () {
374 | this.paths = [];
375 | };
376 | Node.prototype = {
377 | connect: function (node, value) {
378 | this.paths.push({
379 | node: node,
380 | value: value
381 | });
382 | return this;
383 | },
384 | any: function () {
385 | if (this.paths.length == 0)
386 | return false;
387 | var index = Math.random() * this.paths.length | 0;
388 | return this.paths[index];
389 | },
390 | test: function (value) {
391 | for (var i in this.paths)
392 | if (this.paths[i].value == value)
393 | return this.paths[i];
394 | return false;
395 | }
396 | };
397 |
398 | var reberGrammar = function () {
399 |
400 | var output = new Node();
401 | var n1 = (new Node()).connect(output, 'E');
402 | var n2 = (new Node()).connect(n1, 'S');
403 | var n3 = (new Node()).connect(n1, 'V').connect(n2, 'P');
404 | var n4 = (new Node()).connect(n2, 'X');
405 | n4.connect(n4, 'S');
406 | var n5 = (new Node()).connect(n3, 'V');
407 | n5.connect(n5, 'T');
408 | n2.connect(n5, 'X');
409 | var n6 = (new Node()).connect(n4, 'T').connect(n5, 'P');
410 | var input = (new Node()).connect(n6, 'B');
411 |
412 | return {
413 | input: input,
414 | output: output
415 | }
416 | };
417 |
418 | var embededReberGrammar = function () {
419 | var reber1 = reberGrammar();
420 | var reber2 = reberGrammar();
421 |
422 | var output = new Node();
423 | var n1 = (new Node).connect(output, 'E');
424 | reber1.output.connect(n1, 'T');
425 | reber2.output.connect(n1, 'P');
426 | var n2 = (new Node).connect(reber1.input, 'P').connect(reber2.input,
427 | 'T');
428 | var input = (new Node).connect(n2, 'B');
429 |
430 | return {
431 | input: input,
432 | output: output
433 | }
434 |
435 | };
436 |
437 | var generate = function () {
438 | var node = embededReberGrammar().input;
439 | var next = node.any();
440 | var str = '';
441 | while (next) {
442 | str += next.value;
443 | next = next.node.any();
444 | }
445 | return str;
446 | };
447 |
448 | var test = function (str) {
449 | var node = embededReberGrammar().input;
450 | var i = 0;
451 | var ch = str.charAt(i);
452 | while (i < str.length) {
453 | var next = node.test(ch);
454 | if (!next)
455 | return false;
456 | node = next.node;
457 | ch = str.charAt(++i);
458 | }
459 | return true;
460 | };
461 |
462 | var different = function (array1, array2) {
463 | var max1 = 0;
464 | var i1 = -1;
465 | var max2 = 0;
466 | var i2 = -1;
467 | for (var i in array1) {
468 | if (array1[i] > max1) {
469 | max1 = array1[i];
470 | i1 = i;
471 | }
472 | if (array2[i] > max2) {
473 | max2 = array2[i];
474 | i2 = i;
475 | }
476 | }
477 |
478 | return i1 != i2;
479 | };
480 |
481 | var iteration = 0;
482 | var error = 1;
483 | var table = {
484 | 'B': 0,
485 | 'P': 1,
486 | 'T': 2,
487 | 'X': 3,
488 | 'S': 4,
489 | 'E': 5
490 | };
491 |
492 | var start = Date.now();
493 | while (iteration < iterations && error > criterion) {
494 | var i = 0;
495 | error = 0;
496 |
497 | var sequence = generate();
498 |
499 | var read = sequence.charAt(i);
500 | var predict = sequence.charAt(i + 1);
501 |
502 | while (i < sequence.length - 1) {
503 | var input = [];
504 | var target = [];
505 | for (var j = 0; j < 6; j++) {
506 | input[j] = 0;
507 | target[j] = 0;
508 | }
509 | input[table[read]] = 1;
510 | target[table[predict]] = 1;
511 |
512 | var output = this.network.activate(input);
513 |
514 | if (different(output, target))
515 | this.network.propagate(rate, target);
516 |
517 | read = sequence.charAt(++i);
518 | predict = sequence.charAt(i + 1);
519 |
520 | error += cost(target, output);
521 | }
522 | error /= sequence.length;
523 | iteration++;
524 | if (iteration % log == 0) {
525 | console.log('iterations:', iteration, ' time:', Date.now() - start,
526 | ' error:', error);
527 | }
528 | }
529 |
530 | return {
531 | iterations: iteration,
532 | error: error,
533 | time: Date.now() - start,
534 | test: test,
535 | generate: generate
536 | }
537 | }
538 |
539 | timingTask(options) {
540 |
541 | if (this.network.inputs() != 2 || this.network.outputs() != 1)
542 | throw new Error('Invalid Network: must have 2 inputs and one output');
543 |
544 | if (typeof options == 'undefined')
545 | options = {};
546 |
547 | function getSamples(trainingSize, testSize) {
548 |
549 | var size = trainingSize + testSize;
550 |
551 | var t = 0;
552 | var set = [];
553 | for (var i = 0; i < size; i++) {
554 | set.push({ input: [0, 0], output: [0] });
555 | }
556 | while (t < size - 20) {
557 | var n = Math.round(Math.random() * 20);
558 | set[t].input[0] = 1;
559 | for (var j = t; j <= t + n; j++) {
560 | set[j].input[1] = n / 20;
561 | set[j].output[0] = 0.5;
562 | }
563 | t += n;
564 | n = Math.round(Math.random() * 20);
565 | for (var k = t + 1; k <= (t + n) && k < size; k++)
566 | set[k].input[1] = set[t].input[1];
567 | t += n;
568 | }
569 |
570 | var trainingSet = [];
571 | var testSet = [];
572 | for (var l = 0; l < size; l++)
573 | (l < trainingSize ? trainingSet : testSet).push(set[l]);
574 |
575 | return {
576 | train: trainingSet,
577 | test: testSet
578 | }
579 | }
580 |
581 | var iterations = options.iterations || 200;
582 | var error = options.error || .005;
583 | var rate = options.rate || [.03, .02];
584 | var log = options.log === false ? false : options.log || 10;
585 | var cost = options.cost || this.cost || Trainer.cost.MSE;
586 | var trainingSamples = options.trainSamples || 7000;
587 | var testSamples = options.trainSamples || 1000;
588 |
589 | var samples = getSamples(trainingSamples, testSamples);
590 |
591 | var result = this.train(samples.train, {
592 | rate: rate,
593 | log: log,
594 | iterations: iterations,
595 | error: error,
596 | cost: cost
597 | });
598 |
599 | return {
600 | train: result,
601 | test: this.test(samples.test)
602 | }
603 | }
604 |
605 | }
--------------------------------------------------------------------------------
/src/architect.ts:
--------------------------------------------------------------------------------
1 | export { Perceptron } from './architectures/Perceptron.js';
2 | export { LSTM } from './architectures/LSTM.js';
3 | export { Liquid } from './architectures/Liquid.js';
4 | export { Hopfield } from './architectures/Hopfield.js';
--------------------------------------------------------------------------------
/src/architectures/Hopfield.js:
--------------------------------------------------------------------------------
1 | import { Network } from '../Network.js';
2 | import { Trainer } from '../Trainer.js';
3 | import { Layer } from '../Layer.js';
4 |
5 | export class Hopfield extends Network {
6 | constructor(size) {
7 | super();
8 | let inputLayer = new Layer(size);
9 | let outputLayer = new Layer(size);
10 |
11 | inputLayer.project(outputLayer, Layer.connectionType.ALL_TO_ALL);
12 |
13 | this.set({
14 | input: inputLayer,
15 | hidden: [],
16 | output: outputLayer
17 | });
18 |
19 | this.trainer = new Trainer(this);
20 | }
21 |
22 | learn(patterns) {
23 | let set = [];
24 | for (let p in patterns)
25 | set.push({
26 | input: patterns[p],
27 | output: patterns[p]
28 | });
29 |
30 | return this.trainer.train(set, {
31 | iterations: 500000,
32 | error: .00005,
33 | rate: 1
34 | });
35 | }
36 |
37 | feed(pattern) {
38 | let output = this.activate(pattern);
39 |
40 | pattern = [];
41 | for (let i in output)
42 | pattern[i] = output[i] > .5 ? 1 : 0;
43 |
44 | return pattern;
45 | }
46 | }
--------------------------------------------------------------------------------
/src/architectures/LSTM.js:
--------------------------------------------------------------------------------
1 | import { Network } from '../Network.js';
2 | import { Layer } from '../Layer.js';
3 |
4 | export class LSTM extends Network {
5 | constructor() {
6 | super();
7 | let args = Array.prototype.slice.call(arguments);
8 | if (args.length < 3)
9 | throw new Error("not enough layers (minimum 3) !!");
10 |
11 | let last = args.pop();
12 | let option = {
13 | peepholes: Layer.connectionType.ALL_TO_ALL,
14 | hiddenToHidden: false,
15 | outputToHidden: false,
16 | outputToGates: false,
17 | inputToOutput: true,
18 | };
19 | if (typeof last != 'number') {
20 | let outputs = args.pop();
21 | if (last.hasOwnProperty('peepholes'))
22 | option.peepholes = last.peepholes;
23 | if (last.hasOwnProperty('hiddenToHidden'))
24 | option.hiddenToHidden = last.hiddenToHidden;
25 | if (last.hasOwnProperty('outputToHidden'))
26 | option.outputToHidden = last.outputToHidden;
27 | if (last.hasOwnProperty('outputToGates'))
28 | option.outputToGates = last.outputToGates;
29 | if (last.hasOwnProperty('inputToOutput'))
30 | option.inputToOutput = last.inputToOutput;
31 | } else {
32 | let outputs = last;
33 | }
34 |
35 | let inputs = args.shift();
36 | let layers = args;
37 |
38 | let inputLayer = new Layer(inputs);
39 | let hiddenLayers = [];
40 | let outputLayer = new Layer(outputs);
41 |
42 | let previous = null;
43 |
44 | for (let i = 0; i < layers.length; i++) {
45 | let size = layers[i];
46 |
47 | let inputGate = new Layer(size).set({
48 | bias: 1
49 | });
50 | let forgetGate = new Layer(size).set({
51 | bias: 1
52 | });
53 | let memoryCell = new Layer(size);
54 | let outputGate = new Layer(size).set({
55 | bias: 1
56 | });
57 |
58 | hiddenLayers.push(inputGate);
59 | hiddenLayers.push(forgetGate);
60 | hiddenLayers.push(memoryCell);
61 | hiddenLayers.push(outputGate);
62 |
63 | let input = inputLayer.project(memoryCell);
64 | inputLayer.project(inputGate);
65 | inputLayer.project(forgetGate);
66 | inputLayer.project(outputGate);
67 |
68 | if (previous != null) {
69 | let cell = previous.project(memoryCell);
70 | previous.project(inputGate);
71 | previous.project(forgetGate);
72 | previous.project(outputGate);
73 | }
74 |
75 | let output = memoryCell.project(outputLayer);
76 |
77 | let self = memoryCell.project(memoryCell);
78 |
79 | if (option.hiddenToHidden)
80 | memoryCell.project(memoryCell, Layer.connectionType.ALL_TO_ELSE);
81 |
82 | if (option.outputToHidden)
83 | outputLayer.project(memoryCell);
84 |
85 | if (option.outputToGates) {
86 | outputLayer.project(inputGate);
87 | outputLayer.project(outputGate);
88 | outputLayer.project(forgetGate);
89 | }
90 |
91 | memoryCell.project(inputGate, option.peepholes);
92 | memoryCell.project(forgetGate, option.peepholes);
93 | memoryCell.project(outputGate, option.peepholes);
94 |
95 | inputGate.gate(input, Layer.gateType.INPUT);
96 | forgetGate.gate(self, Layer.gateType.ONE_TO_ONE);
97 | outputGate.gate(output, Layer.gateType.OUTPUT);
98 | if (previous != null)
99 | inputGate.gate(cell, Layer.gateType.INPUT);
100 |
101 | previous = memoryCell;
102 | }
103 |
104 | if (option.inputToOutput)
105 | inputLayer.project(outputLayer);
106 |
107 | this.set({
108 | input: inputLayer,
109 | hidden: hiddenLayers,
110 | output: outputLayer
111 | });
112 | }
113 | }
--------------------------------------------------------------------------------
/src/architectures/Liquid.js:
--------------------------------------------------------------------------------
1 | import { Network } from '../Network.js';
2 | import { Layer } from '../Layer.js';
3 |
4 | export class Liquid extends Network {
5 | constructor(inputs, hidden, outputs, connections, gates) {
6 | super();
7 | let inputLayer = new Layer(inputs);
8 | let hiddenLayer = new Layer(hidden);
9 | let outputLayer = new Layer(outputs);
10 |
11 | let neurons = hiddenLayer.neurons();
12 | let connectionList = [];
13 |
14 | for (let i = 0; i < connections; i++) {
15 | let from = Math.random() * neurons.length | 0;
16 | let to = Math.random() * neurons.length | 0;
17 | let connection = neurons[from].project(neurons[to]);
18 | connectionList.push(connection);
19 | }
20 |
21 | for (let j = 0; j < gates; j++) {
22 | let gater = Math.random() * neurons.length | 0;
23 | let connection = Math.random() * connectionList.length | 0;
24 | neurons[gater].gate(connectionList[connection]);
25 | }
26 |
27 | inputLayer.project(hiddenLayer);
28 | hiddenLayer.project(outputLayer);
29 |
30 | this.set({
31 | input: inputLayer,
32 | hidden: [hiddenLayer],
33 | output: outputLayer
34 | });
35 | }
36 | }
--------------------------------------------------------------------------------
/src/architectures/Perceptron.js:
--------------------------------------------------------------------------------
1 | import { Network } from '../Network.js';
2 | import { Layer } from '../Layer.js';
3 |
4 | export class Perceptron extends Network {
5 | constructor() {
6 | super();
7 | let args = Array.prototype.slice.call(arguments);
8 | if (args.length < 3)
9 | throw new Error('not enough layers (minimum 3) !!');
10 |
11 | let inputs = args.shift();
12 | let outputs = args.pop();
13 | let layers = args;
14 |
15 | let input = new Layer(inputs);
16 | let hidden = [];
17 | let output = new Layer(outputs);
18 |
19 | let previous = input;
20 |
21 | for (let i = 0; i < layers.length; i++) {
22 | let size = layers[i];
23 | let layer = new Layer(size);
24 | hidden.push(layer);
25 | previous.project(layer);
26 | previous = layer;
27 | }
28 | previous.project(output);
29 |
30 | this.set({
31 | input: input,
32 | hidden: hidden,
33 | output: output
34 | });
35 | }
36 | }
--------------------------------------------------------------------------------