├── src
    ├── neural
    │   ├── neuralnetwork.cpp
    │   ├── edge.h
    │   ├── edge.cpp
    │   ├── neuron.h
    │   ├── neuralnetwork.h
    │   ├── layer.h
    │   ├── layer.cpp
    │   └── neuron.cpp
    ├── optimizer
    │   ├── optimizer.h
    │   ├── shakingtree.h
    │   ├── backpropagation.h
    │   ├── optimizer.cpp
    │   ├── backpropagation.cpp
    │   └── shakingtree.cpp
    ├── misc
    │   ├── functions.h
    │   └── functions.cpp
    ├── dataset
    │   ├── dataset.h
    │   └── dataset.cpp
    └── neuralmain.cpp
├── LICENSE
├── README.md
├── .gitignore
└── data1000.txt


/src/neural/neuralnetwork.cpp:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Whiax/NeuralNetworkCpp/HEAD/src/neural/neuralnetwork.cpp


--------------------------------------------------------------------------------
/src/optimizer/optimizer.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "../dataset/dataset.h"
 4 | #include "../neural/neuralnetwork.h"
 5 | 
 6 | 
 7 | class Optimizer
 8 | {
 9 | public:
10 | 	Optimizer();
11 | 	~Optimizer();
12 | 
13 | 	virtual void minimize();
14 | 
15 | 	void setNeuralNetwork(NeuralNetwork* net);
16 | 
17 | 	void setDataset(Dataset* dataset);
18 | 
19 | 	double getScore(Datatype d, int limit = -1);
20 | 
21 | 	void minimizeThread();
22 | 
23 | protected:
24 | 	NeuralNetwork* _n;
25 | 
26 | 	Dataset* _d;
27 | 
28 | };
29 | 
30 | 


--------------------------------------------------------------------------------
/src/misc/functions.h:
--------------------------------------------------------------------------------
 1 | #ifndef FUNCTIONS_H
 2 | #define FUNCTIONS_H
 3 | 
 4 | 
 5 | #include <iostream>
 6 | #include <vector>
 7 | #include <utility>
 8 | #include <limits>
 9 | #include <string>
10 | 
11 | using namespace std;
12 | 
13 | //Sigmoid Function
14 | double sigmoid(double x);
15 | double sigmoid_derivative(double x);
16 | 
17 | //Relu Function
18 | double relu(double x);
19 | double relu_derivative(double x);
20 | 
21 | //Random float getter function
22 | double random(double low,double high);
23 | 
24 | double distanceVector(const vector<double>& v1, const vector<double>& v2);
25 | 
26 | 
27 | #endif // FUNCTIONS_H
28 | 


--------------------------------------------------------------------------------
/src/dataset/dataset.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include <vector>
 4 | 
 5 | using namespace std;
 6 | 
 7 | enum Datatype {
 8 | 	TRAIN,
 9 | 	TEST
10 | };
11 | 
12 | class Dataset
13 | {
14 | public:
15 | 	Dataset(string filename);
16 | 	~Dataset();
17 | 
18 | 	const vector<const vector<double>*>& getIns(Datatype d) const;
19 | 
20 | 	const vector<const vector<double>*>& getOuts(Datatype d) const;
21 | 
22 | 	void split(double ptrain);
23 | 
24 | private:
25 | 	vector<vector<double>> _ins;
26 | 	vector<vector<double>> _outs;
27 | 
28 | 	vector<const vector<double>*> _train_ins;
29 | 	vector<const vector<double>*> _train_outs;
30 | 
31 | 	vector<const vector<double>*> _test_ins;
32 | 	vector<const vector<double>*> _test_outs;
33 | };
34 | 
35 | 


--------------------------------------------------------------------------------
/src/optimizer/shakingtree.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | // CD
 3 | 
 4 | #include "optimizer.h"
 5 | #include <random>
 6 | 
 7 | class Shakingtree : public Optimizer
 8 | {
 9 | public:
10 | 	Shakingtree();
11 | 	~Shakingtree();
12 | 
13 | 	void minimize();
14 | 
15 | 	void minimizeBasic();
16 | 
17 | 	void minimizeBasicLarger();
18 | 
19 | 	void minimizeComplex();
20 | 
21 | 	void minimizeBasicPerLayer();
22 | 
23 | 	void mapParameters();
24 | 
25 | private:
26 | 	default_random_engine _generator;
27 | 	vector<Edge*> _p;
28 | 	vector<vector<Edge*>> _p2;
29 | 	vector<uint> _p_ids;
30 | 
31 | 	vector<vector<double>> _shift;
32 | 	vector<double> _delta_score;
33 | 	int _itmod = 10; //state how much test you want to accumulate before accepting the delta score
34 | 	double _step = 0.05;
35 | 
36 | 	uint _total_iter = 0;
37 | 	uint _nogoodscore_iter = 0;
38 | 
39 | 
40 | };
41 | 
42 | 


--------------------------------------------------------------------------------
/src/optimizer/backpropagation.h:
--------------------------------------------------------------------------------
 1 | #pragma once
 2 | 
 3 | #include "../misc/functions.h"
 4 | #include "../neural/neuralnetwork.h"
 5 | #include "../dataset/dataset.h"
 6 | #include "optimizer.h"
 7 | #include <unordered_map>
 8 | 
 9 | extern double LEARNING_RATE;
10 | 
11 | 
12 | class Backpropagation : public Optimizer
13 | {
14 | 
15 | public:
16 | 	void setLearningRate(double lr);
17 | 
18 | 	vector<vector<vector<double>>> getBackpropagationShifts(const vector<double>& in, const vector<double>& out);
19 | 
20 | 	void backpropagate(const vector<const vector<double>*>& ins, const vector<const vector<double>*>& outs);
21 | 
22 | 	vector<Layer*> getLayers();
23 | 
24 | 	void minimize();
25 | 
26 | 	void setBatchSize(size_t bs);
27 | 
28 | private:
29 | 	size_t _batch_size = 20;
30 | };
31 | 
32 | 
33 | 
34 | 
35 | 
36 | 
37 | 
38 | 
39 | 
40 | 
41 | 
42 | 
43 | 
44 | 
45 | 
46 | 
47 | 


--------------------------------------------------------------------------------
/src/misc/functions.cpp:
--------------------------------------------------------------------------------
 1 | #include "functions.h"
 2 | 
 3 | //Sigmoid Function
 4 | double sigmoid(double x)
 5 | {
 6 | 	return 1.0 / (1.0 + exp(-x));
 7 | }
 8 | 
 9 | double sigmoid_derivative(double x)
10 | {
11 | 	return sigmoid(x) * (1 - sigmoid(x));
12 | }
13 | 
14 | //Relu Function
15 | double relu(double x)
16 | {
17 | 	if (x > 0)
18 | 		return x;
19 | 	return 0;
20 | }
21 | double relu_derivative(double x)
22 | {
23 | 	if (x > 0)
24 | 		return 1;
25 | 	return 0;
26 | }
27 | 
28 | //Random float getter function
29 | double random(double low, double high)
30 | {
31 |     return low + static_cast <double> (rand()) /( static_cast <double> (RAND_MAX/(high-low)));
32 | }
33 | 
34 | double distanceVector(const vector<double>& v1, const vector<double>& v2)
35 | {
36 | 	double d = 0;
37 | 	for (size_t i = 0; i < v1.size(); i++)
38 | 		d += (v1[i] - v2[i])*(v1[i] - v2[i]);
39 | 	return d;
40 | }
41 | 
42 | 
43 | 
44 | 
45 | 


--------------------------------------------------------------------------------
/src/optimizer/optimizer.cpp:
--------------------------------------------------------------------------------
 1 | #include "optimizer.h"
 2 | #include <thread>
 3 | 
 4 | 
 5 | Optimizer::Optimizer()
 6 | {
 7 | 	LEARNING_RATE = 1;
 8 | }
 9 | 
10 | 
11 | Optimizer::~Optimizer()
12 | {
13 | }
14 | 
15 | void Optimizer::minimize()
16 | {
17 | 	cout << "not implemented" << endl;
18 | }
19 | 
20 | void Optimizer::minimizeThread()
21 | {
22 | 	/*vector<thread> t;
23 | 	for (size_t i = 0; i < 10; i++)
24 | 		t.push_back(move(std::thread(&Optimizer::minimize, this)));
25 | 	for (size_t i = 0; i < t.size(); i++)
26 | 		t[i].join();*/
27 | 
28 | 	std::thread t(&Optimizer::minimize, this);
29 | 	t.join();
30 | }
31 | 
32 | void Optimizer::setDataset(Dataset* dataset)
33 | {
34 | 	_d = dataset;
35 | }
36 | 
37 | void Optimizer::setNeuralNetwork(NeuralNetwork* net)
38 | {
39 | 	_n = net;
40 | }
41 | 
42 | 
43 | double Optimizer::getScore(Datatype d, int limit)
44 | {
45 | 	return _n->predictAllForScore(*_d,d, limit);
46 | }
47 | 
48 | 


--------------------------------------------------------------------------------
/src/neural/edge.h:
--------------------------------------------------------------------------------
 1 | #ifndef EDGE_H
 2 | #define EDGE_H
 3 | 
 4 | 
 5 | #include <iostream>
 6 | 
 7 | #include <vector>
 8 | #include <limits>
 9 | 
10 | class NeuralNetwork;
11 | class Neuron;
12 | class Layer;
13 | using namespace std;
14 | 
15 | typedef unsigned int uint;
16 | 
17 | 
18 | extern double LEARNING_RATE;
19 | 
20 | //Edge between two neurons
21 | class Edge
22 | {
23 | public:
24 |     Edge(Neuron* n, Neuron* start, double w );
25 | 
26 | 	Neuron* neuron() const;
27 | 
28 | 	Neuron* neuronb() const;
29 | 
30 | 	double weight();
31 | 
32 | 	double* weightP();
33 | 
34 |     void propagate(double neuron_output);
35 | 
36 |     void alterWeight(double w);
37 | 
38 | 	void shiftWeight(double dw);
39 | 
40 | 	double getLastShift() const;
41 | 
42 | 	void resetLastShift();
43 | 
44 | 	double backpropagationMemory() const;
45 | 
46 | 	void setBackpropagationMemory(double v);
47 | 
48 | public:
49 | 	Neuron* _n = nullptr;
50 | 	Neuron* _nb = nullptr;
51 |     double _w = 0.0;
52 | 	double _last_shift = 0;
53 | 
54 | 	double _backpropagation_memory;
55 | 
56 | };
57 | 
58 | #endif // EDGE_H
59 | 


--------------------------------------------------------------------------------
/src/neural/edge.cpp:
--------------------------------------------------------------------------------
 1 | #include "edge.h"
 2 | #include "neuron.h"
 3 | 
 4 | Edge::Edge(Neuron *n, Neuron* nb, double w) :  _n(n), _nb(nb), _w(w)
 5 | {
 6 | 
 7 | }
 8 | 
 9 | Neuron *Edge::neuron() const
10 | {
11 |     return _n;
12 | }
13 | 
14 | Neuron* Edge::neuronb() const
15 | {
16 | 	return _nb;
17 | }
18 | 
19 | double Edge::weight() 
20 | {
21 |     return _w;
22 | }
23 | 
24 | double* Edge::weightP()
25 | {
26 | 	return &_w;
27 | }
28 | 
29 | void Edge::propagate(double neuron_output)
30 | {
31 | 	neuron()->addAccumulated(neuron_output * weight());
32 | }
33 | 
34 | void Edge::alterWeight(double w)
35 | {
36 |     _w = w;
37 | 
38 | }
39 | 
40 | void Edge::shiftWeight(double dw)
41 | {
42 | 	dw *= LEARNING_RATE;
43 | 	_w += dw;
44 | 	_last_shift = dw;
45 | }
46 | 
47 | void Edge::resetLastShift()
48 | {
49 | 	_w -= _last_shift;
50 | }
51 | 
52 | double Edge::getLastShift() const
53 | {
54 | 	return _last_shift;
55 | }
56 | 
57 | 
58 | double Edge::backpropagationMemory() const
59 | {
60 | 	return _backpropagation_memory;
61 | }
62 | 
63 | void Edge::setBackpropagationMemory(double v)
64 | {
65 | 	_backpropagation_memory = v;
66 | }
67 | 
68 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | MIT License
 2 | 
 3 | Copyright (c) 2019 Whiax on Github
 4 | 
 5 | Permission is hereby granted, free of charge, to any person obtaining a copy
 6 | of this software and associated documentation files (the "Software"), to deal
 7 | in the Software without restriction, including without limitation the rights
 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 | 
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 | 
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 | 


--------------------------------------------------------------------------------
/src/dataset/dataset.cpp:
--------------------------------------------------------------------------------
 1 | #include "dataset.h"
 2 | 
 3 | #include <fstream>
 4 | #include <algorithm>
 5 | #include "../misc/functions.h"
 6 | 
 7 | Dataset::Dataset(string filename)
 8 | {
 9 | 	ifstream infile(filename);
10 | 	vector<double> line;
11 | 	double a;
12 | 	while (!infile.eof())
13 | 	{
14 | 		infile >> a;
15 | 		line.push_back(a);
16 | 
17 | 		if (infile.get() == '\n')
18 | 		{
19 | 			double out = line[line.size() - 1];
20 | 			line.pop_back();
21 | 
22 | 			_ins.push_back(line);
23 | 			_outs.push_back({ out });
24 | 			line.clear();
25 | 		}
26 | 	}
27 | }
28 | 
29 | Dataset::~Dataset()
30 | {
31 | }
32 | 
33 | void Dataset::split(double ptrain)
34 | {
35 | 	for (size_t i = 0; i < _ins.size(); i++)
36 | 	{
37 | 		if (random(0, 1) < ptrain)
38 | 		{
39 | 			_train_ins.push_back(&_ins[i]);
40 | 			_train_outs.push_back(&_outs[i]);
41 | 		}
42 | 		else
43 | 		{
44 | 			_test_ins.push_back(&_ins[i]);
45 | 			_test_outs.push_back(&_outs[i]);
46 | 		}
47 | 	}
48 | }
49 | 
50 | 
51 | const vector<const vector<double>*>& Dataset::getIns(Datatype d) const
52 | { 
53 | 	if(d == Datatype::TRAIN)
54 | 		return _train_ins;
55 | 	return _test_ins;
56 | }
57 | 
58 | const vector<const vector<double>*>& Dataset::getOuts(Datatype d) const
59 | {
60 | 	if (d == Datatype::TRAIN)
61 | 		return _train_outs;
62 | 	return _test_outs;
63 | }
64 | 
65 | 
66 | 
67 | 
68 | 
69 | 


--------------------------------------------------------------------------------
/src/neural/neuron.h:
--------------------------------------------------------------------------------
 1 | #ifndef NEURON_H
 2 | #define NEURON_H
 3 | 
 4 | #include "edge.h"
 5 | #include "../misc/functions.h"
 6 | #include "layer.h"
 7 | 
 8 | #include <iostream>
 9 | 
10 | #include <vector>
11 | #include <limits>
12 | 
13 | class NeuralNetwork;
14 | class Neuron;
15 | class Layer;
16 | using namespace std;
17 | 
18 | typedef unsigned int uint;
19 | 
20 | enum ActivationFunction
21 | {
22 | 	LINEAR,
23 | 	SIGMOID,
24 | 	RELU
25 | };
26 | 
27 | class Neuron
28 | {
29 | 
30 | public:
31 |     Neuron(int id_neuron, Layer* layer, ActivationFunction function = LINEAR, bool is_bias = false);
32 | 
33 |     ~Neuron();
34 | 
35 |     void trigger();
36 | 
37 |     double in();
38 | 
39 |     double output();
40 | 
41 | 	double outputDerivative();
42 | 
43 | 	double outputRaw();
44 | 
45 |     void clean();
46 | 
47 |     void addAccumulated(double v);
48 | 
49 | 	void addNext(Neuron* n);
50 | 
51 | 	void addPrevious(Edge* e);
52 | 
53 |     int getNeuronId() const;
54 | 
55 |     void setAccumulated(double v);
56 | 
57 |     void alterWeights(const vector<double>& weights);
58 | 
59 | 	vector<double*> getWeights();
60 | 
61 | 	vector<Edge*> getEdges();
62 | 
63 | 	void randomizeAllWeights(double abs_value);
64 | 
65 |     string toString();
66 | 
67 | 	void shiftWeights(float range);
68 | 
69 | 	void shiftBackWeights(const vector<double>& range);
70 | 
71 | 	vector<double> getBackpropagationShifts(const vector<double>& target);
72 | 
73 | 	bool isBias() const;
74 | 
75 | public:
76 |     Layer* _layer = NULL;
77 |     int _id_neuron = 0;
78 |     double _accumulated = 0.0;
79 | 
80 | 	double _threshold = 0.0;
81 | 	vector<Edge*> _next;
82 | 	vector<Edge*> _previous;
83 | 	ActivationFunction _activation_function;
84 | 	bool _is_bias = false;
85 | 
86 | 
87 | };
88 | 
89 | #endif // NEURON_H
90 | 


--------------------------------------------------------------------------------
/src/neural/neuralnetwork.h:
--------------------------------------------------------------------------------
 1 | #ifndef NEURALNETWORK_H
 2 | #define NEURALNETWORK_H
 3 | 
 4 | #include "../misc/functions.h"
 5 | #include "layer.h"
 6 | #include "../dataset/dataset.h"
 7 | #include <unordered_map>
 8 | 
 9 | #define RAND_MAX_WEIGHT 1
10 | 
11 | #include <iostream>
12 | 
13 | #include <vector>
14 | #include <limits>
15 | 
16 | class NeuralNetwork;
17 | class Neuron;
18 | class Layer;
19 | using namespace std;
20 | 
21 | typedef unsigned int uint;
22 | 
23 | 
24 | class NeuralNetwork
25 | {
26 | public:
27 | 	NeuralNetwork();
28 | 
29 |     ~NeuralNetwork();
30 | 
31 | 	void autogenerate(bool randomize = true);
32 | 
33 | 	void addLayer(unordered_map<string, double> parameters);
34 | 
35 |     void clean();
36 | 
37 | 	void setInput(vector<double> in);
38 | 
39 |     void trigger();
40 | 
41 |     vector<double> output();
42 | 
43 | 	string outputString();
44 | 
45 |     void connectComplete();
46 | 
47 | 	void alterWeights(const vector<vector<vector<double>>>& weights);
48 | 
49 | 	void shiftBackWeights(const vector<vector<vector<double>>>& weights);
50 | 
51 | 	vector<vector<vector<double*>>> getWeights();
52 | 
53 | 	vector<vector<vector<Edge*>>> getEdges();
54 | 
55 |     void randomizeAllWeights();
56 | 
57 |     double loss(const vector<double>& in, const vector<double>& out);
58 | 
59 | 	double loss(const vector<vector<double>*>& ins, const vector<vector<double>*>& outs);
60 | 
61 |     string toString();
62 | 
63 | 	void shiftWeights(float percentage_of_range);
64 | 
65 | 	vector<double> predict(const vector<double>& in);
66 | 
67 | 	double predictAllForScore(const Dataset& dataset, Datatype d = TEST, int limit=-1);
68 | 
69 | 	double predictPartialForScore(const Dataset& dataset);
70 | 
71 | 	vector<Layer*> getLayers();
72 | 
73 | 
74 | 
75 | 
76 | 
77 | public:
78 |     vector<Layer*> _layers;
79 | 	double _fitness;
80 | 
81 | 	vector<unordered_map<string,double>> _configuration;
82 | };
83 | 
84 | #endif // NEURALNETWORK_H
85 | 


--------------------------------------------------------------------------------
/src/neural/layer.h:
--------------------------------------------------------------------------------
 1 | #ifndef LAYER_H
 2 | #define LAYER_H
 3 | 
 4 | #include "neuron.h"
 5 | #include <unordered_map>
 6 | 
 7 | #include <iostream>
 8 | 
 9 | #include <vector>
10 | #include <limits>
11 | 
12 | class NeuralNetwork;
13 | class Neuron;
14 | class Layer;
15 | using namespace std;
16 | 
17 | typedef unsigned int uint;
18 | 
19 | enum LayerType
20 | {
21 | 	STANDARD = 0, //Standard layer : fully connected perceptrons
22 | 	OUTPUT, // Output : No bias neuron
23 | 	INPUT, // Input: Standard input (output of neurons is outputRaw() )
24 | 	SOFTMAX //K-Class Classification Layer 
25 | 
26 | };
27 | 
28 | enum ActivationFunction;
29 | 
30 | //Layer of the network
31 | class Layer
32 | {
33 | public:
34 |     Layer(int id_layer, NeuralNetwork* net, unordered_map<string,double> parameters);
35 | 
36 |     ~Layer();
37 | 
38 | 	int getId() const;
39 | 
40 | 	void initLayer();
41 | 
42 |     void clean();
43 | 
44 |     void trigger();
45 | 
46 |     void connectComplete(Layer* next);
47 | 
48 |     vector<double> output();
49 | 
50 |     const vector<Neuron *>& neurons() const;
51 | 
52 | 	void alterWeights(const vector<vector<double> >& weights);
53 | 
54 | 	void shiftBackWeights(const vector<vector<double> >& weights);
55 | 
56 | 	vector<vector<double*> > getWeights();
57 | 
58 | 	vector<vector<Edge*> > getEdges();
59 | 
60 | 	void randomizeAllWeights(double abs_value);
61 | 
62 |     string toString();
63 | 
64 | 	void shiftWeights(float range);
65 | 
66 | 	const unordered_map<string, double>& getParameters() const;
67 | 
68 | 	vector<vector<double>> getBackpropagationShifts(const vector<double>& target);
69 | 
70 | 	LayerType getType() const;
71 | 
72 | 	ActivationFunction getActivation() const;
73 | 
74 | 	NeuralNetwork* getNet() const { return _net; }
75 | 
76 | public:
77 | 	NeuralNetwork* _net;
78 |     int _id_layer;
79 |     vector<Neuron*> _neurons;
80 | 	LayerType _type;
81 | 	ActivationFunction _activation;
82 | 	unordered_map<string, double> _parameters;
83 | };
84 | 
85 | #endif // LAYER_H
86 | 


--------------------------------------------------------------------------------
/src/optimizer/backpropagation.cpp:
--------------------------------------------------------------------------------
 1 | #include "Backpropagation.h"
 2 | 
 3 | 
 4 | void Backpropagation::setLearningRate(double lr)
 5 | {
 6 | 	LEARNING_RATE = lr;
 7 | }
 8 | 
 9 | void Backpropagation::setBatchSize(size_t bs)
10 | {
11 | 	_batch_size = bs;
12 | }
13 | 
14 | void Backpropagation::minimize()
15 | {
16 | 	vector<const vector<double>*> batch_in(_batch_size);
17 | 	vector<const vector<double>*> batch_out(_batch_size);
18 | 
19 | 	for (size_t i = 0; i < _batch_size; i++)
20 | 	{
21 | 		int z = rand() % _d->getIns(TRAIN).size();
22 | 		batch_in[i] = _d->getIns(TRAIN)[z];
23 | 		batch_out[i] = _d->getOuts(TRAIN)[z];
24 | 	}
25 | 	backpropagate(batch_in, batch_out);
26 | }
27 | 
28 | 
29 | vector<vector<vector<double>>> Backpropagation::getBackpropagationShifts(const vector<double>& in, const vector<double>& out)
30 | {
31 | 	vector<vector<vector<double>>> dw(_n->getLayers().size());
32 | 	auto out_exp = _n->predict(in);
33 | 	for (int i = _n->getLayers().size() - 1; i >= 1; --i)
34 | 	{
35 | 		auto _dw = move(_n->getLayers()[i]->getBackpropagationShifts(out));
36 | 		dw[_n->getLayers()[i]->getId()] = _dw;
37 | 	}
38 | 	return move(dw);
39 | 
40 | }
41 | 
42 | void  Backpropagation::backpropagate(const vector<const vector<double>*>& ins, const vector<const vector<double>*>& outs)
43 | {
44 | 	vector<vector<vector<double>>> dw(_n->getLayers().size());
45 | 	bool is_init = false;
46 | 	for (size_t i = 0; i < ins.size(); i++)
47 | 	{
48 | 		auto in = ins[i];
49 | 		auto out = outs[i];
50 | 		auto _dw = getBackpropagationShifts(*in, *out);
51 | 		if (!is_init)
52 | 		{
53 | 			for (size_t j = 0; j < _dw.size(); j++)
54 | 			{
55 | 				dw[j].resize(_dw[j].size());
56 | 				for (size_t k = 0; k < _dw[j].size(); k++)
57 | 					dw[j][k].resize(_dw[j][k].size(), 0);
58 | 			}
59 | 		}
60 | 		for (size_t j = 0; j < _dw.size(); j++)
61 | 			for (size_t k = 0; k < _dw[j].size(); k++)
62 | 				for (size_t l = 0; l < _dw[j][k].size(); l++)
63 | 					dw[j][k][l] += _dw[j][k][l]; //edge l, neuron k, layer j
64 | 	}
65 | 	for (size_t j = 0; j < dw.size(); j++)
66 | 		for (size_t k = 0; k < dw[j].size(); k++)
67 | 			for (size_t l = 0; l < dw[j][k].size(); l++)
68 | 				dw[j][k][l] /= ins.size();
69 | 
70 | 	_n->shiftBackWeights(dw);
71 | }
72 | 
73 | 


--------------------------------------------------------------------------------
/src/neuralmain.cpp:
--------------------------------------------------------------------------------
  1 | 
  2 | #include "neural/neuralnetwork.h"
  3 | #include "misc/functions.h"
  4 | #include "optimizer/backpropagation.h"
  5 | #include "optimizer/shakingtree.h"
  6 | #include "dataset/dataset.h"
  7 | 
  8 | #include <ctime>
  9 | #include <numeric>
 10 | #include <fstream>
 11 | 
 12 | 
 13 | #include <iostream>
 14 | 
 15 | #include <vector>
 16 | #include <limits>
 17 | #include <thread>
 18 | 
 19 | class NeuralNetwork;
 20 | class Neuron;
 21 | class Layer;
 22 | using namespace std;
 23 | 
 24 | typedef unsigned int uint;
 25 | 
 26 | 
 27 | 
 28 | double LEARNING_RATE = 0.5000;
 29 | 
 30 | 
 31 | 
 32 | 
 33 | //Main function
 34 | int main(int argc, char *argv[])
 35 | {
 36 | 	//Init random and logs if required
 37 | 	srand(uint(time(0)));
 38 | 	//ofstream logs("logs.txt");
 39 | 
 40 | 
 41 | 
 42 | 	//Neural network definition
 43 | 	NeuralNetwork n;
 44 | 	size_t n_hidden_layer = 5;
 45 | 	n.addLayer({ { "type", LayerType::INPUT },{ "size",2 }});
 46 | 	for (size_t i = 0; i < n_hidden_layer; i++)
 47 | 		n.addLayer({ { "type", LayerType::STANDARD },{ "size",10} ,{ "activation",ActivationFunction::SIGMOID } });
 48 | 	n.addLayer({ { "type", LayerType::OUTPUT},{ "size",1} ,{ "activation",ActivationFunction::SIGMOID } });
 49 | 	n.autogenerate();
 50 | 
 51 | 
 52 | 	//Create the dataset
 53 | 	Dataset data("data1000.txt");
 54 | 	data.split(0.8);
 55 | 
 56 | 
 57 | 	//Create the optimizer
 58 | 	/*Shakingtree opt;
 59 | 	opt.setNeuralNetwork(&n);
 60 | 	opt.setDataset(&data);
 61 | 	opt.mapParameters();*/
 62 | 
 63 | 	Backpropagation opt;
 64 | 	opt.setBatchSize(60);
 65 |         LEARNING_RATE = 0.5;
 66 | 	opt.setNeuralNetwork(&n);
 67 | 	opt.setDataset(&data);
 68 | 
 69 | 
 70 | 
 71 | 	//Init the main training loop (nb: the goal is to lower the score, score = loss)
 72 | 	double lr_reduce_amplitude = 0.9;
 73 | 	int lr_reduce_schedule = 500;
 74 | 	int n_iteration = 50000;
 75 | 	int validate_every = 10;
 76 | 	double mintest = 1;
 77 | 	int i = 0;
 78 | 	clock_t t = clock();
 79 | 	while (i < n_iteration)
 80 | 	{
 81 | 		//For Backpropagation: The optimizer reads a batch, pass it in the neural network, computes and apply gradients
 82 | 		//For ShakingTree: The behaviour depends but the overall idea is to try random parameters and keep the good ones
 83 | 		opt.minimize();
 84 | 
 85 | 		//Evaluate the score on test data
 86 | 		if (i % validate_every == 0)
 87 | 		{
 88 | 			double strain = n.predictAllForScore(data, TRAIN);
 89 | 			double stest = n.predictAllForScore(data);
 90 | 			mintest = stest < mintest ? stest : mintest;
 91 | 			cout << "it:" << i << '\t' << "    test_score:" << stest << "    train_score:" << strain << "   (best_test_score : " << mintest << ")" << endl;
 92 | 
 93 | 			double delta_t = (clock() - t) / 1000.0;
 94 | 			//logs << i << ";" << delta_t << ";" << stest << ";" << strain << ";" << mintest << endl;
 95 | 		}
 96 | 
 97 | 		//Reduce learning rate
 98 | 		if (i % lr_reduce_schedule == 0)
 99 | 		{
100 | 			LEARNING_RATE = LEARNING_RATE * lr_reduce_amplitude;
101 | 			cout << LEARNING_RATE << endl;
102 | 		}
103 | 		i++;
104 | 	}
105 | 
106 | 	getchar();
107 |     return 0;
108 | }
109 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # NeuralNetworkCpp
 2 | Basic deep learning framework in C++ / new ShakingTree optimization algorithm
 3 | 
 4 | All information can be found here:
 5 | https://medium.com/@hyugen.ai/neural-network-in-c-from-scratch-and-backprop-free-optimizer-d2a34bb92688
 6 | 
 7 | /!\ The implementation is very basic and mostly made to learn, play and try things with deep neural networks. There's no convolutional networks, no transformers, no Adam etc..
 8 | For a professional deep learning library in C++ I'll recommend http://dlib.net/ or the C++ API of Pytorch and Tensorflow.
 9 | 
10 | PS: I worked on this project with Visual Studio / Windows and it seems that some indentations were incorrectly encoded, sorry about that
11 | 
12 | ## Framework Design
13 | ### 1. Folders (latest version)
14 | - Neural: Contains classes related to the neural network
15 | - Dataset: Contains one simple dataset class
16 | - Optimizer: Contains a generic optimizer class, a classical backpropagation optimizer and an exotic optimizer
17 | - Misc: Contains activation functions and their derivatives
18 | 
19 | ### 2. Neural Classes
20 | - NeuralNetwork.h/.cpp: the main class containing a model. A NeuralNetwork is composed of multiple layers
21 | - Layer.h/.cpp: A Layer is composed of multiple neurons. It has a type (INPUT/STANDARD/OUTPUT) because it doesn’t work the same way depending on its type and it has an activation function
22 | - Neuron.h/.cpp: A Neuron has a parent layer. A neuron accumulates the input of the neurons connected to it (_accumulated), it outputs that input to its edges after processing it with its _activation_function (which you can change on a per-neuron basis). A neuron knows its _next and _previous edges.
23 | - Edge.h/.cpp: An Edge knows its next neuron (_n), the neuron its coming from (_nb), it has a weight (_w), it can memorize how much its weight was shift (_last_shift) last time it was, and it has a backpropagation memory so that it can retain a part of the chain rule to avoid useless computations
24 | 
25 | ### 3. Dataset / Optimizer
26 | - Dataset.h/.cpp: It gives the data. One input/output is just a vector (again, no matrices, no images). So the list of all inputs is a vector of vector. The Dataset class is also responsible for doing the train/test split.
27 | - Optimizer.h/.cpp: Abstract class that links the network and the dataset together. Child classes must implement a “minimize” method that is called during the optimization process to minimize the loss.
28 | - Backpropagation.h/.cpp: Implementation of the backpropagation optimizer (though the backpropagation computations are in the neuron class for simplification). This class is mostly there to call these methods in the neuron class and to apply the new computed weights. It uses a batch size, the learning rate is a global variable because it’s common to all implemented optimizers and it’s easier to access it from anywhere.
29 | - ShakingTree.h/.cpp: Exotic optimizer that’ll be detailed later.
30 | 
31 | ## Computation example
32 | The algorithm was also made to exactly reproduce the article from Matt Mazur on his website: https://mattmazur.com/2015/03/17/a-step-by-step-backpropagation-example/  
33 | Just use the “mattmazur_step_by_step” branch on git, this branch serves only that purpose and contains older code than the code on “master”.
34 | 
35 | 


--------------------------------------------------------------------------------
/src/neural/layer.cpp:
--------------------------------------------------------------------------------
  1 | #include "layer.h"
  2 | 
  3 | Layer::Layer(int id_layer, NeuralNetwork* net, unordered_map<string, double> parameters)
  4 | {
  5 |     _id_layer = id_layer;
  6 | 	_net = net;
  7 | 	_parameters = parameters;
  8 | 	_type = static_cast<LayerType>(static_cast<int>(parameters["type"])); //erk
  9 | 	_activation = static_cast<ActivationFunction>(static_cast<int>(_parameters["activation"]));
 10 | 	initLayer();
 11 | }
 12 | 
 13 | Layer::~Layer()
 14 | {
 15 |     for(Neuron* n : _neurons)
 16 |         delete n;
 17 |     _neurons.clear();
 18 | }
 19 | 
 20 | int Layer::getId() const
 21 | {
 22 | 	return _id_layer;
 23 | }
 24 | 
 25 | void Layer::initLayer()
 26 | {
 27 | 	_neurons.clear();
 28 | 
 29 | 	if (_type == LayerType::STANDARD || _type == LayerType::INPUT)
 30 | 	{
 31 | 		_parameters["size"] += 1; //bias for the next layer
 32 | 		_neurons.reserve(static_cast<int>(_parameters["size"]));
 33 | 		for (int i_neuron = 0; i_neuron < _parameters["size"]; ++i_neuron)
 34 | 			_neurons.push_back(new Neuron(i_neuron, this, _activation, i_neuron == _neurons.capacity()-1));
 35 | 	}
 36 | 	else if (_type == LayerType::OUTPUT)
 37 | 	{
 38 | 		_neurons.reserve(static_cast<int>(_parameters["size"]));
 39 | 		for (int i_neuron = 0; i_neuron < _parameters["size"]; ++i_neuron)
 40 | 			_neurons.push_back(new Neuron(i_neuron, this, _activation));
 41 | 	}
 42 | }
 43 | 
 44 | void Layer::clean()
 45 | {
 46 |     for(Neuron* n : _neurons)
 47 |         n->clean();
 48 | }
 49 | 
 50 | void Layer::trigger()
 51 | {
 52 |     for(Neuron* n : _neurons)
 53 |         n->trigger();
 54 | }
 55 | 
 56 | 
 57 | void Layer::connectComplete(Layer *next)
 58 | {
 59 |     for(Neuron* n1 : _neurons)
 60 |         for(Neuron* n2 : next->_neurons)
 61 | 			if(!n2->isBias())
 62 | 				n1->addNext(n2);
 63 | }
 64 | 
 65 | vector<double> Layer::output()
 66 | {
 67 |     vector<double> outputs;
 68 |     outputs.reserve(_neurons.size());
 69 |     for(Neuron* n : _neurons)
 70 |         outputs.push_back(n->output());
 71 |     return std::move(outputs);
 72 | 
 73 | }
 74 | 
 75 | const vector<Neuron *>& Layer::neurons() const
 76 | {
 77 |     return _neurons;
 78 | }
 79 | 
 80 | void Layer::alterWeights(const vector<vector<double> >& weights)
 81 | {
 82 |     for(size_t i_neuron=0;i_neuron < weights.size(); ++i_neuron)
 83 |         _neurons[i_neuron]->alterWeights(weights[i_neuron]);
 84 | }
 85 | 
 86 | void Layer::shiftBackWeights(const vector<vector<double> >& weights)
 87 | {
 88 | 	for (size_t i_neuron = 0; i_neuron < _neurons.size(); ++i_neuron)
 89 | 		_neurons[i_neuron]->shiftBackWeights(weights[i_neuron]);
 90 | }
 91 | 
 92 | vector<vector<double*> > Layer::getWeights()
 93 | {
 94 | 	vector<vector<double*>> w;
 95 | 	w.reserve(_neurons.size());
 96 | 	for (size_t i_neuron = 0; i_neuron < _neurons.size(); ++i_neuron)
 97 | 		w.push_back(std::move(_neurons[i_neuron]->getWeights()));
 98 | 	return std::move(w);
 99 | }
100 | 
101 | vector<vector<Edge*> > Layer::getEdges()
102 | {
103 | 	vector<vector<Edge*>> w;
104 | 	w.reserve(_neurons.size());
105 | 	for (size_t i_neuron = 0; i_neuron < _neurons.size(); ++i_neuron)
106 | 		w.push_back(std::move(_neurons[i_neuron]->getEdges()));
107 | 	return std::move(w);
108 | }
109 | 
110 | void Layer::randomizeAllWeights(double abs_value)
111 | {
112 |     for(Neuron* neuron : _neurons)
113 |         neuron->randomizeAllWeights(abs_value);
114 | }
115 | 
116 | string Layer::toString()
117 | {
118 |     string str = "layer:" + to_string(_id_layer) + "\n";
119 |     for(Neuron* neuron : _neurons)
120 |         str += neuron->toString() + "\n";
121 |     return str;
122 | }
123 | 
124 | void Layer::shiftWeights(float range)
125 | {
126 | 	for(Neuron* neuron : _neurons)
127 | 		neuron->shiftWeights(range);
128 | }
129 | 
130 | const unordered_map<string, double>& Layer::getParameters() const
131 | {
132 | 	return _parameters;
133 | }
134 | 
135 | vector<vector<double>> Layer::getBackpropagationShifts(const vector<double>& target)
136 | {
137 | 	vector<vector<double>> dw(_neurons.size());
138 | 	for (size_t i = 0; i < _neurons.size(); i++)
139 | 	{
140 | 		Neuron* n = _neurons[i];
141 | 		dw[i] = n->getBackpropagationShifts(target);
142 | 	}
143 | 	return dw;
144 | }
145 | 
146 | LayerType Layer::getType() const
147 | {
148 | 	return _type;
149 | }
150 | 
151 | ActivationFunction Layer::getActivation() const
152 | {
153 | 	return _activation;
154 | }
155 | 


--------------------------------------------------------------------------------
/src/optimizer/shakingtree.cpp:
--------------------------------------------------------------------------------
  1 | #include "shakingtree.h"
  2 | #include <algorithm>
  3 | #include <random>
  4 | #include <chrono>
  5 | 
  6 | 
  7 | 
  8 | Shakingtree::Shakingtree()
  9 | {
 10 | 
 11 | }
 12 | 
 13 | 
 14 | Shakingtree::~Shakingtree()
 15 | {
 16 | }
 17 | 
 18 | 
 19 | 
 20 | void Shakingtree::minimize()
 21 | {
 22 | 	minimizeComplex();
 23 | }
 24 | 
 25 | 
 26 | void Shakingtree::minimizeBasic()
 27 | {
 28 | 	mapParameters();
 29 | 
 30 | 	//get a score
 31 | 	int batch_size = 20;
 32 | 	int weight_amplitude = 5;
 33 | 	double s = getScore(TRAIN, batch_size);
 34 | 
 35 | 	//choose a parameter to change
 36 | 	int i = rand() % _p.size();
 37 | 	double oldp = _p[i]->weight();
 38 | 	_p[i]->alterWeight(random(-weight_amplitude, weight_amplitude));
 39 | 
 40 | 	//evaluate the new score
 41 | 	double news = getScore(TRAIN, batch_size);
 42 | 
 43 | 	//if the new score (loss) is bigger, we keep the old weight
 44 | 	if (news > s)
 45 | 		_p[i]->alterWeight(oldp);
 46 | 	return;
 47 | }
 48 | 
 49 | void Shakingtree::minimizeBasicLarger()
 50 | {
 51 | 	mapParameters();
 52 | 
 53 | 	//get a score
 54 | 	int batch_size = 100;
 55 | 	int weight_amplitude = 5;
 56 | 	size_t n_new_parameters = 5;// int(0.1 * _p_ids.size());
 57 | 	unsigned seed = (unsigned int)(std::chrono::system_clock::now().time_since_epoch().count());
 58 | 	std::shuffle(_p_ids.begin(), _p_ids.end(), std::default_random_engine(seed));
 59 | 	double s = getScore(TRAIN, batch_size);
 60 | 
 61 | 	//choose multiple parameters to change
 62 | 	vector<double> old_p;
 63 | 	for (size_t j = 0; j < n_new_parameters; j++)
 64 | 	{
 65 | 		old_p.push_back(_p[_p_ids[j]]->weight());
 66 | 		_p[_p_ids[j]]->alterWeight(random(-weight_amplitude, weight_amplitude));
 67 | 	}
 68 | 
 69 | 	//evaluate the new score
 70 | 	double new_s = getScore(TRAIN, batch_size);
 71 | 
 72 | 	//if the new score (loss) is bigger, we keep the old weight
 73 | 	if (new_s > s)
 74 | 		for (size_t j = 0; j < n_new_parameters; j++)
 75 | 			_p[_p_ids[j]]->alterWeight(old_p[j]);
 76 | 	return;
 77 | }
 78 | 
 79 | 
 80 | void Shakingtree::minimizeComplex()
 81 | {
 82 | 	mapParameters();
 83 | 	
 84 | 	//PHASE 1 We compute the previous score
 85 | 	size_t EVALSIZE = 100;
 86 | 	srand(_total_iter);
 87 | 	double score = getScore(TRAIN, EVALSIZE);
 88 | 
 89 | 
 90 | 	//We apply the shift to the weights
 91 | 	std::normal_distribution<double> rnorm(0, _step);
 92 | 	vector<double> neww;
 93 | 	neww.resize(_p.size());
 94 | 	for (size_t i = 0; i < _p.size(); i++)
 95 | 	{
 96 | 		neww[i] = rnorm(_generator);
 97 | 
 98 | 		//apply the delta
 99 | 		_p[i]->shiftWeight(neww[i]);
100 | 	}
101 | 
102 | 	//PHASE 2 : We compute the delta
103 | 	//evaluate score
104 | 	srand(_total_iter);
105 | 	double delta_score = getScore(TRAIN, EVALSIZE) - score;
106 | 
107 | 	_delta_score.push_back(delta_score);
108 | 	_shift.push_back(neww);
109 | 
110 | 
111 | 	//PHASE 2B : We remove the shift
112 | 	for (size_t i = 0; i < _p.size(); i++)
113 | 	{
114 | 		//remove the delta
115 | 		_p[i]->resetLastShift();
116 | 	}
117 | 
118 | 
119 | 	//PHASE 3 : With a large enough memory, we hope to be able to analyze the delta score and the shift such that we know what a good shift is
120 | 	if (_shift.size() == _itmod)
121 | 	{
122 | 
123 | 		//apply the averaged shift
124 | 		uint gscore = 0;
125 | 		for (size_t j = 0; j < _shift.size(); j++)
126 | 			if (_delta_score[j] < 0) // || _total_iter % 10 == 0)
127 | 			{
128 | 				for (size_t i = 0; i < _p.size(); i++)
129 | 				{
130 | 					double wsum = 0;
131 | 					wsum += _shift[j][i];
132 | 					_p[i]->shiftWeight(wsum*LEARNING_RATE);
133 | 				}
134 | 				gscore++;
135 | 			}
136 | 		
137 | 		//are we getting better? if no, other strategies may be applied
138 | 		if (gscore == 0)
139 | 			_nogoodscore_iter++;
140 | 		else
141 | 			_nogoodscore_iter = 0;
142 | 
143 | 		//clear shift, delta score and increment iter
144 | 		_shift.clear();
145 | 		_delta_score.clear();
146 | 		_total_iter++;
147 | 	}
148 | 
149 | 
150 | 	return;
151 | }
152 | 
153 | 
154 | void Shakingtree::minimizeBasicPerLayer()
155 | {
156 | 	mapParameters();
157 | 
158 | 	
159 | 	vector<Edge*>& layer = _p2[rand()%_p2.size()];
160 | 
161 | 	for (size_t i = 0; i < 1000; i++)
162 | 	{
163 | 		double s = getScore(TRAIN, 100);
164 | 		double neww = random(-7, 7);
165 | 		int i_edge = rand() % layer.size();
166 | 		double oldw = layer[i_edge]->weight();
167 | 		layer[i_edge]->alterWeight(neww);
168 | 		double news = getScore(TRAIN, 100);
169 | 		if (news > s)
170 | 			layer[i_edge]->shiftWeight(oldw);
171 | 	}
172 | 
173 | 	return;
174 | }
175 | 
176 | 
177 | // easier access to parameters for the optimizer
178 | void Shakingtree::mapParameters()
179 | {
180 | 	if (_p.size() == 0)
181 | 	{
182 | 		auto& w = _n->getEdges();
183 | 		for (size_t i = 0; i < w.size(); i++)
184 | 			for (size_t j = 0; j < w[i].size(); j++)
185 | 			{
186 | 				for (size_t k = 0; k < w[i][j].size(); k++)
187 | 					_p.push_back(w[i][j][k]);
188 | 				_p2.push_back(move(w[i][j]));
189 | 			}
190 | 		cout << _p.size() << " mapped parameters" << endl;
191 | 	}
192 | 	for (size_t i = 0; i < _p.size(); i++)  _p_ids.push_back(i);
193 | }
194 | 
195 | 
196 | 
197 | 
198 | 
199 | 
200 | 
201 | 


--------------------------------------------------------------------------------
/src/neural/neuron.cpp:
--------------------------------------------------------------------------------
  1 | #include "neuron.h"
  2 | #include <algorithm>
  3 | #include "neuralnetwork.h"
  4 | 
  5 | Neuron::Neuron(int id_neuron, Layer* layer, ActivationFunction function, bool is_bias):
  6 |     _id_neuron(id_neuron),
  7 |     _layer(layer),
  8 | 	_activation_function(function),
  9 | 	_is_bias(is_bias)
 10 | {	
 11 | }
 12 | 
 13 | Neuron::~Neuron()
 14 | {
 15 |     for(Edge* e : _next)
 16 |         delete e;
 17 | }
 18 | 
 19 | void Neuron::trigger()
 20 | {
 21 | 	for (Edge* e : _next)
 22 | 	{
 23 | 		//cout << this->getNeuronId() << "->" << e->neuron()->getNeuronId() << " " << output() << "*" << e->weight() << "=" << output()*e->weight() << "("<< outputRaw() << ")" << endl;
 24 | 		e->propagate(output());
 25 | 	}
 26 | 
 27 | }
 28 | 
 29 | double Neuron::in()
 30 | {
 31 |     return _accumulated;
 32 | }
 33 | 
 34 | double Neuron::output()
 35 | {
 36 | 	if (_is_bias)
 37 | 		return 1;
 38 |     if(_layer->getType() == LayerType::INPUT)
 39 |         return outputRaw();
 40 | 
 41 | 	//return random(-10, 10);
 42 | 	if (_activation_function == ActivationFunction::LINEAR)
 43 | 		return _accumulated;
 44 | 	if(_activation_function == ActivationFunction::RELU)
 45 | 		return relu(_accumulated);
 46 | 	if (_activation_function == ActivationFunction::SIGMOID)
 47 | 		return sigmoid(_accumulated);
 48 | 	return outputRaw();
 49 | }
 50 | 
 51 | double Neuron::outputDerivative()
 52 | {
 53 | 	if (_activation_function == ActivationFunction::LINEAR)
 54 | 		return 1;
 55 | 	if (_activation_function == ActivationFunction::RELU)
 56 | 		return relu_derivative(output());
 57 | 	if (_activation_function == ActivationFunction::SIGMOID)
 58 | 		return sigmoid_derivative(outputRaw());
 59 | 	return _accumulated;
 60 | }
 61 | 
 62 | double Neuron::outputRaw()
 63 | {
 64 |     return _accumulated;
 65 | }
 66 | 
 67 | void Neuron::clean()
 68 | {
 69 |     setAccumulated(0);
 70 | }
 71 | 
 72 | void Neuron::addAccumulated(double v)
 73 | {
 74 | 	//cout << this->_layer->getId() << ":" << this->getNeuronId() << " added " << v << " on " << _accumulated << endl;
 75 | 
 76 |     setAccumulated(_accumulated + v);
 77 | }
 78 | 
 79 | void Neuron::addNext(Neuron *n)
 80 | {
 81 |     _next.push_back(new Edge(n, this, random(-5, 5)));
 82 | 	n->addPrevious(_next.back());
 83 | }
 84 | 
 85 | void Neuron::addPrevious(Edge* e)
 86 | {
 87 | 	_previous.push_back(e);
 88 | }
 89 | 
 90 | int Neuron::getNeuronId() const
 91 | {
 92 |     return _id_neuron;
 93 | }
 94 | 
 95 | void Neuron::setAccumulated(double v)
 96 | {
 97 | 
 98 |     _accumulated = v;
 99 | }
100 | 
101 | void Neuron::alterWeights(const vector<double>& weights)
102 | {
103 |     for(size_t i_edge=0; i_edge < weights.size(); ++i_edge)
104 |         _next[i_edge]->alterWeight(weights[i_edge]);
105 | }
106 | 
107 | vector<double*> Neuron::getWeights()
108 | {
109 | 	vector<double*> w;
110 | 	w.reserve(_next.size());
111 | 	for (size_t i_edge = 0; i_edge < _next.size(); ++i_edge)
112 | 		w.push_back(_next[i_edge]->weightP());
113 | 	return std::move(w);
114 | }
115 | 
116 | vector<Edge*> Neuron::getEdges()
117 | {
118 | 	vector<Edge*> w;
119 | 	w.reserve(_next.size());
120 | 	for (size_t i_edge = 0; i_edge < _next.size(); ++i_edge)
121 | 		w.push_back(_next[i_edge]);
122 | 	return std::move(w);
123 | }
124 | 
125 | void Neuron::randomizeAllWeights(double abs_value)
126 | {
127 |     for(Edge* e : _next)
128 | 		e->alterWeight(random(-abs_value, abs_value));
129 | }
130 | 
131 | 
132 | string Neuron::toString()
133 | {
134 |     string weights;
135 |     for(Edge* e : _next)
136 |         weights.append( to_string(e->weight()) + ",");
137 |     string str =  "[" +  to_string(_layer->getId()) + "," + to_string(_id_neuron) + "]" +"("+ weights +")";
138 |     return str;
139 | }
140 | 
141 | void Neuron::shiftWeights(float range)
142 | {
143 | 	for (Edge* e : _next)
144 | 		e->alterWeight(e->weight() + random(-range, range));
145 | }
146 | 
147 | void Neuron::shiftBackWeights(const vector<double>& w)
148 | {
149 | 	for (size_t i = 0; i < _previous.size(); i++)
150 | 		_previous[i]->shiftWeight(w[i]);
151 | }
152 | 
153 | //gradient descent
154 | vector<double> Neuron::getBackpropagationShifts(const vector<double>& target)
155 | {
156 | 	vector<double> dw(_previous.size(),0);
157 | 	if (_layer->getType() == LayerType::OUTPUT)
158 | 	{
159 | 		double d0 = output();
160 | 		double d1 = output() - target[this->getNeuronId()];
161 | 		double d2 = outputDerivative();
162 | 		for (size_t i = 0; i < _previous.size(); ++i)
163 | 		{
164 | 			dw[i] = (-d1*d2*_previous[i]->neuronb()->output());
165 | 			_previous[i]->setBackpropagationMemory(d1*d2);
166 | 		}
167 | 		//cout << _layer->getId() << " " << d1 << " " << d2 << " " << d3 << " " << d1*d2*d3 << endl;
168 | 
169 | 	}
170 | 	else 
171 | 	{
172 | 		double d = 0;
173 | 		for (size_t i = 0; i < _next.size(); i++)
174 | 			d += _next[i]->backpropagationMemory() * _next[i]->weight();
175 | 		d *= outputDerivative();
176 | 		for (size_t i = 0; i < _previous.size(); i++)
177 | 		{
178 | 			_previous[i]->setBackpropagationMemory(d);
179 | 			dw[i] = -d * _previous[i]->neuronb()->output();
180 | 		}
181 | 	}
182 | 	return dw;
183 | }
184 | 
185 | bool Neuron::isBias() const
186 | {
187 | 	return _is_bias;
188 | }
189 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | ## Ignore Visual Studio temporary files, build results, and
  2 | ## files generated by popular Visual Studio add-ons.
  3 | ##
  4 | ## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
  5 | 
  6 | # User-specific files
  7 | *.rsuser
  8 | *.suo
  9 | *.user
 10 | *.userosscache
 11 | *.sln.docstates
 12 | 
 13 | # User-specific files (MonoDevelop/Xamarin Studio)
 14 | *.userprefs
 15 | 
 16 | # Mono auto generated files
 17 | mono_crash.*
 18 | 
 19 | # Build results
 20 | [Dd]ebug/
 21 | [Dd]ebugPublic/
 22 | [Rr]elease/
 23 | [Rr]eleases/
 24 | x64/
 25 | x86/
 26 | [Ww][Ii][Nn]32/
 27 | [Aa][Rr][Mm]/
 28 | [Aa][Rr][Mm]64/
 29 | bld/
 30 | [Bb]in/
 31 | [Oo]bj/
 32 | [Ll]og/
 33 | [Ll]ogs/
 34 | 
 35 | # Visual Studio 2015/2017 cache/options directory
 36 | .vs/
 37 | # Uncomment if you have tasks that create the project's static files in wwwroot
 38 | #wwwroot/
 39 | 
 40 | # Visual Studio 2017 auto generated files
 41 | Generated\ Files/
 42 | 
 43 | # MSTest test Results
 44 | [Tt]est[Rr]esult*/
 45 | [Bb]uild[Ll]og.*
 46 | 
 47 | # NUnit
 48 | *.VisualState.xml
 49 | TestResult.xml
 50 | nunit-*.xml
 51 | 
 52 | # Build Results of an ATL Project
 53 | [Dd]ebugPS/
 54 | [Rr]eleasePS/
 55 | dlldata.c
 56 | 
 57 | # Benchmark Results
 58 | BenchmarkDotNet.Artifacts/
 59 | 
 60 | # .NET Core
 61 | project.lock.json
 62 | project.fragment.lock.json
 63 | artifacts/
 64 | 
 65 | # ASP.NET Scaffolding
 66 | ScaffoldingReadMe.txt
 67 | 
 68 | # StyleCop
 69 | StyleCopReport.xml
 70 | 
 71 | # Files built by Visual Studio
 72 | *_i.c
 73 | *_p.c
 74 | *_h.h
 75 | *.ilk
 76 | *.meta
 77 | *.obj
 78 | *.iobj
 79 | *.pch
 80 | *.pdb
 81 | *.ipdb
 82 | *.pgc
 83 | *.pgd
 84 | *.rsp
 85 | *.sbr
 86 | *.tlb
 87 | *.tli
 88 | *.tlh
 89 | *.tmp
 90 | *.tmp_proj
 91 | *_wpftmp.csproj
 92 | *.log
 93 | *.vspscc
 94 | *.vssscc
 95 | .builds
 96 | *.pidb
 97 | *.svclog
 98 | *.scc
 99 | 
100 | # Chutzpah Test files
101 | _Chutzpah*
102 | 
103 | # Visual C++ cache files
104 | ipch/
105 | *.aps
106 | *.ncb
107 | *.opendb
108 | *.opensdf
109 | *.sdf
110 | *.cachefile
111 | *.VC.db
112 | *.VC.VC.opendb
113 | 
114 | # Visual Studio profiler
115 | *.psess
116 | *.vsp
117 | *.vspx
118 | *.sap
119 | 
120 | # Visual Studio Trace Files
121 | *.e2e
122 | 
123 | # TFS 2012 Local Workspace
124 | $tf/
125 | 
126 | # Guidance Automation Toolkit
127 | *.gpState
128 | 
129 | # ReSharper is a .NET coding add-in
130 | _ReSharper*/
131 | *.[Rr]e[Ss]harper
132 | *.DotSettings.user
133 | 
134 | # TeamCity is a build add-in
135 | _TeamCity*
136 | 
137 | # DotCover is a Code Coverage Tool
138 | *.dotCover
139 | 
140 | # AxoCover is a Code Coverage Tool
141 | .axoCover/*
142 | !.axoCover/settings.json
143 | 
144 | # Coverlet is a free, cross platform Code Coverage Tool
145 | coverage*.json
146 | coverage*.xml
147 | coverage*.info
148 | 
149 | # Visual Studio code coverage results
150 | *.coverage
151 | *.coveragexml
152 | 
153 | # NCrunch
154 | _NCrunch_*
155 | .*crunch*.local.xml
156 | nCrunchTemp_*
157 | 
158 | # MightyMoose
159 | *.mm.*
160 | AutoTest.Net/
161 | 
162 | # Web workbench (sass)
163 | .sass-cache/
164 | 
165 | # Installshield output folder
166 | [Ee]xpress/
167 | 
168 | # DocProject is a documentation generator add-in
169 | DocProject/buildhelp/
170 | DocProject/Help/*.HxT
171 | DocProject/Help/*.HxC
172 | DocProject/Help/*.hhc
173 | DocProject/Help/*.hhk
174 | DocProject/Help/*.hhp
175 | DocProject/Help/Html2
176 | DocProject/Help/html
177 | 
178 | # Click-Once directory
179 | publish/
180 | 
181 | # Publish Web Output
182 | *.[Pp]ublish.xml
183 | *.azurePubxml
184 | # Note: Comment the next line if you want to checkin your web deploy settings,
185 | # but database connection strings (with potential passwords) will be unencrypted
186 | *.pubxml
187 | *.publishproj
188 | 
189 | # Microsoft Azure Web App publish settings. Comment the next line if you want to
190 | # checkin your Azure Web App publish settings, but sensitive information contained
191 | # in these scripts will be unencrypted
192 | PublishScripts/
193 | 
194 | # NuGet Packages
195 | *.nupkg
196 | # NuGet Symbol Packages
197 | *.snupkg
198 | # The packages folder can be ignored because of Package Restore
199 | **/[Pp]ackages/*
200 | # except build/, which is used as an MSBuild target.
201 | !**/[Pp]ackages/build/
202 | # Uncomment if necessary however generally it will be regenerated when needed
203 | #!**/[Pp]ackages/repositories.config
204 | # NuGet v3's project.json files produces more ignorable files
205 | *.nuget.props
206 | *.nuget.targets
207 | 
208 | # Microsoft Azure Build Output
209 | csx/
210 | *.build.csdef
211 | 
212 | # Microsoft Azure Emulator
213 | ecf/
214 | rcf/
215 | 
216 | # Windows Store app package directories and files
217 | AppPackages/
218 | BundleArtifacts/
219 | Package.StoreAssociation.xml
220 | _pkginfo.txt
221 | *.appx
222 | *.appxbundle
223 | *.appxupload
224 | 
225 | # Visual Studio cache files
226 | # files ending in .cache can be ignored
227 | *.[Cc]ache
228 | # but keep track of directories ending in .cache
229 | !?*.[Cc]ache/
230 | 
231 | # Others
232 | ClientBin/
233 | ~$*
234 | *~
235 | *.dbmdl
236 | *.dbproj.schemaview
237 | *.jfm
238 | *.pfx
239 | *.publishsettings
240 | orleans.codegen.cs
241 | 
242 | # Including strong name files can present a security risk
243 | # (https://github.com/github/gitignore/pull/2483#issue-259490424)
244 | #*.snk
245 | 
246 | # Since there are multiple workflows, uncomment next line to ignore bower_components
247 | # (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
248 | #bower_components/
249 | 
250 | # RIA/Silverlight projects
251 | Generated_Code/
252 | 
253 | # Backup & report files from converting an old project file
254 | # to a newer Visual Studio version. Backup files are not needed,
255 | # because we have git ;-)
256 | _UpgradeReport_Files/
257 | Backup*/
258 | UpgradeLog*.XML
259 | UpgradeLog*.htm
260 | ServiceFabricBackup/
261 | *.rptproj.bak
262 | 
263 | # SQL Server files
264 | *.mdf
265 | *.ldf
266 | *.ndf
267 | 
268 | # Business Intelligence projects
269 | *.rdl.data
270 | *.bim.layout
271 | *.bim_*.settings
272 | *.rptproj.rsuser
273 | *- [Bb]ackup.rdl
274 | *- [Bb]ackup ([0-9]).rdl
275 | *- [Bb]ackup ([0-9][0-9]).rdl
276 | 
277 | # Microsoft Fakes
278 | FakesAssemblies/
279 | 
280 | # GhostDoc plugin setting file
281 | *.GhostDoc.xml
282 | 
283 | # Node.js Tools for Visual Studio
284 | .ntvs_analysis.dat
285 | node_modules/
286 | 
287 | # Visual Studio 6 build log
288 | *.plg
289 | 
290 | # Visual Studio 6 workspace options file
291 | *.opt
292 | 
293 | # Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
294 | *.vbw
295 | 
296 | # Visual Studio LightSwitch build output
297 | **/*.HTMLClient/GeneratedArtifacts
298 | **/*.DesktopClient/GeneratedArtifacts
299 | **/*.DesktopClient/ModelManifest.xml
300 | **/*.Server/GeneratedArtifacts
301 | **/*.Server/ModelManifest.xml
302 | _Pvt_Extensions
303 | 
304 | # Paket dependency manager
305 | .paket/paket.exe
306 | paket-files/
307 | 
308 | # FAKE - F# Make
309 | .fake/
310 | 
311 | # CodeRush personal settings
312 | .cr/personal
313 | 
314 | # Python Tools for Visual Studio (PTVS)
315 | __pycache__/
316 | *.pyc
317 | 
318 | # Cake - Uncomment if you are using it
319 | # tools/**
320 | # !tools/packages.config
321 | 
322 | # Tabs Studio
323 | *.tss
324 | 
325 | # Telerik's JustMock configuration file
326 | *.jmconfig
327 | 
328 | # BizTalk build output
329 | *.btp.cs
330 | *.btm.cs
331 | *.odx.cs
332 | *.xsd.cs
333 | 
334 | # OpenCover UI analysis results
335 | OpenCover/
336 | 
337 | # Azure Stream Analytics local run output
338 | ASALocalRun/
339 | 
340 | # MSBuild Binary and Structured Log
341 | *.binlog
342 | 
343 | # NVidia Nsight GPU debugger configuration file
344 | *.nvuser
345 | 
346 | # MFractors (Xamarin productivity tool) working folder
347 | .mfractor/
348 | 
349 | # Local History for Visual Studio
350 | .localhistory/
351 | 
352 | # BeatPulse healthcheck temp database
353 | healthchecksdb
354 | 
355 | # Backup folder for Package Reference Convert tool in Visual Studio 2017
356 | MigrationBackup/
357 | 
358 | # Ionide (cross platform F# VS Code tools) working folder
359 | .ionide/
360 | 
361 | # Fody - auto-generated XML schema
362 | FodyWeavers.xsd
363 | *.sln
364 | *.vcxproj
365 | *.vcxproj.filters
366 | *.props
367 | 


--------------------------------------------------------------------------------
/data1000.txt:
--------------------------------------------------------------------------------
   1 | 1.15273578388546	0.21257737635896	1
   2 | 0.94964258992563	1.8001246679056	1
   3 | 0.506268134785943	-0.0112497775875888	1
   4 | 0.610730927814847	0.470970411270559	1
   5 | 0.862559348787322	1.1486276447477	1
   6 | 1.66209698227057	0.365684394029286	1
   7 | -0.115745809494604	1.92412193259498	1
   8 | 1.59167031312994	0.666033402629308	1
   9 | 1.12210901623784	1.84910490800389	1
  10 | 0.484298833869621	1.35922179808722	1
  11 | 0.570435262952333	0.766154163931393	1
  12 | 0.535927479671258	1.45390643172483	1
  13 | 1.23393182866655	1.0404115558217	1
  14 | -0.802659223067349	0.866246634546946	1
  15 | 0.195308227674012	-0.187620736441331	1
  16 | 1.15372499789863	-0.473659029283657	1
  17 | 1.09905644294908	0.724274956167921	1
  18 | 0.4814725156014	0.751641903860981	1
  19 | 0.891882704097461	0.763697530722009	1
  20 | 0.925284655634086	-0.0888097893967152	1
  21 | 1.26698083954738	0.57337326480739	1
  22 | 0.999891361574983	0.51349489451765	1
  23 | 0.631475412126529	0.709457465949987	1
  24 | 0.374463421604801	0.235788476982297	1
  25 | 0.236458274779874	2.1462989567808	1
  26 | 1.24235322354423	0.333800471101213	1
  27 | 1.18540048818619	0.815117546506841	1
  28 | 0.205835368993754	-0.16965635714065	1
  29 | 1.20617009212636	0.485826537608206	1
  30 | 1.13224982270702	1.34054634979285	1
  31 | 1.6014329311445	1.99016870282553	1
  32 | -0.412564556768198	1.96646806867303	1
  33 | 0.681182363675673	1.64761827959398	1
  34 | 1.33380193153544	2.1251143628991	1
  35 | 1.74271881053947	1.74183413213457	1
  36 | 1.46478637907494	0.939442683097405	1
  37 | 0.858700760505539	2.68199250057107	1
  38 | 0.62856544874415	1.59236713606392	1
  39 | 0.798534752324444	1.23226014255292	1
  40 | 1.95473508588377	1.71148849572201	1
  41 | 1.12344586268295	3.33076503074603	1
  42 | 1.2848047010717	1.78341344490108	1
  43 | 1.23018513738646	0.388461612251349	1
  44 | 1.20155955220768	-0.155009646737939	1
  45 | 0.940232835614571	-0.289009766300016	1
  46 | 1.34505505733819	1.12265276350426	1
  47 | 0.888441469815569	-0.744533570275477	1
  48 | 0.278896467484737	0.414647280837905	1
  49 | 1.1345857872055	1.96513541678531	1
  50 | 0.969694352865065	0.411647669684608	1
  51 | 1.61431717737327	2.27613306230595	1
  52 | 0.964606892904836	1.73572178078532	1
  53 | 0.845942404468101	-0.258966134896808	1
  54 | 1.94917848166363	0.755821604503314	1
  55 | 1.00641570444928	1.46397461260891	1
  56 | 0.649392086957547	1.38267534027376	1
  57 | 0.891897554419674	1.17861197299092	1
  58 | 1.78320338876946	0.912206773468311	1
  59 | 0.304604436844502	1.49788089689692	1
  60 | 1.6736543971619	2.43223365403931	1
  61 | 1.92231549666514	0.730618660491931	1
  62 | 0.26365121484938	1.60241825529422	1
  63 | 0.929567242548111	1.22017242031295	1
  64 | -0.300567976127808	1.6473766998607	1
  65 | 0.452299065559125	-1.13578139529919	1
  66 | 1.14376897717928	0.381152973361921	1
  67 | 1.8694251917924	0.869489491274765	1
  68 | 1.44951839905884	1.09964305743715	1
  69 | 1.68108647272693	0.677914765806214	1
  70 | 1.46105980083593	-0.278110666241197	1
  71 | 1.34830297406977	0.671075629066109	1
  72 | 1.15314971783743	1.19984610022602	1
  73 | 1.25455825171213	1.92360604261621	1
  74 | 1.31428069899045	0.921275012321399	1
  75 | 0.785637185356404	-0.315035697858343	1
  76 | 0.347497104643629	1.30353203356169	1
  77 | 0.66962716048591	2.66949729236113	1
  78 | 0.556183351254885	1.31874620647405	1
  79 | 1.29205969653884	1.11336769112604	1
  80 | 1.34710370732163	2.02670023634836	1
  81 | 0.943673397603429	0.489035732453922	1
  82 | 1.24933669275498	0.710829704248796	1
  83 | 0.753084676134577	1.07642031497203	1
  84 | 0.141252676299188	0.949339553287678	1
  85 | 1.42779818397039	2.10639564098084	1
  86 | 1.62402414062259	1.83115826446729	1
  87 | 1.18686871154805	1.04263685698642	1
  88 | 0.88101567255999	1.09274010249891	1
  89 | 1.98715178595941	1.87327154104575	1
  90 | 1.83937222862361	0.169436153165957	1
  91 | 1.28833298548511	0.360506978812219	1
  92 | 0.612952158000388	1.26533850084362	1
  93 | 0.518211411181762	1.01125747694075	1
  94 | 1.39661130267157	-1.15171118816806	1
  95 | 0.166151005818215	0.867102199809384	1
  96 | 1.68514825439068	0.584068128698417	1
  97 | 1.06399021644791	0.754771128504615	1
  98 | 1.86027621703725	0.0383136785709635	1
  99 | 0.831901921666795	0.836224515095568	1
 100 | 1.61074231632055	1.59425600209037	1
 101 | 0.800897651415284	-0.028670376038973	1
 102 | 1.19837679159158	0.268153758326714	1
 103 | 1.79762701056046	-0.290040868659326	1
 104 | 0.290908364553274	0.969128754469186	1
 105 | 0.589039696897695	0.459792399506209	1
 106 | 1.5094012474072	0.609148671588815	1
 107 | 0.687528267761284	0.148741774310458	1
 108 | 0.513527692808704	1.54948197098197	1
 109 | 0.94559365483116	1.92699317853382	1
 110 | 0.533325976397152	1.35622820987855	1
 111 | 1.63565660848274	1.33431187006952	1
 112 | 0.764043855875862	1.28968619062575	1
 113 | 0.846607239280878	1.69420136462182	1
 114 | 0.912073792208978	0.925227278120343	1
 115 | 0.879896632582671	0.525351208329327	1
 116 | 1.26769889959745	0.879528664964687	1
 117 | 1.81570820894252	1.61682882916921	1
 118 | 0.939591162436833	2.46224560816321	1
 119 | 0.616232547500556	1.29537080554575	1
 120 | 1.39070515308644	0.470291808527469	1
 121 | 0.184699308637125	-0.315076367952515	1
 122 | 1.02386269297491	0.0635865835576069	1
 123 | 0.673763323930456	-0.201592874886233	1
 124 | 1.05889324719987	1.66780156135932	1
 125 | 0.58890010896017	0.738931047948522	1
 126 | 0.623081133736952	0.944738229216627	1
 127 | 1.44985728499651	1.67023345155028	1
 128 | 1.03376580699826	0.860759401390624	1
 129 | 0.451141175120225	0.812220845983542	1
 130 | 2.05999797531524	-0.286556635428574	1
 131 | 1.3286976962344	2.19974919135649	1
 132 | 1.07917585994208	1.86875921782443	1
 133 | 2.57541270569501	1.55052529811226	1
 134 | 1.08436141498605	1.01591319844901	1
 135 | 0.807720575180992	1.17066696027883	1
 136 | 0.981740822251095	-0.558663675279141	1
 137 | 1.16807498475822	0.855860585640406	1
 138 | 0.877213719353517	0.336318491444371	1
 139 | 0.656837123539596	0.517036433207043	1
 140 | 1.96662410000034	0.529030758002474	1
 141 | 0.765119672916317	1.98185025538059	1
 142 | 1.36425445231822	0.10881948720051	1
 143 | 1.368955474552	-0.468343803812417	1
 144 | 0.966211806132058	2.26118258202606	1
 145 | 0.389140163923078	1.102384362767	1
 146 | 0.732997107373459	0.784604715541476	1
 147 | 0.18238726033065	-0.422153156918587	1
 148 | 1.23004111208067	-0.421729773143255	1
 149 | 0.689951856767005	0.353141430875348	1
 150 | 0.877105285082842	0.57370069511857	1
 151 | 0.707920724255824	1.56884012856074	1
 152 | 0.642320878814432	1.991165754675	1
 153 | 0.00352650312557046	0.266735953115617	1
 154 | 0.703859578167859	2.12686923622759	1
 155 | 1.00366843587678	1.06463932092761	1
 156 | 1.42291055179853	-0.265571954258707	1
 157 | 0.918068709046335	0.616523545658727	1
 158 | 0.677838635117428	2.88960891104262	1
 159 | 0.753773788232949	0.264665201188754	1
 160 | 1.41804055695753	0.812363840341614	1
 161 | 1.0220265996941	-0.490213621600202	1
 162 | 1.55213366784787	1.18600684194808	1
 163 | 0.33142761766819	1.5268942688687	1
 164 | 0.258419930004294	0.651314603457965	1
 165 | 0.532872474308711	1.11231093945315	1
 166 | 1.1067969160795	-0.0862738816719824	1
 167 | 0.84963129874547	0.924203564086759	1
 168 | 0.306767606167871	2.40702609121339	1
 169 | 1.04723668754705	2.28586794953061	1
 170 | 1.20516590204108	-0.583628168921166	1
 171 | 0.70708314689913	-0.0330669208883589	1
 172 | 1.53827291013415	0.222480969557574	1
 173 | -0.00822984325754006	1.69737176843978	1
 174 | 1.64199770220872	0.254079278473743	1
 175 | 0.742955731445666	-0.130097503802909	1
 176 | 0.276395357037131	0.604699880657067	1
 177 | 1.256868660166	0.569375176151526	1
 178 | 0.980571251343329	1.72396407800168	1
 179 | 1.83919272978002	-0.0240679305724716	1
 180 | 0.552527332879548	1.51005652990031	1
 181 | 0.996605089911655	-0.978864609215645	1
 182 | 0.818536395879315	1.25441392366363	1
 183 | 0.768517147334897	-0.9294489989771	1
 184 | 1.02430727313239	2.17403351050379	1
 185 | 1.49718523851638	0.0198126392794394	1
 186 | 1.36064821440667	1.11565784541853	1
 187 | 1.27692281677156	1.57066906906399	1
 188 | 1.3308673666024	1.67283310977511	1
 189 | 0.529497435969829	-0.0945139917053626	1
 190 | 1.54894079172058	1.02670765728235	1
 191 | 0.601044469296191	2.42949987104468	1
 192 | 0.541302781752324	0.942740318164391	1
 193 | 0.542315991963865	1.95414856721451	1
 194 | 0.903697260410127	0.862976459461731	1
 195 | 1.41737371233177	2.07539057255077	1
 196 | 0.488759579377833	1.47064620306683	1
 197 | 1.16525294833904	-0.200270753850964	1
 198 | 0.12780473846798	-0.0585620920481993	1
 199 | 1.4811662626545	2.14415509908376	1
 200 | 1.06987480565312	1.02226270116411	1
 201 | 1.42432217035108	1.01396588936911	1
 202 | 1.99523804190944	1.91830802205185	1
 203 | 1.11923461620935	0.67158136683119	1
 204 | 1.26184365579708	1.84076368385382	1
 205 | 0.239540956551794	1.26936112504058	1
 206 | 0.904340850257161	2.22317504435801	1
 207 | 0.802130269087827	0.663302740091138	1
 208 | 0.615671455749375	1.67833476810082	1
 209 | 0.961537340302357	0.939796457770012	1
 210 | 1.07027470229475	0.643594201472018	1
 211 | 0.746980217089311	0.385533382136253	1
 212 | 0.528059388140348	0.61076960748232	1
 213 | 0.74238898749446	0.557109192797668	1
 214 | 0.84575595235736	0.972368484039117	1
 215 | 1.17416189036589	0.705763321500845	1
 216 | 0.477968038701929	1.04749276327224	1
 217 | 0.665899357838583	-0.0865449975637267	1
 218 | 1.24406980623511	0.990223200253242	1
 219 | 1.39276748900997	0.756571187971916	1
 220 | 1.75178477451947	1.22763828618365	1
 221 | 1.05773794250721	-0.016377314427088	1
 222 | 0.888588624173306	0.619603973263414	1
 223 | 0.0905843097763653	1.07056160030309	1
 224 | 1.26303860565074	2.40217508932353	1
 225 | 0.142852769840449	1.26864087961282	1
 226 | 0.536189078154242	0.197736450956469	1
 227 | 0.477274279785008	1.26561103716281	1
 228 | 0.640623737123149	0.198605217649864	1
 229 | 1.49233282157011	0.6539980983025	1
 230 | 0.775751051148346	0.873244525947233	1
 231 | 1.52133584480733	1.26462252162053	1
 232 | 0.329707712917829	0.299544962067256	1
 233 | 1.25903487838341	0.0375961483284746	1
 234 | 1.50506084284886	1.78475176838189	1
 235 | 0.975137491265805	2.50370837476753	1
 236 | 0.647670490179013	-0.29098715124199	1
 237 | 1.1333069614861	0.826358629282567	1
 238 | 0.438774371494321	0.781165251388391	1
 239 | 0.771020847192314	1.64189526462984	1
 240 | 1.20783673371204	-0.395033131953269	1
 241 | 1.14420909789303	1.18294753858309	1
 242 | 0.223990185765801	0.633304967791448	1
 243 | 0.774398813899101	2.41381439569812	1
 244 | 0.488961193014068	1.38611569457277	1
 245 | 1.06202243257586	0.935782354474094	1
 246 | 1.32641923312633	0.92658408669038	1
 247 | 0.411309814301741	1.99980004358614	1
 248 | 1.35894436489617	1.15019595475569	1
 249 | -0.00419049195140397	1.5388431259954	1
 250 | 1.06902319540564	0.98907022309696	1
 251 | 1.49400786682092	1.84738548632523	1
 252 | 1.05065326211142	2.15281700257896	1
 253 | 0.698638206662301	1.31336097417043	1
 254 | 0.410449095515294	0.772702676859215	1
 255 | 1.24312153684995	1.19964900245474	1
 256 | 1.34275533495531	1.24433108026767	1
 257 | 0.808469443891267	-0.186301422977162	1
 258 | 1.16016077209909	0.376026909915934	1
 259 | 1.62066024543593	1.27297953015267	1
 260 | -0.0019892426765562	0.688251864039931	1
 261 | 1.17688650471265	0.835399737422933	1
 262 | 1.57974692140787	0.0118377551910194	1
 263 | 1.38665917073211	1.02548706334481	1
 264 | 1.19960473674093	2.23162516437298	1
 265 | 0.514664714103908	0.231819243112072	1
 266 | 1.11897148994883	1.39817507688454	1
 267 | 0.932805139083077	0.703688592946457	1
 268 | 0.431104891618601	2.57885449202909	1
 269 | 1.17992402467721	-0.407246048187297	1
 270 | 1.50097804132048	3.77425834045326	1
 271 | 1.12186807802839	0.121841623858864	1
 272 | 1.46661089601852	0.755520814442053	1
 273 | 1.80921143542196	1.81498738190751	1
 274 | 1.16998569624983	-0.0711127614278453	1
 275 | 1.45662442716852	1.44259291494202	1
 276 | 0.893089783339124	2.05784535779694	1
 277 | 1.26869554122867	1.45169568004713	1
 278 | 1.59085030753715	0.898508050957702	1
 279 | 0.443694797390242	0.30441707636033	1
 280 | 0.770223059224685	1.9722756401121	1
 281 | 0.443548727715314	1.45228853763036	1
 282 | 2.25418527308121	1.69616985532131	1
 283 | -0.710453948635507	0.33796579603465	1
 284 | 1.23984040497361	0.452229323295914	1
 285 | 0.337113101782864	0.340844272453725	1
 286 | 0.217974657667162	0.779615740763137	1
 287 | 1.8980490526845	-0.0736090937018399	1
 288 | 1.38732526152072	0.66933164444854	1
 289 | 1.13846177396597	2.16451393472161	1
 290 | 0.665892590684329	0.383489582977802	1
 291 | 0.301551357121722	2.23769019866766	1
 292 | 0.753888546801367	0.935888934376158	1
 293 | 1.61483013327294	2.42665745950732	1
 294 | 0.428225915786521	0.280699831228622	1
 295 | 0.539898899136257	1.26849339464119	1
 296 | 0.970353529333019	0.804799048013164	1
 297 | 1.40238212998307	1.10616260451006	1
 298 | 1.39520032606366	0.968258488577923	1
 299 | 1.53536811605198	1.13622177153818	1
 300 | 0.801484556633123	1.4125190105347	1
 301 | 1.87762532862661	-0.175174219269607	1
 302 | 1.22871631538393	0.121975134113979	1
 303 | 1.00213063478924	0.455246324842988	1
 304 | 1.79371690890475	0.138790514271724	1
 305 | 0.791412014890638	-0.164374097570123	1
 306 | 0.511159296864042	0.908432831503239	1
 307 | 2.29560056583369	-0.659854215696834	1
 308 | 1.26061837052792	1.62698259685289	1
 309 | 0.698440187059334	1.11562398525936	1
 310 | 0.735102061451616	0.976938562757828	1
 311 | 1.53097668990848	1.17179270887797	1
 312 | 0.77432807851886	1.60571115414502	1
 313 | 0.692395009719398	0.991705987326756	1
 314 | 1.05849127126632	0.446752822038008	1
 315 | 1.07019641489215	2.1788778072499	1
 316 | 0.752733607946411	0.284353672034458	1
 317 | 1.08128673108432	2.1195897620386	1
 318 | 0.600945638421292	1.15273883212847	1
 319 | 1.1780638690328	1.1522772326508	1
 320 | 0.918460964740724	0.44116340803307	1
 321 | 0.628386208133821	0.219193156182268	1
 322 | 0.949349435872163	0.845858112985058	1
 323 | 1.28377137575748	1.94196339383427	1
 324 | 1.15397636209944	0.186154786163679	1
 325 | 0.648980322315949	0.948517955914686	1
 326 | 1.49583680289564	1.29124177284433	1
 327 | 0.917602435190343	-0.343660645586631	1
 328 | 0.903641934193245	1.62138023094814	1
 329 | 1.32783284052679	1.00683870664764	1
 330 | 1.51922551480233	0.669364696267903	1
 331 | 0.523129065898582	1.44836949597447	1
 332 | 1.30147157085243	1.4947325556444	1
 333 | 0.721766125278754	1.26013721678694	1
 334 | 1.09856412429887	0.569466855724271	1
 335 | 0.416274179056099	0.195764048998364	1
 336 | 1.2455221453431	2.24007888871553	1
 337 | 2.51348136345849	1.03752984705614	1
 338 | 1.545677788741	1.71349799937475	1
 339 | 1.32747549551899	0.707223299602905	1
 340 | 1.73569458558995	1.21714665381737	1
 341 | 0.976620087093916	-0.105474561117053	1
 342 | 1.82180162551835	1.00706162171708	1
 343 | 0.0420654976335457	1.13549146455019	1
 344 | 1.00393994571408	1.72255601220701	1
 345 | 1.47342159858955	1.22808963210437	1
 346 | 1.62360617217519	2.65187374207472	1
 347 | 0.936990405826646	-0.436948222988959	1
 348 | 0.897921051553	-0.120854236495305	1
 349 | 2.0715145077717	1.75724847407964	1
 350 | 1.16830794134911	2.94256399615744	1
 351 | 1.25229861506318	1.98067774142994	1
 352 | 0.955642488641036	0.388553361265765	1
 353 | 0.428266861786153	2.17295204040958	1
 354 | 1.30192506839436	1.93677084852954	1
 355 | 0.237433934107722	-0.180397230489396	1
 356 | 1.4242842126376	3.17813653974749	1
 357 | 1.17693649916869	0.634791151846117	1
 358 | 1.62646851508772	-0.306519604914079	1
 359 | 0.83004966232207	0.134161301539317	1
 360 | 0.709689023504871	-0.355420965163006	1
 361 | 1.50161054654416	1.49283142669529	1
 362 | 1.11867467023887	-0.256680349350084	1
 363 | 0.792278565770811	1.56100002945267	1
 364 | 1.27832974209975	1.66669426645969	1
 365 | 0.928388825117697	1.19627701968076	1
 366 | 1.47502329178345	1.65980437560816	1
 367 | 0.519587840507383	1.18836850057509	1
 368 | 1.16003607398035	0.403567579418214	1
 369 | 1.56729543181633	0.745014742554934	1
 370 | 0.734505042212934	1.40609524401851	1
 371 | 0.715664565446361	-0.216749183825402	1
 372 | 1.31627046564396	1.96511134372555	1
 373 | 1.25710265386194	0.314010381841631	1
 374 | 1.01855381989735	2.0840335279672	1
 375 | 1.62180099647263	1.85160784041522	1
 376 | 0.305569051063366	1.15158016121276	1
 377 | 0.316283186887344	1.98986081071442	1
 378 | 1.0983533411222	1.76760340227538	1
 379 | -0.225854166546228	1.92277020738106	1
 380 | 0.54500059534264	0.187169572366993	1
 381 | 0.284728750408284	0.707411091468733	1
 382 | 0.241294688314468	0.270215782082366	1
 383 | 0.551045750192106	0.928992334249651	1
 384 | 1.25414046000125	1.7715541122688	1
 385 | 0.704094969823891	1.23426480257283	1
 386 | 0.888429814734981	1.86682477010194	1
 387 | 0.853218496068636	0.883266742027534	1
 388 | 1.16194222897221	0.324827584099396	1
 389 | 0.887598949714155	1.38441270123288	1
 390 | 1.19608326518853	0.173675275412025	1
 391 | 1.42158161843846	1.13886627447106	1
 392 | 1.48852608068914	0.205952442891781	1
 393 | 1.05466101753272	2.3288537450024	1
 394 | 0.553947132819449	2.57837411355345	1
 395 | 0.759490894806114	0.16418195217842	1
 396 | 0.865158380525638	0.725562908534306	1
 397 | 1.10708029772628	-0.311690388206348	1
 398 | 1.87069145325502	2.16489979765885	1
 399 | 0.969618908807015	0.439172501492142	1
 400 | 1.02095985272485	0.755749020066072	1
 401 | 0.871834967550099	0.783575347480216	1
 402 | 0.321328563778363	0.925545014484535	1
 403 | 0.547720463603196	0.0663348755874101	1
 404 | 0.838556870050553	0.692462083387595	1
 405 | 0.786394608412176	0.769707901868353	1
 406 | 0.42492715105517	2.7449123718136	1
 407 | 1.10511109334242	0.362243378217285	1
 408 | 1.17960367522342	-0.173092398612374	1
 409 | 0.73013533138462	1.49608882249293	1
 410 | 1.38340783924411	0.0291311967484638	1
 411 | 1.73181701227123	2.07210527322189	1
 412 | 1.46893651626362	1.0694691852915	1
 413 | 1.99639478072402	0.930688991657256	1
 414 | 1.84463165502424	0.924112613927926	1
 415 | 1.58133342426003	0.97794708726642	1
 416 | 0.775460902837926	1.64670932894364	1
 417 | 0.876834391074975	-0.0810441106786295	1
 418 | 1.49606694443288	1.48676809188498	1
 419 | 1.02769205247979	0.367965422545627	1
 420 | 0.209058550425646	0.307973586306804	1
 421 | 1.05302942084877	0.58188305547858	1
 422 | 0.790889782181003	0.84012870795805	1
 423 | 0.906283744265202	1.74064171280474	1
 424 | 0.406434258083711	1.23847521950428	1
 425 | 0.170407285531031	2.21898160843297	1
 426 | 0.816444210270217	1.4418433769058	1
 427 | 1.39538165353368	0.486008564385953	1
 428 | 1.88110791658582	1.26132348988836	1
 429 | 1.22272315482987	0.413179767781294	1
 430 | 0.966623469818824	1.23723754655581	1
 431 | 1.25155452535912	0.571353707742376	1
 432 | 0.866521657564928	2.0374428829957	1
 433 | 1.56523743666909	1.38743435227743	1
 434 | 1.32261605631686	0.341371624469599	1
 435 | 0.118920859342392	1.04888862925321	1
 436 | 0.413009527746384	1.58485893215135	1
 437 | 1.5104572733682	1.77276387529576	1
 438 | 0.861232865137762	0.608064238788054	1
 439 | 1.47098659417915	0.0847145417984967	1
 440 | 0.579944665249717	1.97036194906407	1
 441 | 1.43411965207425	1.62532706184605	1
 442 | 0.872804720466879	-0.370739253768121	1
 443 | 1.16398472178182	0.632883732258493	1
 444 | 1.31402632590769	0.579553167054653	1
 445 | 1.13088656910311	-0.499502799051652	1
 446 | 0.54976913667242	0.614291931555717	1
 447 | 1.04798913222114	1.04701851840656	1
 448 | 0.740045685669019	2.29704473564781	1
 449 | 1.03822820003474	0.637018250609876	1
 450 | -0.74338179318549	1.15063337255293	1
 451 | 0.820125206998538	0.587211058963277	1
 452 | 0.870181032491549	1.14529156806058	1
 453 | 0.766456407881779	1.11860850702646	1
 454 | 1.17766443963384	-0.679852629827624	1
 455 | 0.725384781442923	0.395803883827534	1
 456 | 1.64716975974069	1.1936492767845	1
 457 | 0.797165176868915	1.52903291228394	1
 458 | 1.2361023595214	1.5955074587853	1
 459 | 1.37663331732663	0.383429211749744	1
 460 | 0.514157482486713	2.40195082025437	1
 461 | 0.289593466305051	0.108742880391006	1
 462 | 0.800005634720055	1.47552561507344	1
 463 | 0.981307122819778	1.9680461793143	1
 464 | 0.56585366996678	-0.0874618838014947	1
 465 | 1.05173665584909	-0.239670974546157	1
 466 | 2.1308288004231	1.03473817604654	1
 467 | 1.58007673726025	1.91463799747517	1
 468 | 0.721147736756677	1.22191496085559	1
 469 | 0.434385206108127	2.30606676011047	1
 470 | 1.425596821983	-0.536867567149436	1
 471 | 1.71690865923454	1.74619955829964	1
 472 | 0.996687366863778	0.948419999284287	1
 473 | 0.616096260296861	1.89631552013086	1
 474 | 0.288588774365101	0.623652355949765	1
 475 | 0.999641825389546	-0.707870080517336	1
 476 | 0.975772641739718	1.3007860032344	1
 477 | 0.715757365956153	1.18163187980749	1
 478 | 0.649465881725845	1.60722480697682	1
 479 | 1.40313792752957	0.661666656633481	1
 480 | 0.430775406465006	0.307033147274252	1
 481 | 1.92361783104624	1.73509802998443	1
 482 | 0.90457761581173	1.927924313677	1
 483 | 0.0359108446671724	0.284418654863892	1
 484 | 0.752060190827249	0.333362372273614	1
 485 | 1.55757241922598	2.634830930418	1
 486 | 0.551914270802119	1.44739814575118	1
 487 | 1.80402058834943	1.65725895445731	1
 488 | 0.501812307830681	0.795177393334747	1
 489 | 1.40450712737624	2.40061660999058	1
 490 | 1.46853227519165	1.14344011705947	1
 491 | 1.30943676621046	0.977702670546646	1
 492 | 1.08649041469557	1.61153846487519	1
 493 | 0.976877641231015	0.224809938186128	1
 494 | 1.47009817188671	-0.372485298790092	1
 495 | 1.1414058550446	1.85109373333138	1
 496 | 1.30447282350995	0.252935605327651	1
 497 | 0.634530445546232	0.808216292421395	1
 498 | 1.4896537813545	-0.474988785411294	1
 499 | 0.983517409890666	0.986031466179977	1
 500 | 1.38951061116934	1.37367806182902	1
 501 | -1.39749318422114	0.985549603584835	0
 502 | -0.911447115125852	1.43428326206072	0
 503 | -0.279593131863745	-0.331516651249231	0
 504 | 0.0623640873285474	0.033500582932919	0
 505 | -1.22552576432444	-0.196157830267638	0
 506 | 0.350150392094052	0.492471201074218	0
 507 | -2.08131224129796	-0.571824312813781	0
 508 | -1.17587089914121	0.455253475842037	0
 509 | -1.05960813592082	-0.480898498419671	0
 510 | -0.406038875663884	-0.177239351760164	0
 511 | -0.717075884254233	0.0299079102653143	0
 512 | 0.136912999483108	0.225540284338881	0
 513 | -0.409546614016324	0.0631814213546873	0
 514 | -1.9334556046073	-0.803385456119395	0
 515 | -1.04379966298024	-0.641916413944376	0
 516 | -0.737256753821335	0.281814660329348	0
 517 | -0.171380389608303	0.57014699913633	0
 518 | -0.379608901896138	-0.310783999000651	0
 519 | -1.57619908597744	-0.0417200927601575	0
 520 | -1.32197537063588	0.147270599506783	0
 521 | -1.17878293660732	0.483671647282531	0
 522 | -1.22223106225823	-0.314332502860819	0
 523 | -1.4443415308193	0.0222090241066165	0
 524 | 0.0828057490296348	0.292742664098642	0
 525 | -0.215821365682687	-0.80074534854777	0
 526 | -0.982842797773595	0.0900666719202211	0
 527 | -1.49707408664257	0.567293997466077	0
 528 | -2.25112954841801	-0.617722889473354	0
 529 | -0.945851698852427	0.588856140350531	0
 530 | -1.65846646342996	-0.0706542849225753	0
 531 | -1.01954791671794	0.214837074676336	0
 532 | -0.990751207577067	0.601679242828798	0
 533 | -0.871565343974621	-0.354351871839786	0
 534 | 0.967761859629561	-0.679751801393788	0
 535 | -1.58940528392106	-0.616317311044864	0
 536 | 0.75832660630791	-0.092852297692803	0
 537 | 1.16159691490401	0.10034100518588	0
 538 | -1.86767434518462	0.335416990417245	0
 539 | -0.935446220370281	-0.430619942322295	0
 540 | -0.96769084014006	0.168474109454183	0
 541 | -1.33409525761342	0.177744935452753	0
 542 | -1.42743028505492	0.660740240060747	0
 543 | -1.36024501625721	-0.276147309875671	0
 544 | -1.62126605568091	0.505935910625282	0
 545 | -0.738606384607395	-0.467174740962911	0
 546 | -0.563907389679999	-0.137074733386573	0
 547 | -1.18839105619963	-0.38679473009889	0
 548 | -0.975057460208833	1.2105635626773	0
 549 | -1.81052365284981	-0.0752260780951495	0
 550 | -0.994782857074826	-0.485824719770811	0
 551 | -0.315687884402258	0.392973877959139	0
 552 | -2.03620252692477	-0.266812065070683	0
 553 | -0.793963668353933	-0.160890652490274	0
 554 | 0.774560511223425	-0.665392144160722	0
 555 | -1.12041562342481	0.0173956503236053	0
 556 | -3.04849573962648	0.042375036644456	0
 557 | -1.05396554273338	0.0298382226342729	0
 558 | 0.211036341332572	0.310722099461093	0
 559 | -0.647330553341865	-0.876891533987667	0
 560 | -1.5123249694229	-0.562267090451313	0
 561 | -1.137691334723	-0.328189909620457	0
 562 | -0.593911430180213	-0.0652204240628438	0
 563 | -0.499714807308026	0.970307588226678	0
 564 | -0.949863855127405	-0.424102100810738	0
 565 | 0.224109250175034	0.545377833720991	0
 566 | -1.1121682899594	-0.591453632399506	0
 567 | -1.40692123497578	-0.0369937304704969	0
 568 | -0.716767549804229	-0.0843139080425041	0
 569 | -2.03567455880553	0.387941767126996	0
 570 | -2.12427785918547	-0.266550777118865	0
 571 | -1.16224779910542	0.861926190722275	0
 572 | -0.6432091790682	0.21487813986094	0
 573 | -2.30864075778359	-0.456312023153919	0
 574 | -0.665351735187041	-0.122596626474153	0
 575 | -0.380843569566092	0.616630020484767	0
 576 | -1.3064767761115	-0.058875487961859	0
 577 | -2.3293628314213	0.890939410631868	0
 578 | -0.689513880191774	-0.143470427368946	0
 579 | -1.42573277131319	-0.379073060620355	0
 580 | -0.800598740495026	1.35022952283774	0
 581 | -1.49707713868531	1.39012774937504	0
 582 | -1.5090366394352	0.55453065966415	0
 583 | -0.385225369847746	0.663296942744759	0
 584 | -1.42418218506111	0.0952035090354734	0
 585 | -0.808643063346449	-0.133431258607252	0
 586 | -0.940807132852305	-0.847851167369185	0
 587 | 0.122856581790163	-0.0945975122290785	0
 588 | -1.81668270508442	0.343064783906849	0
 589 | -1.13985376313057	0.486177110427628	0
 590 | 0.110858306710959	-0.542824703357505	0
 591 | -0.352147105012369	0.260517612027789	0
 592 | -1.53543883862493	-0.599779350383844	0
 593 | -2.00556882691948	0.388478490597612	0
 594 | -1.29152703076659	0.450874325146827	0
 595 | -1.5085982519969	-0.266962335692088	0
 596 | -0.889630288513089	0.300494081436528	0
 597 | -0.239721400432427	0.00519159211607685	0
 598 | -1.99593644854164	-1.19859217591587	0
 599 | -0.821911413825429	-0.844382140799278	0
 600 | -0.531720344451361	-0.293884915027372	0
 601 | 0.470200146240839	0.562881864928679	0
 602 | -0.599535221742001	-0.687225462722081	0
 603 | 0.0751564945492675	-0.464584089918762	0
 604 | 0.0565744753232256	-0.590798667610334	0
 605 | -1.22685888832993	-0.176732581320622	0
 606 | -1.35706853749228	-0.129789714348081	0
 607 | -1.29904259600944	0.632546807155242	0
 608 | -0.214286071904815	0.964583186730311	0
 609 | -1.01399438946082	-1.02343704946145	0
 610 | 0.0250671627039254	-0.220769637256122	0
 611 | -1.41392088057531	-0.452310398570085	0
 612 | 1.09242020805804	-0.317426278560572	0
 613 | -0.533447827854815	0.586571356856096	0
 614 | -0.815638835277833	-0.118450429355601	0
 615 | -1.32491405035827	-0.274435764822583	0
 616 | -0.661912287045129	-0.569807662606845	0
 617 | -1.69393506052467	0.905357285842849	0
 618 | -0.331705859080453	0.573162886000329	0
 619 | -0.577934361667102	-0.0981712242841455	0
 620 | -1.91445126004964	-0.740184496852681	0
 621 | -2.35538863937829	0.299927512272208	0
 622 | -0.858685411731886	-0.206445081711131	0
 623 | -1.13256364500668	0.27274301873881	0
 624 | 0.0906160870706632	0.0940943272813937	0
 625 | -1.45039484810179	-0.0376116366435751	0
 626 | -0.430463820439493	-0.667304167037488	0
 627 | -1.64898694474877	0.043826607884557	0
 628 | -0.991652672531362	0.173636298105243	0
 629 | -0.744329548290704	-0.279174360519871	0
 630 | -1.91016801080934	-0.0942451478223065	0
 631 | -0.520201195233263	0.648162917802463	0
 632 | -1.1717480059291	0.524028389651994	0
 633 | 0.117691927601347	-0.0977075661425433	0
 634 | 0.476412575120353	-1.13900590520868	0
 635 | -0.632105060153421	0.286250363514944	0
 636 | -1.3051344856675	0.0711065874043525	0
 637 | -1.27291037629561	-0.222702058303346	0
 638 | -1.0865715066876	0.32651848193858	0
 639 | -1.70970205985984	0.629877800367217	0
 640 | -1.07896577317168	-0.583871855607992	0
 641 | -0.424953776800462	-0.0250564447577142	0
 642 | 0.329499368519252	0.697852448559959	0
 643 | 0.519896897286338	-0.279820093017389	0
 644 | -0.911366253277085	-1.0438846117868	0
 645 | -0.670410613394435	0.24949683641101	0
 646 | 0.191872509535987	-0.335461237636613	0
 647 | -0.99925203526341	-1.27877201469585	0
 648 | -1.62556422541239	0.564288397999028	0
 649 | -0.867680294697666	-0.528803639331474	0
 650 | -1.56125979020151	1.05028208936631	0
 651 | -1.43276502459305	-0.142869987379291	0
 652 | -1.46294966496509	-0.441462440888937	0
 653 | -2.48343458637661	0.330966910164533	0
 654 | -0.515366654214948	0.348383072637345	0
 655 | -0.859617291458323	0.246278379883382	0
 656 | -1.37177040149844	0.534210678052021	0
 657 | -1.01735745111111	0.156505551122623	0
 658 | 0.0616634551400483	-0.3843594226967	0
 659 | -1.33944609558415	0.0322126384840761	0
 660 | 0.113956547686258	0.773422269619429	0
 661 | -0.50041667978994	-0.634542882466962	0
 662 | -1.48567408898363	-0.371733954622734	0
 663 | -1.40684144066476	-0.541271847212128	0
 664 | 1.01366217590918	0.168163119587681	0
 665 | -1.56525805160876	-0.219339124522855	0
 666 | -0.994057811562879	-0.371375828363102	0
 667 | -1.28107153503362	-0.726784629227305	0
 668 | -2.02694268226798	-0.580417158427618	0
 669 | -1.27889257929818	0.00630520324847695	0
 670 | -1.53154814667579	-0.403743289562353	0
 671 | -0.936878324818582	-0.466008151901255	0
 672 | -0.92930534525892	0.0752922418120343	0
 673 | -0.797207820292179	-0.384282995250462	0
 674 | -1.81596857401805	0.267514528619325	0
 675 | 0.270297473833591	0.0453282877446888	0
 676 | 1.4778487466122	0.237988348181332	0
 677 | -1.76373578996187	-0.299328677893239	0
 678 | -1.44306344105048	0.668622793916665	0
 679 | -1.8915988072792	0.180622866122093	0
 680 | -0.603202819136837	-0.275078180913018	0
 681 | -0.185659128875696	-0.154814662476338	0
 682 | -0.314449157255117	-0.492180952477543	0
 683 | -1.22938410033926	0.0210180229066243	0
 684 | -0.620682483353608	-0.379999917015335	0
 685 | -0.947742517462424	0.390705171483198	0
 686 | -2.61378966041632	0.0666074814417724	0
 687 | -0.457711006199624	-0.109571038145409	0
 688 | -1.40532975922183	0.4840319633608	0
 689 | -1.52376161062834	0.154060571557175	0
 690 | -1.91715326715963	-0.271611519188021	0
 691 | -1.64624797506569	0.20568827315171	0
 692 | -2.59783294973035	-0.337262973448863	0
 693 | -1.29243547245548	-0.829836177483822	0
 694 | -1.28591806869764	0.541351488250738	0
 695 | 0.604056806836889	-0.599520428107509	0
 696 | -1.13721684688907	-0.184356204317554	0
 697 | -1.31703626469931	0.473720146472614	0
 698 | -0.946423170652819	0.0322852987658482	0
 699 | -1.83436632586897	-0.337031107993798	0
 700 | -1.50304146427099	0.59916377780685	0
 701 | -0.392527214669102	0.39159848227919	0
 702 | -1.33050256530868	-0.303629478161399	0
 703 | -2.16479815327035	0.264349257697376	0
 704 | -1.50459975643036	0.337435897872548	0
 705 | -0.950905190162815	-0.289752637088557	0
 706 | -0.837869341839662	0.562947973916589	0
 707 | -0.957819454593444	0.846961891121246	0
 708 | -0.777829474948845	0.434899030750202	0
 709 | -1.27569246834901	-0.109065464350906	0
 710 | -1.40373572098178	0.391016056826292	0
 711 | -1.11756366147447	0.0723756851946268	0
 712 | 0.159812758058133	-0.666324214835	0
 713 | -3.75373805359287	0.136342240602767	0
 714 | -1.28866614282154	0.0154113601256191	0
 715 | -0.0207110583619664	0.647489709045255	0
 716 | -1.77471044153959	0.10462851171268	0
 717 | 1.07533320850359	0.213237312932928	0
 718 | -1.23824878435691	0.429783382753027	0
 719 | -0.99055788715403	-0.202279528386627	0
 720 | -2.27665845862815	-0.455227851996691	0
 721 | 0.0861376476032829	-0.337266307033198	0
 722 | -0.423511628374056	-0.369416152123556	0
 723 | -1.52440052591657	0.756492600728346	0
 724 | -0.951994651412354	0.944268393430337	0
 725 | -1.72521525168875	-0.480193668504795	0
 726 | -1.50891683423285	0.258491012144598	0
 727 | -1.21376049702238	-0.198527234984623	0
 728 | -1.91239057625301	-0.638022437334577	0
 729 | -1.20216905857279	-0.439755100618252	0
 730 | 0.636573536046358	0.65661180216449	0
 731 | -1.34020898582428	-0.156201597712555	0
 732 | -1.51878819024828	0.527968476540175	0
 733 | -1.77879252556086	0.266148066465376	0
 734 | -0.569609932932805	-0.740700072336763	0
 735 | -0.583682047686648	-0.404739350043815	0
 736 | -1.42754379108698	-0.0906313379082068	0
 737 | -2.21809511615737	0.155088650023895	0
 738 | -1.05013860740822	-0.495668446249357	0
 739 | -0.932856822061464	-0.306795571298372	0
 740 | -1.26261751648756	-0.0619816058096187	0
 741 | -1.00205734934782	-0.0358183252313542	0
 742 | -0.865720261067755	-0.0872669562500311	0
 743 | -1.43764133271121	-0.354051511519117	0
 744 | 0.488928434480355	-0.00871149651427618	0
 745 | -0.359130061891404	-0.328027819467016	0
 746 | -1.38624571330246	-0.360767769119243	0
 747 | -1.71990215411562	0.0766234491083213	0
 748 | 1.22841056492106	-0.129478659809081	0
 749 | -0.753412975204029	-0.580094426674869	0
 750 | -0.315949114145464	0.255147011947452	0
 751 | -1.68016963483709	-0.793387231456783	0
 752 | -0.512130518447544	0.161641136719224	0
 753 | -1.31936073336425	0.17983000384599	0
 754 | -0.806013244523706	0.0949782859508813	0
 755 | -0.583309625373024	-0.823112721873236	0
 756 | 0.535837945241265	-0.434081705487123	0
 757 | -1.98870019180297	-0.0949028508716752	0
 758 | -2.64310110729009	-0.0134258101255887	0
 759 | -1.55212596968762	-0.239190173792675	0
 760 | 0.0745873958529104	0.307728883610949	0
 761 | -0.948163626814259	0.774529396351254	0
 762 | -1.2930221040489	0.791741788992659	0
 763 | -2.49582167755956	1.25367685545452	0
 764 | 0.255263744491205	0.494730861205604	0
 765 | -1.87893412788772	-0.324458823842342	0
 766 | -1.55338307695828	0.538178715767168	0
 767 | -0.548805884923596	0.513963907460375	0
 768 | -2.77178351773076	-0.502682235358601	0
 769 | -2.10996035199904	-0.64095631356305	0
 770 | -0.901054250177411	0.252748444364689	0
 771 | -2.60758381141409	-0.563945645298168	0
 772 | -0.865593976823225	0.178270671993677	0
 773 | -0.352319702818708	-0.473689843903947	0
 774 | -0.951430609891137	0.0610551843993262	0
 775 | -2.55219318454085	-0.0597760585924911	0
 776 | -0.0310514515610355	-0.605696375013528	0
 777 | 0.831758873167438	-0.60728597914708	0
 778 | -1.62963063937761	0.239251096360587	0
 779 | -0.162474865150271	-0.351552833394105	0
 780 | -0.698671765459251	-0.587199455439791	0
 781 | -2.59288793558677	0.58850837779576	0
 782 | -1.7661897813131	-0.292445254934933	0
 783 | -2.16977892634013	-0.266807482249593	0
 784 | -0.886099516692567	-0.404068795104717	0
 785 | -0.88624527933695	0.745433845518303	0
 786 | 0.390690887749745	-0.739353238398635	0
 787 | -2.45933094481917	-0.179739444512051	0
 788 | -0.404305597916272	-0.102566510062539	0
 789 | -2.14328717975115	-0.238358798094037	0
 790 | -0.099741805973122	0.220638539706736	0
 791 | -0.397583914273204	0.00416425232968286	0
 792 | -0.301438590398234	-0.550366373893303	0
 793 | -0.194350036353455	0.0550309326768173	0
 794 | -0.512631700148408	-0.151463071924876	0
 795 | -0.742299218155122	-1.06263243647816	0
 796 | -0.659612246859779	-0.378947916572908	0
 797 | -1.18286498616045	0.828979524978022	0
 798 | -1.00291159417728	-0.541466291330296	0
 799 | -1.5224784358033	0.702763107688537	0
 800 | -0.630844623111695	-0.318259767520662	0
 801 | 0.0470195559238111	-0.376647560227209	0
 802 | -1.13651918598233	-0.658996808105236	0
 803 | -1.08818065236591	0.795657364395117	0
 804 | -1.51478441901524	-0.0129223289284753	0
 805 | -0.418751105487177	-0.223946837570077	0
 806 | -0.247037419854112	-1.29159446865045	0
 807 | -1.55275903831119	0.399091356236516	0
 808 | 0.164410158494758	0.591662570691816	0
 809 | -1.36753582319214	0.274403396799225	0
 810 | 0.0498979703270894	-0.219775709944898	0
 811 | -1.21551702092193	0.0534520748398819	0
 812 | -0.20464990453838	1.05575568779506	0
 813 | -0.204895073404884	-0.594977242396899	0
 814 | -0.365056876439705	-1.01010809774212	0
 815 | -0.85092228594489	0.565012105655136	0
 816 | -1.39276783824424	-0.252887693834947	0
 817 | -1.49203861142788	-0.152670478285577	0
 818 | -1.6116595409278	-0.420873196627198	0
 819 | -0.180374116984691	-0.0774536643128701	0
 820 | -1.91573376847129	0.333793634251238	0
 821 | 0.136170642158111	0.578234650560165	0
 822 | 0.512488354125105	-0.364453604674437	0
 823 | -0.540694839198791	0.39516074310686	0
 824 | -1.55915164429066	-0.276171090633634	0
 825 | -0.246323047871912	-0.423315840833307	0
 826 | -2.03676254278709	-0.183935239628597	0
 827 | -0.758308215130326	-0.481267346632568	0
 828 | -0.517665105571216	0.0865913505927904	0
 829 | -1.74119123931553	0.32278551348713	0
 830 | -1.24954300779941	-0.0665143931514143	0
 831 | -1.05696592710097	0.201051296045855	0
 832 | -1.52754034617708	0.0454244923909537	0
 833 | 0.219622372100038	-0.174525036763087	0
 834 | -1.31841121822574	-0.0183386027900041	0
 835 | -0.232574250894206	-0.674846273673104	0
 836 | -0.598658811959627	0.363630369563425	0
 837 | -1.72659858536196	0.651325374063764	0
 838 | -0.879616968596377	0.202987879175916	0
 839 | 0.154888436735082	-0.0454485407485691	0
 840 | -0.868099312792456	-0.0419193634356992	0
 841 | -1.02580268929999	1.3034170453391	0
 842 | -1.03301010850956	0.858242520680424	0
 843 | -0.988418307708935	-0.194346247757719	0
 844 | -0.776812453164885	-0.034657533386177	0
 845 | -0.71891883903797	0.113547535180375	0
 846 | -1.186519630675	-0.36389086444896	0
 847 | -0.744352209840149	0.536227709639581	0
 848 | -1.85593974991559	-0.513919217823439	0
 849 | -2.26037569438581	-0.219214560401861	0
 850 | -3.4676434868714	-0.157112977186275	0
 851 | -1.69064809405278	0.00502385325662466	0
 852 | -1.96057950104747	0.275613198968991	0
 853 | -0.200715237083291	-0.13885893525076	0
 854 | -1.26280928690252	-0.403256295878004	0
 855 | -1.12690383855434	-0.0333376188386926	0
 856 | -2.05509158839284	0.625840579165476	0
 857 | 0.253374606232061	0.369383603220393	0
 858 | -0.495630711074975	0.249189580111485	0
 859 | -0.368728846822168	-0.429582517644948	0
 860 | -2.28728747884602	-0.87433800920226	0
 861 | -1.165449746259	-0.109165802985936	0
 862 | -0.58982889908297	-0.468092199903836	0
 863 | -0.727416363645933	0.890667857638712	0
 864 | -0.511115655814361	0.597478637648244	0
 865 | -1.25588267852291	0.14450262681337	0
 866 | -0.359602605142426	-0.881114535662492	0
 867 | -0.989697911686273	0.463591845058143	0
 868 | -1.35335798696937	-0.296108271230774	0
 869 | -0.771704560450914	1.3826815154726	0
 870 | -0.391976334026319	-0.512288632065009	0
 871 | -1.10940554126178	0.0994406799766734	0
 872 | 0.729052581720658	0.884836401081968	0
 873 | -0.628811012873177	0.521393408096077	0
 874 | -0.480686869531287	-0.065403883458919	0
 875 | -0.383516363760605	-0.888416148301969	0
 876 | -1.70063594291867	-0.673322001116872	0
 877 | -2.53355832905035	-0.210065855266664	0
 878 | -1.62202463791833	0.846321654168779	0
 879 | -1.39844353120313	0.384527436866878	0
 880 | -0.645931347438984	-0.676822028296522	0
 881 | -1.51040627231234	-1.06874656697346	0
 882 | -1.9824190473867	-0.486322888805057	0
 883 | -1.01537191025446	-0.00709221364903112	0
 884 | -0.436496970588004	1.08064541168857	0
 885 | -1.39282607578988	-0.392898771570298	0
 886 | -1.23721903036465	-0.335926080516723	0
 887 | -0.663662042575549	-0.468432143236444	0
 888 | -1.3684859732033	-1.131039622541	0
 889 | -1.53586479824212	-1.23149384006604	0
 890 | -1.71738552965941	0.0444336570373006	0
 891 | -1.31952852658322	0.552489281593351	0
 892 | -1.58841450271725	-0.477973926281008	0
 893 | -0.771341052216769	0.460205875755107	0
 894 | -0.937363886734026	1.14354573818139	0
 895 | -1.07012728496307	-0.109379901351478	0
 896 | -1.44882556562225	0.376914892070296	0
 897 | -1.03189751925489	-0.171771442403817	0
 898 | -0.301928568200167	-0.456605399625221	0
 899 | -0.555917603020932	0.411861392801597	0
 900 | -1.95299546989044	-0.11529177625497	0
 901 | -1.39286907332743	-0.464457957902799	0
 902 | -1.47483065938994	0.787620456052493	0
 903 | -1.42929963115274	0.289667448265046	0
 904 | -2.45713750863841	-0.0750569025108068	0
 905 | -0.769747411061183	-0.855266116814473	0
 906 | -1.68558494645461	-0.28149162450922	0
 907 | -0.673440591039196	-0.0546596056161434	0
 908 | -0.418228313316761	-0.312167839382337	0
 909 | -1.30714575955469	-0.935559850506985	0
 910 | -1.5970750011337	-0.00752825062136378	0
 911 | -2.68647609061493	0.256335754317623	0
 912 | -0.800032267315247	-0.338715661748399	0
 913 | -1.27819011822317	0.0811761866997217	0
 914 | -1.75854241161707	-0.478867385798184	0
 915 | -0.455417631331786	-0.505108871440998	0
 916 | -1.29660293752211	-0.335405858429638	0
 917 | -0.900027094680381	-0.734342771405311	0
 918 | -1.71778389498623	-0.113954779517267	0
 919 | -0.733440470472882	-0.860307548522907	0
 920 | -1.84604225289262	-0.164038643821841	0
 921 | -0.368870995724357	-0.578749901894443	0
 922 | -0.567548820650952	0.0146878520363519	0
 923 | -2.22148440504159	0.568044540964145	0
 924 | -1.78662751860504	0.806971593507674	0
 925 | -0.868899313425635	0.0147024941080341	0
 926 | -1.19698799998085	0.590680370227745	0
 927 | -0.40130902358952	0.0322990120685099	0
 928 | -2.32390731143343	0.157598975591262	0
 929 | -0.385425270971822	0.387093986058983	0
 930 | -1.2397316126701	0.0233734379370202	0
 931 | -2.27974959376869	-0.296121059549187	0
 932 | -1.49956804694025	0.810731757704651	0
 933 | -0.363599083449464	-0.219897135148595	0
 934 | -1.17207302590411	-0.518983193289928	0
 935 | 0.456712721754358	0.265917673264076	0
 936 | 0.442083493315476	-0.232808432840593	0
 937 | -0.27582834299859	0.852387444346183	0
 938 | -1.51431482860157	0.225800221285498	0
 939 | -3.12725028071092	-0.460407550033264	0
 940 | -0.914048780217227	-0.0663382335406132	0
 941 | -0.724819440644158	0.675946497933172	0
 942 | 0.905229008483119	0.0109197781815243	0
 943 | -1.38931184556227	0.0241368081608682	0
 944 | -1.27074231266529	0.490050194517221	0
 945 | -1.51087492614524	0.280125283646864	0
 946 | -0.564091031964395	0.250405328537557	0
 947 | -1.87352229944376	0.346878465180186	0
 948 | 0.274497165209224	-0.26490504974117	0
 949 | -1.2401108697225	0.514469175480009	0
 950 | -2.15800792578363	0.139985446552181	0
 951 | -0.365078710017099	-0.675839342789936	0
 952 | -1.30799604620896	0.396891265890849	0
 953 | -2.31125233369336	-0.369227384631367	0
 954 | -2.18524019847125	-0.842454342125667	0
 955 | -1.7362019482704	0.056868663875735	0
 956 | 0.186831010620888	-0.480643428059876	0
 957 | -1.08378757175638	0.736117539388377	0
 958 | -1.45080281883559	-0.818348801129604	0
 959 | -0.538386248329674	-0.35519558416166	0
 960 | -0.695167177393409	0.0935517862933743	0
 961 | -0.842143551840165	0.0281490025856951	0
 962 | -0.296903533234365	0.681910960891652	0
 963 | -0.715316955987152	0.722500295835593	0
 964 | 0.0372972810018959	0.547461681279367	0
 965 | -1.1441885685425	0.624378424878712	0
 966 | -0.608229848230109	0.417694367823113	0
 967 | -0.665458899051289	0.390752249721358	0
 968 | -0.147063167118097	-0.379110079034783	0
 969 | 0.106432747740739	-0.753300074019485	0
 970 | -0.777694049711968	0.0293946193110111	0
 971 | -0.702707631629676	-1.70817331411227	0
 972 | -1.40106469676448	0.490021631669798	0
 973 | -2.41356828053794	-0.730861175039163	0
 974 | -1.31805052510774	0.411096342895992	0
 975 | -2.36356299897417	0.818511805716514	0
 976 | -2.01274454430115	-0.418700986896372	0
 977 | -0.509024294119344	-0.189614014099415	0
 978 | -1.60998137204938	0.643294423277885	0
 979 | -1.06428858907298	-0.211343216848474	0
 980 | 0.3389663150943	0.161316138413044	0
 981 | -0.312809625696717	0.760010307418753	0
 982 | -0.748167180224074	0.568665497179553	0
 983 | 1.11703496565528	0.035993304319519	0
 984 | -0.367725254505071	-0.683119698649635	0
 985 | -0.72682660756747	0.325482863139218	0
 986 | -0.926943114261518	0.247329007843866	0
 987 | -0.762608866980205	0.403413116252543	0
 988 | -0.57901639480793	0.283069059481364	0
 989 | 0.245595350954563	-0.618802468415028	0
 990 | -0.749881318558903	0.773881874971387	0
 991 | -0.390489372999601	0.829646461697676	0
 992 | 0.317615859847871	-0.409265547148963	0
 993 | -3.05116466295164	0.730364746128395	0
 994 | -2.38437042313205	-0.0145063193012744	0
 995 | -0.655264931020618	-0.326521019684332	0
 996 | -0.202448273128766	0.341767702890859	0
 997 | -0.282061998443923	0.196197536631052	0
 998 | 0.0150269723530856	0.634925051580362	0
 999 | -1.63111124422064	0.150443474507196	0
1000 | -1.56183886287452	-1.15462534006559	0
1001 | 


--------------------------------------------------------------------------------