├── .gitignore ├── ASM └── Linux │ └── nn.asm ├── Bash ├── README.md ├── bash.png └── nn.sh ├── C++ └── nn.cpp ├── C └── C.cpp ├── CUDA └── nn.py ├── Cargo.lock ├── Cargo.toml ├── Dart └── nn.dart ├── Go └── nn.go ├── Java └── nn.java ├── Javascript └── nn.js ├── Julia └── nn.jl ├── LICENSE ├── Python └── nn.py ├── R └── nn.r ├── README.md ├── Rust └── main.rs ├── Typescript └── nn.ts └── p5js ├── ReadMe.md └── nn.js /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | -------------------------------------------------------------------------------- /ASM/Linux/nn.asm: -------------------------------------------------------------------------------- 1 | ; Neural Network asm x64 for Linux 2 | 3 | ; nasm -f elf64 nn.asm -o nn.o 4 | ; gcc nn.o -o nn -no-pie 5 | 6 | extern printf 7 | extern exit 8 | 9 | section .data 10 | inputs dq 1.0, 2.0, 3.0 11 | weights dq 0.5, 0.5, 0.5 12 | bias dq 1.0 13 | result dq 0.0 14 | cutoff dq 0.5 15 | fmt_out db "Output: %lf", 10, 0 16 | msg_yes db "Neuron fired!", 10, 0 17 | msg_no db "Neuron silent.", 10, 0 18 | 19 | section .text 20 | global main 21 | 22 | main: 23 | push rbp 24 | mov rbp, rsp 25 | 26 | xorpd xmm0, xmm0 27 | xorpd xmm1, xmm1 28 | 29 | mov rcx, 3 30 | .loop: 31 | movsd xmm1, [inputs + rcx*8 - 8] 32 | mulsd xmm1, [weights + rcx*8 - 8] 33 | addsd xmm0, xmm1 34 | loop .loop 35 | 36 | addsd xmm0, [bias] 37 | movsd [result], xmm0 38 | 39 | mov rdi, fmt_out 40 | movq xmm0, [result] 41 | mov rax, 1 42 | call printf 43 | 44 | movsd xmm1, [cutoff] 45 | ucomisd xmm0, xmm1 46 | jae .fire 47 | 48 | mov rdi, msg_no 49 | xor rax, rax 50 | call printf 51 | jmp .done 52 | 53 | .fire: 54 | mov rdi, msg_yes 55 | xor rax, rax 56 | call printf 57 | 58 | .done: 59 | xor rdi, rdi 60 | call exit 61 | 62 | mov rsp, rbp 63 | pop rbp 64 | ret 65 | -------------------------------------------------------------------------------- /Bash/README.md: -------------------------------------------------------------------------------- 1 | # Neural Network In Bash 2 | 3 | for anyone wondering what the output is 4 | 5 | ![bash](./bash.png) 6 | -------------------------------------------------------------------------------- /Bash/bash.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cneuralnetwork/Neural-Networks-in-every-Language/e866274677bf4f33c633b7ec5feb7838e79f31d2/Bash/bash.png -------------------------------------------------------------------------------- /Bash/nn.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | NUM_INPUTS=3 4 | NUM_HIDDEN=4 5 | NUM_OUTPUTS=2 6 | 7 | for ((i = 0; i < NUM_INPUTS * NUM_HIDDEN; i++)); do 8 | weights_ih[$i]=$(awk -v seed=$RANDOM 'BEGIN { srand(seed); printf("%.6f\n", rand() - 0.5)}') 9 | done 10 | 11 | for ((i = 0; i < NUM_HIDDEN * NUM_OUTPUTS; i++)); do 12 | weights_ho[$i]=$(awk -v seed=$RANDOM 'BEGIN { srand(seed); printf("%.6f\n", rand() - 0.5)}') 13 | done 14 | 15 | for ((i = 0; i < NUM_HIDDEN; i++)); do 16 | bias_h[$i]=$(awk -v seed=$RANDOM 'BEGIN { srand(seed); printf("%.6f\n", rand() - 0.5)}') 17 | done 18 | 19 | for ((i = 0; i < NUM_OUTPUTS; i++)); do 20 | bias_o[$i]=$(awk -v seed=$RANDOM 'BEGIN { srand(seed); printf("%.6f\n", rand() - 0.5)}') 21 | done 22 | 23 | sigmoid() { 24 | local x=$1 25 | echo "scale=10; 1 / (1 + e(-($x)))" | bc -l 26 | } 27 | 28 | relu() { 29 | local x=$1 30 | awk -v x="$x" 'BEGIN {print (x < 0) ? 0 : x}' 31 | } 32 | 33 | derivative_sigmoid() { 34 | local x=$1 35 | echo "scale=10; $x * (1 - $x)" | bc -l 36 | } 37 | 38 | derivative_relu() { 39 | local x=$1 40 | awk -v x="$x" 'BEGIN {print (x < 0) ? 0 : 1}' 41 | } 42 | 43 | forward_pass() { 44 | local inputs=("$@") 45 | local hidden_layer=() 46 | local output_layer=() 47 | 48 | for ((i = 0; i < NUM_HIDDEN; i++)); do 49 | local sum=0 50 | for ((j = 0; j < NUM_INPUTS; j++)); do 51 | sum=$(echo "scale=10; $sum + ${inputs[$j]} * ${weights_ih[$((i*NUM_INPUTS+j))]}" | bc -l) 52 | done 53 | sum=$(echo "scale=10; $sum + ${bias_h[$i]}" | bc -l) 54 | hidden_layer[$i]=$(relu $sum) 55 | done 56 | 57 | for ((i = 0; i < NUM_OUTPUTS; i++)); do 58 | local sum=0 59 | for ((j = 0; j < NUM_HIDDEN; j++)); do 60 | sum=$(echo "scale=10; $sum + ${hidden_layer[$j]} * ${weights_ho[$((i*NUM_HIDDEN+j))]}" | bc -l) 61 | done 62 | sum=$(echo "scale=10; $sum + ${bias_o[$i]}" | bc -l) 63 | output_layer[$i]=$(sigmoid $sum) 64 | done 65 | 66 | echo "${output_layer[@]}" 67 | } 68 | 69 | backward_pass() { 70 | local inputs=("$@") 71 | local targets=("${@: -2}") 72 | local hidden_layer=() 73 | local output_layer=() 74 | local error_gradients=() 75 | local hidden_gradients=() 76 | 77 | for ((i = 0; i < NUM_HIDDEN; i++)); do 78 | local sum=0 79 | for ((j = 0; j < NUM_INPUTS; j++)); do 80 | sum=$(echo "scale=10; $sum + ${inputs[$j]} * ${weights_ih[$((i*NUM_INPUTS+j))]}" | bc -l) 81 | done 82 | sum=$(echo "scale=10; $sum + ${bias_h[$i]}" | bc -l) 83 | hidden_layer[$i]=$(relu $sum) 84 | done 85 | 86 | for ((i = 0; i < NUM_OUTPUTS; i++)); do 87 | local sum=0 88 | for ((j = 0; j < NUM_HIDDEN; j++)); do 89 | sum=$(echo "scale=10; $sum + ${hidden_layer[$j]} * ${weights_ho[$((i*NUM_HIDDEN+j))]}" | bc -l) 90 | done 91 | sum=$(echo "scale=10; $sum + ${bias_o[$i]}" | bc -l) 92 | output_layer[$i]=$(sigmoid $sum) 93 | done 94 | 95 | for ((i = 0; i < NUM_OUTPUTS; i++)); do 96 | local error=$(echo "scale=10; ${targets[$i]} - ${output_layer[$i]}" | bc -l) 97 | error_gradients[$i]=$(echo "scale=10; $error * $(derivative_sigmoid ${output_layer[$i]})" | bc -l) 98 | done 99 | 100 | for ((i = 0; i < NUM_HIDDEN; i++)); do 101 | local sum=0 102 | for ((j = 0; j < NUM_OUTPUTS; j++)); do 103 | sum=$(echo "scale=10; $sum + ${error_gradients[$j]} * ${weights_ho[$((j*NUM_HIDDEN+i))]}" | bc -l) 104 | done 105 | hidden_gradients[$i]=$(echo "scale=10; $sum * $(derivative_relu ${hidden_layer[$i]})" | bc -l) 106 | done 107 | 108 | for ((i = 0; i < NUM_INPUTS*NUM_HIDDEN; i++)); do 109 | local weight=${weights_ih[$i]} 110 | local input=${inputs[$(($i%NUM_INPUTS))]} 111 | local gradient=${hidden_gradients[$(($i/NUM_INPUTS))]} 112 | weights_ih[$i]=$(echo "scale=10; $weight + 0.1 * $input * $gradient" | bc -l) 113 | done 114 | 115 | for ((i = 0; i < NUM_HIDDEN*NUM_OUTPUTS; i++)); do 116 | local weight=${weights_ho[$i]} 117 | local hidden=${hidden_layer[$(($i%NUM_HIDDEN))]} 118 | local gradient=${error_gradients[$(($i/NUM_HIDDEN))]} 119 | weights_ho[$i]=$(echo "scale=10; $weight + 0.1 * $hidden * $gradient" | bc -l) 120 | done 121 | 122 | for ((i = 0; i < NUM_HIDDEN; i++)); do 123 | bias_h[$i]=$(echo "scale=10; ${bias_h[$i]} + 0.1 * ${hidden_gradients[$i]}" | bc -l) 124 | done 125 | 126 | for ((i = 0; i < NUM_OUTPUTS; i++)); do 127 | bias_o[$i]=$(echo "scale=10; ${bias_o[$i]} + 0.1 * ${error_gradients[$i]}" | bc -l) 128 | done 129 | } 130 | 131 | # Training 132 | for ((epoch = 0; epoch < 100; epoch++)); do 133 | inputs=(1 2 3) 134 | targets=(0.5 0.8) 135 | backward_pass "${inputs[@]}" "${targets[@]}" 136 | done 137 | 138 | # Testing 139 | inputs=(1 2 3) 140 | outputs=($(forward_pass "${inputs[@]}")) 141 | echo "Output: ${outputs[@]}" -------------------------------------------------------------------------------- /C++/nn.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | #include 6 | 7 | // Activation function (sigmoid) 8 | double sigmoid(const double &x) { 9 | return 1 / (1 + std::exp(-x)); 10 | } 11 | 12 | // Derivative of sigmoid 13 | double sigmoid_derivative(const double &x) { 14 | return x * (1 - x); 15 | } 16 | 17 | // Softmax function for turning output vector in probabilities 18 | // Mostly used in multiclass classification problems 19 | std::vector softmax(const std::vector &z) { 20 | std::vector softmax_vector(z.size()); 21 | for (std::size_t i = 0; i < z.size(); i++) { 22 | softmax_vector[i] = std::exp(z[i]); 23 | } 24 | double denominator = std::accumulate(softmax_vector.begin(), softmax_vector.end(), 0); 25 | for (std::size_t i = 0; i < z.size(); i++) { 26 | softmax_vector[i] = softmax_vector[i] / denominator; // makes each value less than 1, and sum of all values equal to 1 27 | } 28 | return softmax_vector; 29 | } 30 | 31 | int main() { 32 | // Training data (XOR gate) 33 | std::vector> training_inputs = { 34 | {0, 0}, 35 | {0, 1}, 36 | {1, 0}, 37 | {1, 1} 38 | }; 39 | 40 | std::vector> training_outputs = { 41 | {0}, 42 | {1}, 43 | {1}, 44 | {0} 45 | }; 46 | 47 | // Network architecture (2 input, 2 hidden, 1 output) 48 | int num_inputs = 2; 49 | int num_hidden = 2; 50 | int num_outputs = 1; 51 | 52 | // Weights and biases (randomly initialized) 53 | std::vector> hidden_weights(num_inputs, std::vector(num_hidden)); 54 | std::vector hidden_biases(num_hidden); 55 | std::vector> output_weights(num_hidden, std::vector(num_outputs)); 56 | std::vector output_biases(num_outputs); 57 | 58 | // Training loop 59 | int num_epochs = 10000; 60 | double learning_rate = 0.1; 61 | for (int epoch = 0; epoch < num_epochs; ++epoch) { 62 | for (int i = 0; i < training_inputs.size(); ++i) { 63 | // Forward pass 64 | std::vector hidden_layer(num_hidden); 65 | for (int j = 0; j < num_hidden; ++j) { 66 | double weighted_sum = 0; 67 | for (int k = 0; k < num_inputs; ++k) { 68 | weighted_sum += training_inputs[i][k] * hidden_weights[k][j]; 69 | } 70 | hidden_layer[j] = sigmoid(weighted_sum + hidden_biases[j]); 71 | } 72 | 73 | std::vector output_layer(num_outputs); 74 | for (int j = 0; j < num_outputs; ++j) { 75 | double weighted_sum = 0; 76 | for (int k = 0; k < num_hidden; ++k) { 77 | weighted_sum += hidden_layer[k] * output_weights[k][j]; 78 | } 79 | output_layer[j] = sigmoid(weighted_sum + output_biases[j]); 80 | } 81 | 82 | // Backpropagation 83 | std::vector output_errors(num_outputs); 84 | for (int j = 0; j < num_outputs; ++j) { 85 | output_errors[j] = (training_outputs[i][j] - output_layer[j]) * sigmoid_derivative(output_layer[j]); 86 | } 87 | 88 | std::vector hidden_errors(num_hidden); 89 | for (int j = 0; j < num_hidden; ++j) { 90 | double error_sum = 0; 91 | for (int k = 0; k < num_outputs; ++k) { 92 | error_sum += output_errors[k] * output_weights[j][k]; 93 | } 94 | hidden_errors[j] = error_sum * sigmoid_derivative(hidden_layer[j]); 95 | } 96 | 97 | // Update weights and biases 98 | for (int j = 0; j < num_outputs; ++j) { 99 | for (int k = 0; k < num_hidden; ++k) { 100 | output_weights[k][j] += learning_rate * output_errors[j] * hidden_layer[k]; 101 | } 102 | output_biases[j] += learning_rate * output_errors[j]; 103 | } 104 | 105 | for (int j = 0; j < num_hidden; ++j) { 106 | for (int k = 0; k < num_inputs; ++k) { 107 | hidden_weights[k][j] += learning_rate * hidden_errors[j] * training_inputs[i][k]; 108 | } 109 | hidden_biases[j] += learning_rate * hidden_errors[j]; 110 | } 111 | } 112 | } 113 | 114 | // Test the network 115 | for (int i = 0; i < training_inputs.size(); ++i) { 116 | std::vector output_layer(num_outputs); 117 | // ... (forward pass to get output_layer) 118 | std::cout << "Input: " << training_inputs[i][0] << " " << training_inputs[i][1] 119 | << ", Output: " << output_layer[0] << std::endl; 120 | } 121 | 122 | return 0; 123 | } 124 | 125 | -------------------------------------------------------------------------------- /C/C.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | // Simple NN based on XOR function 6 | 7 | // Setting weights 8 | double init_weights() { return ((double)rand()) / ((double)RAND_MAX); } 9 | 10 | // Sigmoid function 11 | double sigmoid(double x) { return 1 / (1 + exp(-x)); } 12 | 13 | // Derivative of sigmoid 14 | double dsigmoid(double x) { return x * (1 - x); } 15 | 16 | // Shuffle function to help neural net learn faster 17 | void shuffle(int *array, size_t n) { 18 | if (n > 1) { 19 | size_t i; 20 | for (i = 0; i < n - 1; i++) { 21 | size_t j = i + rand() / (RAND_MAX / (n - i) + 1); 22 | int t = array[j]; 23 | array[j] = array[i]; 24 | array[i] = t; 25 | } 26 | } 27 | } 28 | 29 | #define numInputs 2 30 | #define numOutputs 1 31 | #define HiddenNodes 2 32 | #define TrainingSet 4 33 | 34 | int main(void) { 35 | const double lr = 0.1f; 36 | 37 | double hiddenLayer[HiddenNodes]; 38 | double outputLayer[numOutputs]; 39 | 40 | double hiddenLayerBias[HiddenNodes]; 41 | double outputLayerBias[numOutputs]; 42 | 43 | double hiddenWeights[numInputs][HiddenNodes]; 44 | double outputWeights[HiddenNodes][numOutputs]; 45 | 46 | double training_inputs[TrainingSet][numInputs] = { 47 | {0.0f, 0.0f}, 48 | {1.0f, 0.0f}, 49 | {0.0f, 1.0f}, 50 | {1.0f, 1.0f} 51 | }; 52 | 53 | double training_outputs[TrainingSet][numOutputs] = { 54 | {0.0f}, 55 | {1.0f}, 56 | {1.0f}, 57 | {0.0f} 58 | }; 59 | 60 | // Initialize weights and biases 61 | for (int i = 0; i < numInputs; i++) { 62 | for (int j = 0; j < HiddenNodes; j++) { 63 | hiddenWeights[i][j] = init_weights(); // Initializing weights to input layers 64 | } 65 | } 66 | for (int i = 0; i < HiddenNodes; i++) { 67 | for (int j = 0; j < numOutputs; j++) { 68 | outputWeights[i][j] = init_weights(); // Initializing output weights as random weights 69 | } 70 | } 71 | for (int i = 0; i < HiddenNodes; i++) { 72 | hiddenLayerBias[i] = init_weights(); // Initializing hidden layer biases 73 | } 74 | for (int i = 0; i < numOutputs; i++) { 75 | outputLayerBias[i] = init_weights(); // Initializing output biases 76 | } 77 | 78 | int trainingSetOrder[] = {0, 1, 2, 3}; 79 | int numberOfEpochs = 10000; 80 | 81 | // Open a file for writing results 82 | FILE *outputFile = fopen("nn_output.txt", "w"); 83 | if (!outputFile) { 84 | fprintf(stderr, "Could not open file for writing.\n"); 85 | return 1; 86 | } 87 | 88 | // Training the neural network for n number of epochs 89 | for (int epochs = 0; epochs < numberOfEpochs; epochs++) { 90 | shuffle(trainingSetOrder, TrainingSet); 91 | 92 | for (int x = 0; x < TrainingSet; x++) { 93 | int i = trainingSetOrder[x]; 94 | 95 | // Forward pass 96 | // Computing hidden layer activation 97 | for (int j = 0; j < HiddenNodes; j++) { 98 | double activation = hiddenLayerBias[j]; 99 | for (int k = 0; k < numInputs; k++) { 100 | activation += training_inputs[i][k] * hiddenWeights[k][j]; 101 | } 102 | hiddenLayer[j] = sigmoid(activation); 103 | } 104 | 105 | // Computing output layer activation 106 | for (int j = 0; j < numOutputs; j++) { 107 | double activation = outputLayerBias[j]; 108 | for (int k = 0; k < HiddenNodes; k++) { 109 | activation += hiddenLayer[k] * outputWeights[k][j]; 110 | } 111 | outputLayer[j] = sigmoid(activation); 112 | } 113 | 114 | // Print output and write to file 115 | char buffer[256]; 116 | snprintf(buffer, sizeof(buffer), "Input: (%g, %g) Output: %g Expected Output: %g\n", 117 | training_inputs[i][0], training_inputs[i][1], 118 | outputLayer[0], training_outputs[i][0]); 119 | fputs(buffer, outputFile); 120 | 121 | // Backpropagation 122 | // Change in output weights 123 | double deltaOutput[numOutputs]; 124 | for (int j = 0; j < numOutputs; j++) { 125 | double error = (training_outputs[i][j] - outputLayer[j]); 126 | deltaOutput[j] = error * dsigmoid(outputLayer[j]); 127 | } 128 | 129 | // Compute change in hidden weights 130 | double deltaHidden[HiddenNodes]; 131 | for (int j = 0; j < HiddenNodes; j++) { 132 | double error = 0.0f; 133 | for (int k = 0; k < numOutputs; k++) { 134 | error += deltaOutput[k] * outputWeights[j][k]; // Fixed indexing 135 | } 136 | deltaHidden[j] = error * dsigmoid(hiddenLayer[j]); 137 | } 138 | 139 | // Applying the changes in output weights 140 | for (int j = 0; j < numOutputs; j++) { 141 | outputLayerBias[j] += deltaOutput[j] * lr; 142 | for (int k = 0; k < HiddenNodes; k++) { 143 | outputWeights[k][j] += hiddenLayer[k] * deltaOutput[j] * lr; 144 | } 145 | } 146 | 147 | // Applying changes in hidden weights 148 | for (int j = 0; j < HiddenNodes; j++) { 149 | hiddenLayerBias[j] += deltaHidden[j] * lr; 150 | for (int k = 0; k < numInputs; k++) { 151 | hiddenWeights[k][j] += training_inputs[i][k] * deltaHidden[j] * lr; // Fixed indexing 152 | } 153 | } 154 | } 155 | } 156 | 157 | // Write final weights and biases to the output file 158 | fputs("Final Hidden Weights:\n", outputFile); 159 | for (int i = 0; i < numInputs; i++) { 160 | for (int j = 0; j < HiddenNodes; j++) { 161 | snprintf(buffer, sizeof(buffer), "%g ", hiddenWeights[i][j]); 162 | fputs(buffer, outputFile); 163 | } 164 | fputs("\n", outputFile); 165 | } 166 | 167 | fputs("Final Hidden Biases:\n", outputFile); 168 | for (int i = 0; i < HiddenNodes; i++) { 169 | snprintf(buffer, sizeof(buffer), "%g ", hiddenLayerBias[i]); 170 | fputs(buffer, outputFile); 171 | } 172 | fputs("\n", outputFile); 173 | 174 | fputs("Final Output Weights:\n", outputFile); 175 | for (int i = 0; i < HiddenNodes; i++) { 176 | for (int j = 0; j < numOutputs; j++) { 177 | snprintf(buffer, sizeof(buffer), "%g ", outputWeights[i][j]); 178 | fputs(buffer, outputFile); 179 | } 180 | fputs("\n", outputFile); 181 | } 182 | 183 | fputs("Final Output Biases:\n", outputFile); 184 | for (int i = 0; i < numOutputs; i++) { 185 | snprintf(buffer, sizeof(buffer), "%g ", outputLayerBias[i]); 186 | fputs(buffer, outputFile); 187 | } 188 | fputs("\n", outputFile); 189 | 190 | // Close the output file 191 | fclose(outputFile); 192 | return 0; 193 | } 194 | -------------------------------------------------------------------------------- /CUDA/nn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from matplotlib import pyplot as plt 4 | 5 | data=pd.read_csv('/kaggle/input/digit-recognizer/train.csv') 6 | 7 | data.head() 8 | 9 | data=np.array(data) 10 | m,n=data.shape 11 | np.random.shuffle(data) 12 | 13 | data_dev=data[0:1000].T 14 | Y_dev=data_dev[0] 15 | X_dev=data_dev[1:n] 16 | X_dev=X_dev/255 17 | 18 | data_train=data[1000:m].T 19 | Y_train=data_train[0] 20 | X_train=data_train[1:n] 21 | X_train=X_train/255 22 | _,m_train=X_train.shape 23 | 24 | def init_params(): 25 | w1=np.random.rand(10,784)-0.5 26 | b1=np.random.rand(10,1)-0.5 27 | w2=np.random.rand(10,10)-0.5 28 | b2=np.random.rand(10,1)-0.5 29 | return w1,b1,w2,b2 30 | 31 | def ReLU(z): 32 | return np.maximum(0,z) 33 | 34 | def softmax(Z): 35 | A = np.exp(Z) / sum(np.exp(Z)) 36 | return A 37 | 38 | def one_hot(z): 39 | one_hot_z=np.zeros((z.size,z.max()+1)) 40 | one_hot_z[np.arange(z.size),z]=1 41 | one_hot_z=one_hot_z.T 42 | return one_hot_z 43 | 44 | def derivative_ReLU(z): 45 | return z>0 46 | 47 | def forward_prop(w1,b1,w2,b2,x): 48 | z1=w1.dot(x)+b1 49 | a1=ReLU(z1) 50 | z2=w2.dot(a1)+b2 51 | a2=softmax(z2) 52 | return z1,a1,z2,a2 53 | 54 | def back_prop(z1,a1,z2,a2,w2,x,y): 55 | m=y.size 56 | one_hot_Y=one_hot(y) 57 | dz2=a2-one_hot_Y 58 | dw2=1/m * dz2.dot(a1.T) 59 | db2=1/m * np.sum(dz2) 60 | dz1 = w2.T.dot(dz2)*derivative_ReLU(z1) 61 | dw1=1/m * dz1.dot(x.T) 62 | db1=1/m * np.sum(dz1) 63 | return dw1,db1,dw2,db2 64 | 65 | def update_params(w1,b1,w2,b2,dw1,db1,dw2,db2,alpha): 66 | w1=w1-alpha*dw1 67 | w2=w2-alpha*dw2 68 | b1=b1-alpha*db1 69 | b2=b2-alpha*db2 70 | return w1,b1,w2,b2 71 | 72 | def get_predictions(a2): 73 | return np.argmax(a2,0) 74 | 75 | def get_accuracy(predictions,y): 76 | print(predictions,y) 77 | return np.sum(predictions==y)/y.size 78 | 79 | def gradient_descent(x,y,it,alpha): 80 | w1,b1,w2,b2=init_params() 81 | for i in range(it): 82 | z1,a1,z2,a2=forward_prop(w1,b1,w2,b2,x) 83 | dw1,db1,dw2,db2=back_prop(z1,a1,z2,a2,w2,x,y) 84 | w1,b1,w2,b2=update_params(w1,b1,w2,b2,dw1,db1,dw2,db2,alpha) 85 | if i%10==0: 86 | print("Iteration",i) 87 | print("Accuracy", get_accuracy(get_predictions(a2),y)) 88 | return w1,b1,w2,b2 89 | 90 | w1,b1,w2,b2=gradient_descent(X_train,Y_train,500,0.1) 91 | 92 | def make_predictions(x,w1,b1,w2,b2): 93 | _,_,_,a2=forward_prop(w1,b1,w2,b2,x) 94 | predictions=get_predictions(a2) 95 | return predictions 96 | 97 | def test_predictions(index,w1,b1,w2,b2): 98 | current_image=X_train[:,index,None] 99 | prediction=make_predictions(X_train[:,index,None],w1,b1,w2,b2) 100 | label=Y_train[index] 101 | print("Prediction : ",prediction) 102 | print("Label : ",label) 103 | current_image=current_image.reshape((28,28))*255 104 | plt.gray() 105 | plt.imshow(current_image,interpolation='nearest') 106 | plt.show() 107 | 108 | dev_predictions=make_predictions(X_dev,w1,b1,w2,b2) 109 | print("Accuracy - ",get_accuracy(dev_predictions,Y_dev)) 110 | -------------------------------------------------------------------------------- /Cargo.lock: -------------------------------------------------------------------------------- 1 | # This file is automatically @generated by Cargo. 2 | # It is not intended for manual editing. 3 | version = 3 4 | 5 | [[package]] 6 | name = "Neural-Networks-in-every-Language" 7 | version = "0.1.0" 8 | dependencies = [ 9 | "rand", 10 | ] 11 | 12 | [[package]] 13 | name = "byteorder" 14 | version = "1.5.0" 15 | source = "registry+https://github.com/rust-lang/crates.io-index" 16 | checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" 17 | 18 | [[package]] 19 | name = "cfg-if" 20 | version = "1.0.0" 21 | source = "registry+https://github.com/rust-lang/crates.io-index" 22 | checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" 23 | 24 | [[package]] 25 | name = "getrandom" 26 | version = "0.2.15" 27 | source = "registry+https://github.com/rust-lang/crates.io-index" 28 | checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7" 29 | dependencies = [ 30 | "cfg-if", 31 | "libc", 32 | "wasi", 33 | ] 34 | 35 | [[package]] 36 | name = "libc" 37 | version = "0.2.159" 38 | source = "registry+https://github.com/rust-lang/crates.io-index" 39 | checksum = "561d97a539a36e26a9a5fad1ea11a3039a67714694aaa379433e580854bc3dc5" 40 | 41 | [[package]] 42 | name = "ppv-lite86" 43 | version = "0.2.20" 44 | source = "registry+https://github.com/rust-lang/crates.io-index" 45 | checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04" 46 | dependencies = [ 47 | "zerocopy", 48 | ] 49 | 50 | [[package]] 51 | name = "proc-macro2" 52 | version = "1.0.86" 53 | source = "registry+https://github.com/rust-lang/crates.io-index" 54 | checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" 55 | dependencies = [ 56 | "unicode-ident", 57 | ] 58 | 59 | [[package]] 60 | name = "quote" 61 | version = "1.0.37" 62 | source = "registry+https://github.com/rust-lang/crates.io-index" 63 | checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" 64 | dependencies = [ 65 | "proc-macro2", 66 | ] 67 | 68 | [[package]] 69 | name = "rand" 70 | version = "0.8.5" 71 | source = "registry+https://github.com/rust-lang/crates.io-index" 72 | checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404" 73 | dependencies = [ 74 | "libc", 75 | "rand_chacha", 76 | "rand_core", 77 | ] 78 | 79 | [[package]] 80 | name = "rand_chacha" 81 | version = "0.3.1" 82 | source = "registry+https://github.com/rust-lang/crates.io-index" 83 | checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88" 84 | dependencies = [ 85 | "ppv-lite86", 86 | "rand_core", 87 | ] 88 | 89 | [[package]] 90 | name = "rand_core" 91 | version = "0.6.4" 92 | source = "registry+https://github.com/rust-lang/crates.io-index" 93 | checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c" 94 | dependencies = [ 95 | "getrandom", 96 | ] 97 | 98 | [[package]] 99 | name = "syn" 100 | version = "2.0.79" 101 | source = "registry+https://github.com/rust-lang/crates.io-index" 102 | checksum = "89132cd0bf050864e1d38dc3bbc07a0eb8e7530af26344d3d2bbbef83499f590" 103 | dependencies = [ 104 | "proc-macro2", 105 | "quote", 106 | "unicode-ident", 107 | ] 108 | 109 | [[package]] 110 | name = "unicode-ident" 111 | version = "1.0.13" 112 | source = "registry+https://github.com/rust-lang/crates.io-index" 113 | checksum = "e91b56cd4cadaeb79bbf1a5645f6b4f8dc5bde8834ad5894a8db35fda9efa1fe" 114 | 115 | [[package]] 116 | name = "wasi" 117 | version = "0.11.0+wasi-snapshot-preview1" 118 | source = "registry+https://github.com/rust-lang/crates.io-index" 119 | checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" 120 | 121 | [[package]] 122 | name = "zerocopy" 123 | version = "0.7.35" 124 | source = "registry+https://github.com/rust-lang/crates.io-index" 125 | checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0" 126 | dependencies = [ 127 | "byteorder", 128 | "zerocopy-derive", 129 | ] 130 | 131 | [[package]] 132 | name = "zerocopy-derive" 133 | version = "0.7.35" 134 | source = "registry+https://github.com/rust-lang/crates.io-index" 135 | checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" 136 | dependencies = [ 137 | "proc-macro2", 138 | "quote", 139 | "syn", 140 | ] 141 | -------------------------------------------------------------------------------- /Cargo.toml: -------------------------------------------------------------------------------- 1 | [package] 2 | name = "Neural-Networks-in-every-Language" 3 | version = "0.1.0" 4 | edition = "2021" 5 | 6 | [dependencies] 7 | rand = "0.8.5" -------------------------------------------------------------------------------- /Dart/nn.dart: -------------------------------------------------------------------------------- 1 | import 'dart:math'; 2 | 3 | double sigmoid(double x) { 4 | return 1.0 / (1.0 + exp(-x)); 5 | } 6 | 7 | double sigmoidDerivative(double x) { 8 | double s = sigmoid(x); 9 | return s * (1.0 - s); 10 | } 11 | 12 | class Matrix { 13 | int rows, cols; 14 | late List> data; 15 | 16 | Matrix(this.rows, this.cols) { 17 | data = List.generate(rows, (_) => List.filled(cols, 0.0)); 18 | } 19 | 20 | void randomize(Random rng) { 21 | for (int i = 0; i < rows; i++) { 22 | for (int j = 0; j < cols; j++) { 23 | data[i][j] = rng.nextDouble() * 2 - 1; 24 | } 25 | } 26 | } 27 | 28 | Matrix transpose() { 29 | Matrix result = Matrix(cols, rows); 30 | for (int i = 0; i < rows; i++) { 31 | for (int j = 0; j < cols; j++) { 32 | result.data[j][i] = data[i][j]; 33 | } 34 | } 35 | return result; 36 | } 37 | 38 | Matrix multiply(Matrix other) { 39 | if (cols != other.rows) { 40 | throw Exception('Incompatible matrix dimensions'); 41 | } 42 | Matrix result = Matrix(rows, other.cols); 43 | for (int i = 0; i < rows; i++) { 44 | for (int j = 0; j < other.cols; j++) { 45 | double sum = 0.0; 46 | for (int k = 0; k < cols; k++) { 47 | sum += data[i][k] * other.data[k][j]; 48 | } 49 | result.data[i][j] = sum; 50 | } 51 | } 52 | return result; 53 | } 54 | 55 | void add(Matrix other) { 56 | if (rows != other.rows || cols != other.cols) { 57 | throw Exception('Incompatible matrix dimensions'); 58 | } 59 | for (int i = 0; i < rows; i++) { 60 | for (int j = 0; j < cols; j++) { 61 | data[i][j] += other.data[i][j]; 62 | } 63 | } 64 | } 65 | 66 | void subtract(Matrix other) { 67 | if (rows != other.rows || cols != other.cols) { 68 | throw Exception('Incompatible matrix dimensions'); 69 | } 70 | for (int i = 0; i < rows; i++) { 71 | for (int j = 0; j < cols; j++) { 72 | data[i][j] -= other.data[i][j]; 73 | } 74 | } 75 | } 76 | 77 | void multiplyScalar(double scalar) { 78 | for (int i = 0; i < rows; i++) { 79 | for (int j = 0; j < cols; j++) { 80 | data[i][j] *= scalar; 81 | } 82 | } 83 | } 84 | 85 | void apply(double Function(double) f) { 86 | for (int i = 0; i < rows; i++) { 87 | for (int j = 0; j < cols; j++) { 88 | data[i][j] = f(data[i][j]); 89 | } 90 | } 91 | } 92 | } 93 | 94 | class NeuralNetwork { 95 | int inputSize, hiddenSize, outputSize; 96 | double learningRate; 97 | late Matrix weightsIH, weightsHO, biasH, biasO; 98 | 99 | NeuralNetwork( 100 | this.inputSize, this.hiddenSize, this.outputSize, this.learningRate) { 101 | weightsIH = Matrix(hiddenSize, inputSize); 102 | weightsHO = Matrix(outputSize, hiddenSize); 103 | biasH = Matrix(hiddenSize, 1); 104 | biasO = Matrix(outputSize, 1); 105 | } 106 | 107 | List forward(List inputArray) { 108 | Matrix inputs = Matrix(inputArray.length, 1); 109 | for (int i = 0; i < inputArray.length; i++) { 110 | inputs.data[i][0] = inputArray[i]; 111 | } 112 | 113 | Matrix hidden = weightsIH.multiply(inputs); 114 | hidden.add(biasH); 115 | hidden.apply(sigmoid); 116 | 117 | Matrix outputs = weightsHO.multiply(hidden); 118 | outputs.add(biasO); 119 | outputs.apply(sigmoid); 120 | 121 | return [outputs, hidden, inputs]; 122 | } 123 | 124 | void train(List inputArray, List targetArray) { 125 | List results = forward(inputArray); 126 | Matrix outputs = results[0], hidden = results[1], inputs = results[2]; 127 | 128 | Matrix targets = Matrix(targetArray.length, 1); 129 | for (int i = 0; i < targetArray.length; i++) { 130 | targets.data[i][0] = targetArray[i]; 131 | } 132 | 133 | Matrix outputErrors = Matrix(outputSize, 1); 134 | for (int i = 0; i < outputSize; i++) { 135 | outputErrors.data[i][0] = targets.data[i][0] - outputs.data[i][0]; 136 | } 137 | 138 | Matrix hiddenErrors = weightsHO.transpose().multiply(outputErrors); 139 | 140 | Matrix gradientsO = Matrix(outputSize, 1); 141 | for (int i = 0; i < outputSize; i++) { 142 | gradientsO.data[i][0] = 143 | outputErrors.data[i][0] * sigmoidDerivative(outputs.data[i][0]); 144 | } 145 | gradientsO.multiplyScalar(learningRate); 146 | 147 | Matrix weightsHODeltas = gradientsO.multiply(hidden.transpose()); 148 | weightsHO.add(weightsHODeltas); 149 | biasO.add(gradientsO); 150 | 151 | Matrix gradientsH = Matrix(hiddenSize, 1); 152 | for (int i = 0; i < hiddenSize; i++) { 153 | gradientsH.data[i][0] = 154 | hiddenErrors.data[i][0] * sigmoidDerivative(hidden.data[i][0]); 155 | } 156 | gradientsH.multiplyScalar(learningRate); 157 | 158 | Matrix weightsIHDeltas = gradientsH.multiply(inputs.transpose()); 159 | weightsIH.add(weightsIHDeltas); 160 | biasH.add(gradientsH); 161 | } 162 | } 163 | 164 | void main() { 165 | Random rng = Random(); 166 | 167 | NeuralNetwork nn = NeuralNetwork(2, 4, 1, 0.1); 168 | 169 | nn.weightsIH.randomize(rng); 170 | nn.weightsHO.randomize(rng); 171 | nn.biasH.randomize(rng); 172 | nn.biasO.randomize(rng); 173 | 174 | List> xorInputs = [ 175 | [0, 0], 176 | [0, 1], 177 | [1, 0], 178 | [1, 1], 179 | ]; 180 | List xorOutputs = [0, 1, 1, 0]; 181 | 182 | int epochs = 100000; 183 | for (int i = 0; i < epochs; i++) { 184 | for (int j = 0; j < xorInputs.length; j++) { 185 | nn.train(xorInputs[j], [xorOutputs[j]]); 186 | } 187 | if (i % 10000 == 0) { 188 | print('$i epochs done'); 189 | } 190 | } 191 | 192 | print('Testing XOR outputs:'); 193 | for (List input in xorInputs) { 194 | Matrix result = nn.forward(input)[0]; 195 | print( 196 | 'Input: ${input[0]} ${input[1]}, Output: ${result.data[0][0].toStringAsFixed(4)}'); 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /Go/nn.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "math/rand" 7 | "time" 8 | ) 9 | 10 | func sigmoid(x float64) float64 { 11 | return 1.0 / (1.0 + math.Exp(-x)) 12 | } 13 | 14 | func sigmoidDerivative(x float64) float64 { 15 | s := sigmoid(x) 16 | return s * (1.0 - s) 17 | } 18 | 19 | type Matrix struct { 20 | Rows, Cols int 21 | Data [][]float64 22 | } 23 | 24 | func NewMatrix(rows, cols int) *Matrix { 25 | m := &Matrix{ 26 | Rows: rows, 27 | Cols: cols, 28 | Data: make([][]float64, rows), 29 | } 30 | for i := range m.Data { 31 | m.Data[i] = make([]float64, cols) 32 | } 33 | return m 34 | } 35 | 36 | func (m *Matrix) Randomize(rng *rand.Rand) { 37 | for i := 0; i < m.Rows; i++ { 38 | for j := 0; j < m.Cols; j++ { 39 | m.Data[i][j] = rng.Float64()*2 - 1 40 | } 41 | } 42 | } 43 | 44 | func (m *Matrix) Transpose() *Matrix { 45 | result := NewMatrix(m.Cols, m.Rows) 46 | for i := 0; i < m.Rows; i++ { 47 | for j := 0; j < m.Cols; j++ { 48 | result.Data[j][i] = m.Data[i][j] 49 | } 50 | } 51 | return result 52 | } 53 | 54 | func (m *Matrix) Multiply(other *Matrix) *Matrix { 55 | if m.Cols != other.Rows { 56 | panic("Incompatible matrix dimensions") 57 | } 58 | result := NewMatrix(m.Rows, other.Cols) 59 | for i := 0; i < m.Rows; i++ { 60 | for j := 0; j < other.Cols; j++ { 61 | sum := 0.0 62 | for k := 0; k < m.Cols; k++ { 63 | sum += m.Data[i][k] * other.Data[k][j] 64 | } 65 | result.Data[i][j] = sum 66 | } 67 | } 68 | return result 69 | } 70 | 71 | func (m *Matrix) Add(other *Matrix) { 72 | if m.Rows != other.Rows || m.Cols != other.Cols { 73 | panic("Incompatible matrix dimensions") 74 | } 75 | for i := 0; i < m.Rows; i++ { 76 | for j := 0; j < m.Cols; j++ { 77 | m.Data[i][j] += other.Data[i][j] 78 | } 79 | } 80 | } 81 | 82 | func (m *Matrix) Subtract(other *Matrix) { 83 | if m.Rows != other.Rows || m.Cols != other.Cols { 84 | panic("Incompatible matrix dimensions") 85 | } 86 | for i := 0; i < m.Rows; i++ { 87 | for j := 0; j < m.Cols; j++ { 88 | m.Data[i][j] -= other.Data[i][j] 89 | } 90 | } 91 | } 92 | 93 | func (m *Matrix) MultiplyScalar(scalar float64) { 94 | for i := 0; i < m.Rows; i++ { 95 | for j := 0; j < m.Cols; j++ { 96 | m.Data[i][j] *= scalar 97 | } 98 | } 99 | } 100 | 101 | func (m *Matrix) Apply(f func(float64) float64) { 102 | for i := 0; i < m.Rows; i++ { 103 | for j := 0; j < m.Cols; j++ { 104 | m.Data[i][j] = f(m.Data[i][j]) 105 | } 106 | } 107 | } 108 | 109 | type NeuralNetwork struct { 110 | InputSize int 111 | HiddenSize int 112 | OutputSize int 113 | LearningRate float64 114 | WeightsIH *Matrix 115 | WeightsHO *Matrix 116 | BiasH *Matrix 117 | BiasO *Matrix 118 | } 119 | 120 | func NewNeuralNetwork(inputSize, hiddenSize, outputSize int, learningRate float64) *NeuralNetwork { 121 | nn := &NeuralNetwork{ 122 | InputSize: inputSize, 123 | HiddenSize: hiddenSize, 124 | OutputSize: outputSize, 125 | LearningRate: learningRate, 126 | WeightsIH: NewMatrix(hiddenSize, inputSize), 127 | WeightsHO: NewMatrix(outputSize, hiddenSize), 128 | BiasH: NewMatrix(hiddenSize, 1), 129 | BiasO: NewMatrix(outputSize, 1), 130 | } 131 | return nn 132 | } 133 | 134 | func (nn *NeuralNetwork) Forward(inputArray []float64) []*Matrix { 135 | inputs := NewMatrix(len(inputArray), 1) 136 | for i, v := range inputArray { 137 | inputs.Data[i][0] = v 138 | } 139 | 140 | hidden := nn.WeightsIH.Multiply(inputs) 141 | hidden.Add(nn.BiasH) 142 | hidden.Apply(sigmoid) 143 | 144 | outputs := nn.WeightsHO.Multiply(hidden) 145 | outputs.Add(nn.BiasO) 146 | outputs.Apply(sigmoid) 147 | 148 | return []*Matrix{outputs, hidden, inputs} 149 | } 150 | 151 | func (nn *NeuralNetwork) Train(inputArray, targetArray []float64) { 152 | results := nn.Forward(inputArray) 153 | outputs, hidden, inputs := results[0], results[1], results[2] 154 | 155 | targets := NewMatrix(len(targetArray), 1) 156 | for i, v := range targetArray { 157 | targets.Data[i][0] = v 158 | } 159 | 160 | outputErrors := NewMatrix(nn.OutputSize, 1) 161 | for i := 0; i < nn.OutputSize; i++ { 162 | outputErrors.Data[i][0] = targets.Data[i][0] - outputs.Data[i][0] 163 | } 164 | 165 | hiddenErrors := nn.WeightsHO.Transpose().Multiply(outputErrors) 166 | 167 | gradientsO := NewMatrix(nn.OutputSize, 1) 168 | for i := 0; i < nn.OutputSize; i++ { 169 | gradientsO.Data[i][0] = outputErrors.Data[i][0] * sigmoidDerivative(outputs.Data[i][0]) 170 | } 171 | gradientsO.MultiplyScalar(nn.LearningRate) 172 | 173 | weightsHODeltas := gradientsO.Multiply(hidden.Transpose()) 174 | nn.WeightsHO.Add(weightsHODeltas) 175 | nn.BiasO.Add(gradientsO) 176 | 177 | gradientsH := NewMatrix(nn.HiddenSize, 1) 178 | for i := 0; i < nn.HiddenSize; i++ { 179 | gradientsH.Data[i][0] = hiddenErrors.Data[i][0] * sigmoidDerivative(hidden.Data[i][0]) 180 | } 181 | gradientsH.MultiplyScalar(nn.LearningRate) 182 | 183 | weightsIHDeltas := gradientsH.Multiply(inputs.Transpose()) 184 | nn.WeightsIH.Add(weightsIHDeltas) 185 | nn.BiasH.Add(gradientsH) 186 | } 187 | 188 | func main() { 189 | rng := rand.New(rand.NewSource(time.Now().UnixNano())) 190 | 191 | nn := NewNeuralNetwork(2, 4, 1, 0.1) 192 | 193 | nn.WeightsIH.Randomize(rng) 194 | nn.WeightsHO.Randomize(rng) 195 | nn.BiasH.Randomize(rng) 196 | nn.BiasO.Randomize(rng) 197 | 198 | xorInputs := [][]float64{ 199 | {0, 0}, 200 | {0, 1}, 201 | {1, 0}, 202 | {1, 1}, 203 | } 204 | xorOutputs := []float64{0, 1, 1, 0} 205 | 206 | epochs := 100000 207 | for i := 0; i < epochs; i++ { 208 | for j := range xorInputs { 209 | nn.Train(xorInputs[j], []float64{xorOutputs[j]}) 210 | } 211 | if i%10000 == 0 { 212 | fmt.Printf("%d epochs done\n", i) 213 | } 214 | } 215 | 216 | fmt.Println("Testing XOR outputs:") 217 | for _, input := range xorInputs { 218 | result := nn.Forward(input)[0] 219 | fmt.Printf("Input: %v %v, Output: %.4f\n", input[0], input[1], result.Data[0][0]) 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /Java/nn.java: -------------------------------------------------------------------------------- 1 | import java.util.ArrayList; 2 | import java.util.List; 3 | import java.util.Random; 4 | 5 | public class NeuralNetwork { 6 | private static double sigmoid(double x) { 7 | return 1 / (1 + Math.exp(-x)); 8 | } 9 | 10 | private static double sigmoid_derivative(double x) { 11 | return x * (1 - x); 12 | } 13 | 14 | public static void main(String[] args) { 15 | List> training_inputs = new ArrayList<>(); 16 | training_inputs.add(List.of(0.0, 0.0)); 17 | training_inputs.add(List.of(0.0, 1.0)); 18 | training_inputs.add(List.of(1.0, 0.0)); 19 | training_inputs.add(List.of(1.0, 1.0)); 20 | 21 | List> training_outputs = new ArrayList<>(); 22 | training_outputs.add(List.of(0.0)); 23 | training_outputs.add(List.of(1.0)); 24 | training_outputs.add(List.of(1.0)); 25 | training_outputs.add(List.of(0.0)); 26 | 27 | int num_inputs = 2; 28 | int num_hidden = 2; 29 | int num_outputs = 1; 30 | 31 | Random random = new Random(); 32 | List> hidden_weights = new ArrayList<>(); 33 | List hidden_biases = new ArrayList<>(); 34 | List> output_weights = new ArrayList<>(); 35 | List output_biases = new ArrayList<>(); 36 | 37 | for (int i = 0; i < num_inputs; i++) { 38 | List row = new ArrayList<>(); 39 | for (int j = 0; j < num_hidden; j++) { 40 | row.add(random.nextDouble() * 2 - 1); 41 | } 42 | hidden_weights.add(row); 43 | } 44 | 45 | for (int i = 0; i < num_hidden; i++) { 46 | hidden_biases.add(random.nextDouble() * 2 - 1); 47 | List row = new ArrayList<>(); 48 | for (int j = 0; j < num_outputs; j++) { 49 | row.add(random.nextDouble() * 2 - 1); 50 | } 51 | output_weights.add(row); 52 | } 53 | 54 | for (int i = 0; i < num_outputs; i++) { 55 | output_biases.add(random.nextDouble() * 2 - 1); 56 | } 57 | 58 | int num_epochs = 10000; 59 | double learning_rate = 0.1; 60 | 61 | for (int epoch = 0; epoch < num_epochs; ++epoch) { 62 | for (int i = 0; i < training_inputs.size(); ++i) { 63 | List hidden_layer = new ArrayList<>(); 64 | for (int j = 0; j < num_hidden; ++j) { 65 | double weighted_sum = 0; 66 | for (int k = 0; k < num_inputs; ++k) { 67 | weighted_sum += training_inputs.get(i).get(k) * hidden_weights.get(k).get(j); 68 | } 69 | hidden_layer.add(sigmoid(weighted_sum + hidden_biases.get(j))); 70 | } 71 | 72 | List output_layer = new ArrayList<>(); 73 | for (int j = 0; j < num_outputs; ++j) { 74 | double weighted_sum = 0; 75 | for (int k = 0; k < num_hidden; ++k) { 76 | weighted_sum += hidden_layer.get(k) * output_weights.get(k).get(j); 77 | } 78 | output_layer.add(sigmoid(weighted_sum + output_biases.get(j))); 79 | } 80 | 81 | List output_errors = new ArrayList<>(); 82 | for (int j = 0; j < num_outputs; ++j) { 83 | output_errors.add((training_outputs.get(i).get(j) - output_layer.get(j)) * sigmoid_derivative(output_layer.get(j))); 84 | } 85 | 86 | List hidden_errors = new ArrayList<>(); 87 | for (int j = 0; j < num_hidden; ++j) { 88 | double error_sum = 0; 89 | for (int k = 0; k < num_outputs; ++k) { 90 | error_sum += output_errors.get(k) * output_weights.get(j).get(k); 91 | } 92 | hidden_errors.add(error_sum * sigmoid_derivative(hidden_layer.get(j))); 93 | } 94 | 95 | for (int j = 0; j < num_outputs; ++j) { 96 | for (int k = 0; k < num_hidden; ++k) { 97 | output_weights.get(k).set(j, output_weights.get(k).get(j) + learning_rate * output_errors.get(j) * hidden_layer.get(k)); 98 | } 99 | output_biases.set(j, output_biases.get(j) + learning_rate * output_errors.get(j)); 100 | } 101 | 102 | for (int j = 0; j < num_hidden; ++j) { 103 | for (int k = 0; k < num_inputs; ++k) { 104 | hidden_weights.get(k).set(j, hidden_weights.get(k).get(j) + learning_rate * hidden_errors.get(j) * training_inputs.get(i).get(k)); 105 | } 106 | hidden_biases.set(j, hidden_biases.get(j) + learning_rate * hidden_errors.get(j)); 107 | } 108 | } 109 | } 110 | 111 | for (int i = 0; i < training_inputs.size(); ++i) { 112 | List hidden_layer = new ArrayList<>(); 113 | for (int j = 0; j < num_hidden; ++j) { 114 | double weighted_sum = 0; 115 | for (int k = 0; k < num_inputs; ++k) { 116 | weighted_sum += training_inputs.get(i).get(k) * hidden_weights.get(k).get(j); 117 | } 118 | hidden_layer.add(sigmoid(weighted_sum + hidden_biases.get(j))); 119 | } 120 | 121 | List output_layer = new ArrayList<>(); 122 | for (int j = 0; j < num_outputs; ++j) { 123 | double weighted_sum = 0; 124 | for (int k = 0; k < num_hidden; ++k) { 125 | weighted_sum += hidden_layer.get(k) * output_weights.get(k).get(j); 126 | } 127 | output_layer.add(sigmoid(weighted_sum + output_biases.get(j))); 128 | } 129 | 130 | System.out.printf("Input: %.0f %.0f, Output: %.4f%n", 131 | training_inputs.get(i).get(0), 132 | training_inputs.get(i).get(1), 133 | output_layer.get(0)); 134 | } 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /Javascript/nn.js: -------------------------------------------------------------------------------- 1 | const epsilon = 1e-15; 2 | const sigmoid = x => 1 / (1 + Math.exp(-x)); 3 | const sigmoidDerivative = x => Math.max(epsilon, x * (1 - x)); 4 | 5 | const create = (rows, cols) => Array.from( 6 | { length: rows }, 7 | () => Array.from({ length: cols }, () => (Math.random() - 0.5) * 0.1) 8 | ); 9 | 10 | const dot = (a, b) => a.reduce((sum, val, i) => sum + val * b[i], 0); 11 | 12 | const train = (inputs, targets, hiddenSize = 2, epochs = 100000, learningRate = 0.1) => { 13 | let w1 = create(inputs[0].length, hiddenSize); 14 | let w2 = create(hiddenSize, 1); 15 | let bias1 = Array(hiddenSize).fill(0); 16 | let bias2 = 0; 17 | 18 | for (let epoch = 0; epoch < epochs; epoch++) { 19 | let totalerr = 0; 20 | 21 | inputs.forEach((input, j) => { 22 | const hidden = w1.map((w, k) => sigmoid(dot(input, w) + bias1[k])); 23 | const output = sigmoid(dot(hidden, w2.map(w => w[0])) + bias2); 24 | const outputerr = targets[j][0] - output; 25 | totalerr += Math.abs(outputerr); 26 | const outputdel = outputerr * sigmoidDerivative(output); 27 | const hiddenErrors = w2.map(w => w[0] * outputdel); 28 | const hiddendels = hiddenErrors.map((err, k) => err * sigmoidDerivative(hidden[k])); 29 | w2 = w2.map((w, k) => [w[0] + learningRate * outputdel * hidden[k]]); 30 | bias2 += learningRate * outputdel; 31 | w1 = w1.map((w, k) => w.map((v, l) => v + learningRate * hiddendels[k] * input[l])); 32 | bias1 = bias1.map((b, k) => b + learningRate * hiddendels[k]); 33 | }); 34 | 35 | if (epoch % 1000 === 0) { 36 | console.log(`epoch ${epoch}: total error = ${totalerr}`); 37 | } 38 | } 39 | 40 | return input => { 41 | const hidden = w1.map((w, k) => sigmoid(dot(input, w) + bias1[k])); 42 | return sigmoid(dot(hidden, w2.map(w => w[0])) + bias2); 43 | }; 44 | }; 45 | 46 | const inputs = [[0, 0], [0, 1], [1, 0], [1, 1]]; 47 | const targets = [[0], [1], [1], [0]]; 48 | 49 | const model = train(inputs, targets); 50 | inputs.forEach(input => console.log(`input: ${input}, output: ${model(input).toFixed(4)}`)); 51 | -------------------------------------------------------------------------------- /Julia/nn.jl: -------------------------------------------------------------------------------- 1 | using Flux 2 | 3 | model = Chain( 4 | Dense(100, 64, relu), 5 | Dropout(0.3), 6 | Dense(64, 128, relu), 7 | Dropout(0.4), 8 | Dense(128, 64, relu), 9 | Dense(64, 10), 10 | softmax 11 | ) 12 | 13 | X = rand(Float32, 100, 1000) 14 | 15 | y = Flux.onehotbatch(rand(1:10, 1000), 1:10) 16 | 17 | loss(x, y) = Flux.crossentropy(model(x), y) 18 | 19 | using Flux: train!, params 20 | 21 | 22 | optimizer = Flux.ADAM() 23 | 24 | 25 | for epoch in 1:100 26 | 27 | Flux.train!(loss, params(model), [(X, y)], optimizer) 28 | 29 | if epoch % 10 == 0 30 | println("Epoch $epoch - Loss: $(loss(X, y))") 31 | end 32 | end 33 | # Predictions 34 | predictions = model(X) 35 | 36 | predicted_labels = Flux.onecold(predictions, 1:10) 37 | 38 | true_labels = Flux.onecold(y, 1:10) 39 | 40 | # Accuracy 41 | accuracy = sum(predicted_labels .== true_labels) / length(true_labels) 42 | println("Accuracy: $accuracy") 43 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2024 neuralnetwork. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Python/nn.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import pandas as pd 3 | from matplotlib import pyplot as plt 4 | 5 | data=pd.read_csv('/kaggle/input/digit-recognizer/train.csv') 6 | 7 | data.head() 8 | 9 | data=np.array(data) 10 | m,n=data.shape 11 | np.random.shuffle(data) 12 | 13 | data_dev=data[0:1000].T 14 | Y_dev=data_dev[0] 15 | X_dev=data_dev[1:n] 16 | X_dev=X_dev/255 17 | 18 | data_train=data[1000:m].T 19 | Y_train=data_train[0] 20 | X_train=data_train[1:n] 21 | X_train=X_train/255 22 | _,m_train=X_train.shape 23 | 24 | def init_params(): 25 | w1=np.random.rand(10,784)-0.5 26 | b1=np.random.rand(10,1)-0.5 27 | w2=np.random.rand(10,10)-0.5 28 | b2=np.random.rand(10,1)-0.5 29 | return w1,b1,w2,b2 30 | 31 | def ReLU(z): 32 | return np.maximum(0,z) 33 | 34 | def softmax(Z): 35 | A = np.exp(Z) / sum(np.exp(Z)) 36 | return A 37 | 38 | def one_hot(z): 39 | one_hot_z=np.zeros((z.size,z.max()+1)) 40 | one_hot_z[np.arange(z.size),z]=1 41 | one_hot_z=one_hot_z.T 42 | return one_hot_z 43 | 44 | def derivative_ReLU(z): 45 | return z>0 46 | 47 | def forward_prop(w1,b1,w2,b2,x): 48 | z1=w1.dot(x)+b1 49 | a1=ReLU(z1) 50 | z2=w2.dot(a1)+b2 51 | a2=softmax(z2) 52 | return z1,a1,z2,a2 53 | 54 | def back_prop(z1,a1,z2,a2,w2,x,y): 55 | m=y.size 56 | one_hot_Y=one_hot(y) 57 | dz2=a2-one_hot_Y 58 | dw2=1/m * dz2.dot(a1.T) 59 | db2=1/m * np.sum(dz2) 60 | dz1 = w2.T.dot(dz2)*derivative_ReLU(z1) 61 | dw1=1/m * dz1.dot(x.T) 62 | db1=1/m * np.sum(dz1) 63 | return dw1,db1,dw2,db2 64 | 65 | def update_params(w1,b1,w2,b2,dw1,db1,dw2,db2,alpha): 66 | w1=w1-alpha*dw1 67 | w2=w2-alpha*dw2 68 | b1=b1-alpha*db1 69 | b2=b2-alpha*db2 70 | return w1,b1,w2,b2 71 | 72 | def get_predictions(a2): 73 | return np.argmax(a2,0) 74 | 75 | def get_accuracy(predictions,y): 76 | print(predictions,y) 77 | return np.sum(predictions==y)/y.size 78 | 79 | def gradient_descent(x,y,it,alpha): 80 | w1,b1,w2,b2=init_params() 81 | for i in range(it): 82 | z1,a1,z2,a2=forward_prop(w1,b1,w2,b2,x) 83 | dw1,db1,dw2,db2=back_prop(z1,a1,z2,a2,w2,x,y) 84 | w1,b1,w2,b2=update_params(w1,b1,w2,b2,dw1,db1,dw2,db2,alpha) 85 | if i%10==0: 86 | print("Iteration",i) 87 | print("Accuracy", get_accuracy(get_predictions(a2),y)) 88 | return w1,b1,w2,b2 89 | 90 | w1,b1,w2,b2=gradient_descent(X_train,Y_train,500,0.1) 91 | 92 | def make_predictions(x,w1,b1,w2,b2): 93 | _,_,_,a2=forward_prop(w1,b1,w2,b2,x) 94 | predictions=get_predictions(a2) 95 | return predictions 96 | 97 | def test_predictions(index,w1,b1,w2,b2): 98 | current_image=X_train[:,index,None] 99 | prediction=make_predictions(X_train[:,index,None],w1,b1,w2,b2) 100 | label=Y_train[index] 101 | print("Prediction : ",prediction) 102 | print("Label : ",label) 103 | current_image=current_image.reshape((28,28))*255 104 | plt.gray() 105 | plt.imshow(current_image,interpolation='nearest') 106 | plt.show() 107 | 108 | dev_predictions=make_predictions(X_dev,w1,b1,w2,b2) 109 | print("Accuracy - ",get_accuracy(dev_predictions,Y_dev)) 110 | 111 | -------------------------------------------------------------------------------- /R/nn.r: -------------------------------------------------------------------------------- 1 | library(testthat) 2 | 3 | sigmoid <- function(x) { 4 | 1 / (1 + exp(-x)) 5 | } 6 | 7 | sigmoid_derivative <- function(x) { 8 | sigmoid(x) * (1 - sigmoid(x)) 9 | } 10 | 11 | NeuralNetwork <- setRefClass( 12 | "NeuralNetwork", 13 | fields = list( 14 | input_size = "numeric", 15 | hidden_size = "numeric", 16 | output_size = "numeric", 17 | learning_rate = "numeric", 18 | weights_ih = "matrix", 19 | weights_ho = "matrix", 20 | bias_h = "matrix", 21 | bias_o = "matrix" 22 | ), 23 | methods = list( 24 | initialize = function(input_size, hidden_size, output_size, learning_rate = 0.01) { 25 | input_size <<- input_size 26 | hidden_size <<- hidden_size 27 | output_size <<- output_size 28 | learning_rate <<- learning_rate 29 | weights_ih <<- matrix(rnorm(input_size * hidden_size, mean = 0, sd = 0.5), nrow = hidden_size, ncol = input_size) 30 | weights_ho <<- matrix(rnorm(hidden_size * output_size, mean = 0, sd = 0.5), nrow = output_size, ncol = hidden_size) 31 | bias_h <<- matrix(rnorm(hidden_size, mean = 0, sd = 0.5), nrow = hidden_size, ncol = 1) 32 | bias_o <<- matrix(rnorm(output_size, mean = 0, sd = 0.5), nrow = output_size, ncol = 1) 33 | }, 34 | 35 | forward = function(input_array) { 36 | inputs <- matrix(input_array, ncol = 1) 37 | hidden <- weights_ih %*% inputs + bias_h 38 | hidden <- apply(hidden, c(1, 2), sigmoid) 39 | output <- weights_ho %*% hidden + bias_o 40 | output <- apply(output, c(1, 2), sigmoid) 41 | list(output = output, hidden = hidden, inputs = inputs) 42 | }, 43 | 44 | train = function(input_array, target_array) { 45 | results <- forward(input_array) 46 | outputs <- results$output 47 | hidden <- results$hidden 48 | inputs <- results$inputs 49 | targets <- matrix(target_array, ncol = 1) 50 | output_errors <- targets - outputs 51 | hidden_errors <- t(weights_ho) %*% output_errors 52 | gradients_o <- output_errors * apply(outputs, c(1, 2), sigmoid_derivative) 53 | gradients_o <- gradients_o * learning_rate 54 | weights_ho_deltas <- gradients_o %*% t(hidden) 55 | weights_ho <<- weights_ho + weights_ho_deltas 56 | bias_o <<- bias_o + gradients_o 57 | gradients_h <- hidden_errors * apply(hidden, c(1, 2), sigmoid_derivative) 58 | gradients_h <- gradients_h * learning_rate 59 | weights_ih_deltas <- gradients_h %*% t(inputs) 60 | weights_ih <<- weights_ih + weights_ih_deltas 61 | bias_h <<- bias_h + gradients_h 62 | } 63 | ) 64 | ) 65 | 66 | test_that("Neural Network initialization works", { 67 | nn <- NeuralNetwork$new(2, 4, 1) 68 | expect_equal(dim(nn$weights_ih), c(4, 2)) 69 | expect_equal(dim(nn$weights_ho), c(1, 4)) 70 | expect_equal(dim(nn$bias_h), c(4, 1)) 71 | expect_equal(dim(nn$bias_o), c(1, 1)) 72 | }) 73 | 74 | test_that("Forward propagation produces expected output shape", { 75 | nn <- NeuralNetwork$new(2, 4, 1) 76 | input <- c(0.5, 0.8) 77 | output <- nn$forward(input) 78 | expect_equal(dim(output$output), c(1, 1)) 79 | }) 80 | 81 | set.seed(123) 82 | xor_inputs <- matrix(c(0,0, 0,1, 1,0, 1,1), ncol=2, byrow=TRUE) 83 | xor_outputs <- c(0, 1, 1, 0) 84 | nn <- NeuralNetwork$new(input_size=2, hidden_size=4, output_size=1) 85 | epochs <- 100000 86 | for(i in 1:epochs) { 87 | for(j in 1:nrow(xor_inputs)) { 88 | nn$train(xor_inputs[j,], xor_outputs[j]) 89 | } 90 | if (i %% 10000 == 0) { 91 | cat(i, " epochs done\n") 92 | } 93 | } 94 | 95 | cat("Testing XOR outputs:\n") 96 | for(i in 1:nrow(xor_inputs)) { 97 | result <- nn$forward(xor_inputs[i,])$output 98 | cat(sprintf("Input: %d %d, Output: %.4f\n", 99 | xor_inputs[i,1], xor_inputs[i,2], result)) 100 | } 101 | 102 | 103 | 104 | 105 | 106 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Neural Networks in Multiple Languages 2 | This project demonstrates the implementation of basic neural networks in multiple programming languages. The goal is to provide simple neural network examples across different languages, illustrating how such models can be developed in each one. 3 | 4 | ### Languages Available 5 | 6 | - [ASM/Linux](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/ASM/Linux) 7 | - [Bash](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Bash) 8 | - [C](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/C) 9 | - [C++](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/C%2B%2B) 10 | - [CUDA](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/CUDA) 11 | - [Dart](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Dart) 12 | - [Go](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Go) 13 | - [Java](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Java) 14 | - [Javascript](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Javascript) 15 | - [Julia](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Julia) 16 | - [Python](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Python) 17 | - [R](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/R) 18 | - [Rust](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Rust) 19 | - [Typescript](https://github.com/cneuralnetwork/Neural-Networks-in-every-Language/tree/main/Typescript) 20 | 21 | ### How to contribute 22 | - Fork this repository 23 | - Make a **FOLDER** naming your language (for eg. `C` for NN in C) 24 | - Add your files to that folder 25 | - Pull request 26 | -------------------------------------------------------------------------------- /Rust/main.rs: -------------------------------------------------------------------------------- 1 | use rand::Rng; 2 | use std::f64; 3 | /* 4 | * Do cargo run to run the program 5 | * On windows, you need wsl to run this 6 | */ 7 | 8 | // Activation function (sigmoid) 9 | fn sigmoid(x: f64) -> f64 { 10 | 1.0 / (1.0 + f64::exp(-x)) 11 | } 12 | 13 | // Derivative of sigmoid 14 | fn sigmoid_derivative(x: f64) -> f64 { 15 | x * (1.0 - x) 16 | } 17 | 18 | fn main() { 19 | // Training data (XOR gate) 20 | let training_inputs = vec![ 21 | vec![0.0, 0.0], 22 | vec![0.0, 1.0], 23 | vec![1.0, 0.0], 24 | vec![1.0, 1.0], 25 | ]; 26 | 27 | let training_outputs = vec![ 28 | vec![0.0], 29 | vec![1.0], 30 | vec![1.0], 31 | vec![0.0], 32 | ]; 33 | 34 | // Network architecture (2 input, 2 hidden, 1 output) 35 | let num_inputs = 2; 36 | let num_hidden = 2; 37 | let num_outputs = 1; 38 | 39 | // Initialize weights and biases randomly 40 | let mut rng = rand::thread_rng(); 41 | let mut hidden_weights = vec![vec![0.0; num_hidden]; num_inputs]; 42 | let mut hidden_biases = vec![0.0; num_hidden]; 43 | let mut output_weights = vec![vec![0.0; num_outputs]; num_hidden]; 44 | let mut output_biases = vec![0.0; num_outputs]; 45 | 46 | for i in 0..num_inputs { 47 | for j in 0..num_hidden { 48 | hidden_weights[i][j] = rng.gen_range(-1.0..1.0); 49 | } 50 | } 51 | for i in 0..num_hidden { 52 | hidden_biases[i] = rng.gen_range(-1.0..1.0); 53 | for j in 0..num_outputs { 54 | output_weights[i][j] = rng.gen_range(-1.0..1.0); 55 | } 56 | } 57 | for i in 0..num_outputs { 58 | output_biases[i] = rng.gen_range(-1.0..1.0); 59 | } 60 | 61 | // Training loop 62 | let num_epochs = 10000; 63 | let learning_rate = 0.1; 64 | for _ in 0..num_epochs { 65 | for i in 0..training_inputs.len() { 66 | // Forward pass 67 | let mut hidden_layer = vec![0.0; num_hidden]; 68 | for j in 0..num_hidden { 69 | let mut weighted_sum = 0.0; 70 | for k in 0..num_inputs { 71 | weighted_sum += training_inputs[i][k] * hidden_weights[k][j]; 72 | } 73 | hidden_layer[j] = sigmoid(weighted_sum + hidden_biases[j]); 74 | } 75 | 76 | let mut output_layer = vec![0.0; num_outputs]; 77 | for j in 0..num_outputs { 78 | let mut weighted_sum = 0.0; 79 | for k in 0..num_hidden { 80 | weighted_sum += hidden_layer[k] * output_weights[k][j]; 81 | } 82 | output_layer[j] = sigmoid(weighted_sum + output_biases[j]); 83 | } 84 | 85 | // Backpropagation 86 | let mut output_errors = vec![0.0; num_outputs]; 87 | for j in 0..num_outputs { 88 | output_errors[j] = (training_outputs[i][j] - output_layer[j]) * sigmoid_derivative(output_layer[j]); 89 | } 90 | 91 | let mut hidden_errors = vec![0.0; num_hidden]; 92 | for j in 0..num_hidden { 93 | let mut error_sum = 0.0; 94 | for k in 0..num_outputs { 95 | error_sum += output_errors[k] * output_weights[j][k]; 96 | } 97 | hidden_errors[j] = error_sum * sigmoid_derivative(hidden_layer[j]); 98 | } 99 | 100 | // Update weights and biases 101 | for j in 0..num_outputs { 102 | for k in 0..num_hidden { 103 | output_weights[k][j] += learning_rate * output_errors[j] * hidden_layer[k]; 104 | } 105 | output_biases[j] += learning_rate * output_errors[j]; 106 | } 107 | 108 | for j in 0..num_hidden { 109 | for k in 0..num_inputs { 110 | hidden_weights[k][j] += learning_rate * hidden_errors[j] * training_inputs[i][k]; 111 | } 112 | hidden_biases[j] += learning_rate * hidden_errors[j]; 113 | } 114 | } 115 | } 116 | 117 | // Test the network 118 | println!("Testing the neural network:"); 119 | for i in 0..training_inputs.len() { 120 | let mut hidden_layer = vec![0.0; num_hidden]; 121 | for j in 0..num_hidden { 122 | let mut weighted_sum = 0.0; 123 | for k in 0..num_inputs { 124 | weighted_sum += training_inputs[i][k] * hidden_weights[k][j]; 125 | } 126 | hidden_layer[j] = sigmoid(weighted_sum + hidden_biases[j]); 127 | } 128 | 129 | let mut output_layer = vec![0.0; num_outputs]; 130 | for j in 0..num_outputs { 131 | let mut weighted_sum = 0.0; 132 | for k in 0..num_hidden { 133 | weighted_sum += hidden_layer[k] * output_weights[k][j]; 134 | } 135 | output_layer[j] = sigmoid(weighted_sum + output_biases[j]); 136 | } 137 | 138 | println!("Input: {:?}, Output: {:.4}", training_inputs[i], output_layer[0]); 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /Typescript/nn.ts: -------------------------------------------------------------------------------- 1 | import * as tf from '@tensorflow/tfjs'; 2 | 3 | function initParams() { 4 | const w1 = tf.randomUniform([10, 784], -0.5, 0.5); 5 | const b1 = tf.randomUniform([10, 1], -0.5, 0.5); 6 | const w2 = tf.randomUniform([10, 10], -0.5, 0.5); 7 | const b2 = tf.randomUniform([10, 1], -0.5, 0.5); 8 | return { w1, b1, w2, b2 }; 9 | } 10 | 11 | function ReLU(z: tf.Tensor) { 12 | return z.relu(); 13 | } 14 | 15 | function softmax(z: tf.Tensor) { 16 | return z.softmax(); 17 | } 18 | 19 | function oneHot(y: number[], numClasses: number): tf.Tensor { 20 | return tf.oneHot(tf.tensor1d(y, 'int32'), numClasses); 21 | } 22 | 23 | function forwardProp(w1: tf.Tensor, b1: tf.Tensor, w2: tf.Tensor, b2: tf.Tensor, x: tf.Tensor) { 24 | const z1 = w1.matMul(x).add(b1); 25 | const a1 = ReLU(z1); 26 | const z2 = w2.matMul(a1).add(b2); 27 | const a2 = softmax(z2); 28 | return { z1, a1, z2, a2 }; 29 | } 30 | 31 | function backProp(z1: tf.Tensor, a1: tf.Tensor, a2: tf.Tensor, w2: tf.Tensor, x: tf.Tensor, y: tf.Tensor) { 32 | const m = y.shape[1]; 33 | const dz2 = a2.sub(y); 34 | const dw2 = dz2.matMul(a1.transpose()).div(m); 35 | const db2 = dz2.sum(1).reshape([10, 1]).div(m); 36 | const dz1 = w2.transpose().matMul(dz2).mul(z1.greater(0)); 37 | const dw1 = dz1.matMul(x.transpose()).div(m); 38 | const db1 = dz1.sum(1).reshape([10, 1]).div(m); 39 | return { dw1, db1, dw2, db2 }; 40 | } 41 | 42 | function updateParams(w1: tf.Tensor, b1: tf.Tensor, w2: tf.Tensor, b2: tf.Tensor, dw1: tf.Tensor, db1: tf.Tensor, dw2: tf.Tensor, db2: tf.Tensor, alpha: number) { 43 | = w1.sub(dw1.mul(alpha)); 44 | = b1.sub(db1.mul(alpha)); 45 | = w2.sub(dw2.mul(alpha)); 46 | = b2.sub(db2.mul(alpha)); 47 | return { w1, b1, w2, b2 }; 48 | } 49 | 50 | function getPredictions(a2: tf.Tensor) { 51 | return a2.argMax(0); 52 | } 53 | 54 | function getAccuracy(predictions: tf.Tensor, y: tf.Tensor) { 55 | return predictions.equal(y.argMax(0)).sum().div(y.shape[1]).arraySync(); 56 | } 57 | 58 | async function gradientDescent(X: tf.Tensor, Y: tf.Tensor, iterations: number, alpha: number) { 59 | t { w1, b1, w2, b2 } = initParams(); 60 | 61 | r (let i = 0; i < iterations; i++) { 62 | t { z1, a1, z2, a2 } = forwardProp(w1, b1, w2, b2, X); 63 | t { dw1, db1, dw2, db2 } = backProp(z1, a1, a2, w2, X, Y); 64 | 1, b1, w2, b2 } = updateParams(w1, b1, w2, b2, dw1, db1, dw2, db2, alpha)); 65 | 66 | i % 10 === 0) { 67 | predictions = getPredictions(a2); 68 | accuracy = getAccuracy(predictions, Y); 69 | e.log(`Iteration ${i}, Accuracy: ${accuracy}`); 70 | 71 | 72 | return { w1, b1, w2, b2 }; 73 | } 74 | 75 | function makePredictions(X: tf.Tensor, w1: tf.Tensor, b1: tf.Tensor, w2: tf.Tensor, b2: tf.Tensor) { 76 | const { a2 } = forwardProp(w1, b1, w2, b2, X); 77 | return getPredictions(a2); 78 | } 79 | 80 | async function main() { 81 | const X_train = tf.randomUniform([784, 1000]); 82 | const Y_train = oneHot([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 10).tile([1, 100]); 83 | 84 | const { w1, b1, w2, b2 } = await gradientDescent(X_train, Y_train, 500, 0.1); 85 | const X_test = tf.randomUniform([784, 10]); // 10 test examples 86 | const predictions = makePredictions(X_test, w1, b1, w2, b2); 87 | edictions.print(); 88 | } 89 | 90 | main(); 91 | 92 | // Testing 93 | 94 | /*import * as tf from '@tensorflow/tfjs'; 95 | 96 | async function testNN() { 97 | const w1 = tf.randomUniform([10, 784], -0.5, 0.5); 98 | const b1 = tf.randomUniform([10, 1], -0.5, 0.5); 99 | const w2 = tf.randomUniform([10, 10], -0.5, 0.5); 100 | const b2 = tf.randomUniform([10, 1], -0.5, 0.5); 101 | 102 | const X_train = tf.randomUniform([784, 1000]); 103 | const Y_train = tf.oneHot(tf.tensor1d([0, 1, 2, 3, 4, 5, 6, 7, 8, 9], 'int32'), 10).tile([100, 1]); 104 | 105 | const { z1, a1, z2, a2 } = forwardProp(w1, b1, w2, b2, X_train); 106 | console.log("Forward Propagation successful"); 107 | } 108 | 109 | testNN(); 110 | */ 111 | -------------------------------------------------------------------------------- /p5js/ReadMe.md: -------------------------------------------------------------------------------- 1 | doesn't do a lot of shit but a simple neural network animation. 2 | -------------------------------------------------------------------------------- /p5js/nn.js: -------------------------------------------------------------------------------- 1 | // https://editor.p5js.org/ 2 | // CTRL+C fucking CTRL+V 3 | 4 | let dots = []; 5 | let lines = []; 6 | 7 | function setup() { 8 | createCanvas(1000, 600); 9 | let layers = [3, 8, 6, 8, 3]; 10 | 11 | for (let i = 0; i < layers.length; i++) { 12 | let layer = []; 13 | let x = map(i, 0, layers.length - 1, 100, width - 100); 14 | for (let j = 0; j < layers[i]; j++) { 15 | let y = map(j, 0, layers[i] - 1, 100, height - 100); 16 | layer.push(new Dot(x, y)); 17 | } 18 | dots.push(layer); 19 | } 20 | 21 | for (let l = 0; l < dots.length - 1; l++) { 22 | let current = dots[l]; 23 | let next = dots[l + 1]; 24 | for (let i = 0; i < current.length; i++) { 25 | for (let j = 0; j < next.length; j++) { 26 | lines.push(new Line(current[i], next[j])); 27 | } 28 | } 29 | } 30 | } 31 | 32 | function draw() { 33 | background(0); 34 | 35 | for (let i = 0; i < lines.length; i++) { 36 | lines[i].update(); 37 | lines[i].show(); 38 | } 39 | 40 | for (let i = 0; i < dots.length; i++) { 41 | for (let j = 0; j < dots[i].length; j++) { 42 | dots[i][j].update(); 43 | dots[i][j].show(); 44 | } 45 | } 46 | } 47 | 48 | class Dot { 49 | constructor(x, y) { 50 | this.x = x; 51 | this.y = y; 52 | this.val = random(1); 53 | this.size = 20; 54 | } 55 | 56 | update() { 57 | this.val = noise(frameCount * 0.01 + this.x * 0.01) * 2 - 1; 58 | } 59 | 60 | show() { 61 | noStroke(); 62 | fill(0, 200, 255, 80); 63 | ellipse(this.x, this.y, this.size + 15); 64 | 65 | fill(0, 200, 255); 66 | ellipse(this.x, this.y, this.size); 67 | 68 | fill(255); 69 | noStroke(); 70 | textSize(12); 71 | textAlign(CENTER, CENTER); 72 | text(nf(this.val, 1, 2), this.x, this.y); 73 | } 74 | } 75 | 76 | class Line { 77 | constructor(fromDot, toDot) { 78 | this.from = fromDot; 79 | this.to = toDot; 80 | this.weight = random(-1, 1); 81 | } 82 | 83 | update() { 84 | this.weight = noise(frameCount * 0.01 + this.from.x * 0.01 + this.to.y * 0.01) * 2 - 1; 85 | } 86 | 87 | show() { 88 | strokeWeight(map(abs(this.weight), 0, 1, 1, 3)); 89 | stroke(this.weight > 0 ? 'rgba(0,255,255,0.7)' : 'rgba(255,50,50,0.7)'); 90 | line(this.from.x, this.from.y, this.to.x, this.to.y); 91 | } 92 | } 93 | --------------------------------------------------------------------------------