├── README.md └── deepnn.jl /README.md: -------------------------------------------------------------------------------- 1 | # DNN_Julia 2 | -------------------------------------------------------------------------------- /deepnn.jl: -------------------------------------------------------------------------------- 1 | using Random 2 | using LinearAlgebra 3 | using Statistics 4 | using Plots 5 | using DelimitedFiles 6 | pyplot() 7 | function sigmoid(X) 8 | sigma = 1 ./(1 .+ exp.(.-X)) 9 | return sigma , X 10 | end 11 | function relu(X) 12 | rel = max.(0,X) 13 | return rel , X 14 | end 15 | 16 | 17 | function init_param(layer_dimensions) 18 | 19 | param = Dict() 20 | 21 | for l=1:length(layer_dimensions)-1 22 | 23 | param[string("W_" , string(l))] = 0.1f0*randn(layer_dimensions[l+1] , layer_dimensions[l]) 24 | param[string("b_" , string(l))] = zeros(layer_dimensions[l+1] , 1) 25 | 26 | end 27 | 28 | return param 29 | 30 | end 31 | 32 | 33 | function forward_linear(A , w , b) 34 | 35 | Z = w*A .+ b 36 | cache = (A , w , b) 37 | 38 | return Z,cache 39 | 40 | end 41 | 42 | function cost_function(AL , Y) 43 | 44 | cost = -mean(Y.*log.(AL) + (1 .- Y).*log.(1 .- AL)) 45 | 46 | return cost 47 | 48 | end 49 | 50 | function backward_linear_step(dZ , cache) 51 | 52 | A_prev , W , b = cache 53 | 54 | m = size(A_prev)[2] 55 | 56 | dW = dZ * (A_prev')/m 57 | db = sum(dZ , dims = 2)/m 58 | dA_prev = (W')* dZ 59 | return dW , db , dA_prev 60 | 61 | end 62 | 63 | function backward_relu(dA , cache_activation) 64 | return dA.*(cache_activation.>0) 65 | end 66 | 67 | function backward_sigmoid(dA , cache_activation) 68 | return dA.*(sigmoid(cache_activation)[1].*(1 .- sigmoid(cache_activation)[1])) 69 | end 70 | 71 | function backward_activation_step(dA , cache , activation) 72 | 73 | linear_cache , cache_activation = cache 74 | if (activation == "relu") 75 | 76 | dZ = backward_relu(dA , cache_activation) 77 | dW , db , dA_prev = backward_linear_step(dZ , linear_cache) 78 | 79 | elseif (activation == "sigmoid") 80 | 81 | dZ = backward_sigmoid(dA , cache_activation) 82 | dW , db , dA_prev = backward_linear_step(dZ , linear_cache) 83 | 84 | end 85 | 86 | return dW , db , dA_prev 87 | 88 | end 89 | 90 | function model_backwards_step(A_l , Y , caches) 91 | 92 | grads = Dict() 93 | 94 | L = length(caches) 95 | 96 | m = size(A_l)[2] 97 | 98 | Y = reshape(Y , size(A_l)) 99 | dA_l = (-(Y./A_l) .+ ((1 .- Y)./( 1 .- A_l))) 100 | current_cache = caches[L] 101 | grads[string("dW_" , string(L))] , grads[string("db_" , string(L))] , grads[string("dA_" , string(L-1))] = backward_activation_step(dA_l , current_cache , "sigmoid") 102 | for l=reverse(0:L-2) 103 | current_cache = caches[l+1] 104 | grads[string("dW_" , string(l+1))] , grads[string("db_" , string(l+1))] , grads[string("dA_" , string(l))] = backward_activation_step(grads[string("dA_" , string(l+1))] , current_cache , "relu") 105 | 106 | end 107 | 108 | return grads 109 | 110 | end 111 | 112 | function update_param(parameters , grads , learning_rate) 113 | 114 | L = Int(length(parameters)/2) 115 | 116 | for l=0:(L-1) 117 | 118 | parameters[string("W_" , string(l+1))] -= learning_rate.*grads[string("dW_" , string(l+1))] 119 | parameters[string("b_",string(l+1))] -= learning_rate.*grads[string("db_",string(l+1))] 120 | 121 | end 122 | 123 | return parameters 124 | 125 | end 126 | 127 | function calculate_activation_forward(A_pre , W , b , function_type) 128 | 129 | if (function_type == "sigmoid") 130 | 131 | Z , linear_step_cache = forward_linear(A_pre , W , b) 132 | A , activation_step_cache = sigmoid(Z) 133 | 134 | elseif (function_type == "relu") 135 | 136 | Z , linear_step_cache = forward_linear(A_pre , W , b) 137 | A , activation_step_cache = relu(Z) 138 | 139 | end 140 | 141 | cache = (linear_step_cache , activation_step_cache) 142 | return A , cache 143 | 144 | end 145 | 146 | function model_forward_step(X , params) 147 | 148 | all_caches = [] 149 | A = X 150 | L = length(params)/2 151 | 152 | for l=1:L-1 153 | A_pre = A 154 | A , cache = calculate_activation_forward(A_pre , params[string("W_" , string(Int(l)))] , params[string("b_" , string(Int(l)))] , "relu") 155 | push!(all_caches , cache) 156 | end 157 | A_l , cache = calculate_activation_forward(A , params[string("W_" , string(Int(L)))] , params[string("b_" , string(Int(L)))] , "sigmoid") 158 | push!(all_caches , cache) 159 | 160 | 161 | return A_l , all_caches 162 | 163 | end 164 | 165 | function check_accuracy(A_L , Y) 166 | A_L = reshape(A_L , size(Y)) 167 | return sum((A_L.>0.5) .== Y)/length(Y) 168 | end 169 | 170 | function train_nn(layers_dimensions , X , Y , learning_rate , n_iter) 171 | 172 | params = init_param(layers_dimensions) 173 | costs = [] 174 | iters = [] 175 | accuracy = [] 176 | for i=1:n_iter 177 | A_l , caches = model_forward_step(X , params) 178 | cost = cost_function(A_l , Y) 179 | acc = check_accuracy(A_l , Y) 180 | grads = model_backwards_step(A_l , Y , caches) 181 | params = update_param(params , grads , learning_rate) 182 | println("Iteration ->" , i) 183 | println("Cost ->" , cost) 184 | println("Accuracy -> " , acc) 185 | push!(iters , i) 186 | push!(costs , cost) 187 | push!(accuracy , acc) 188 | 189 | end 190 | plt = plot(iters , costs ,title = "Cost Function vs Number of Iterations" , lab ="J") 191 | xaxis!("N_Iterations") 192 | yaxis!("J") 193 | plt_2 = plot(iters , accuracy ,title = "Accuracy vs Number of Iterations" , lab ="Acc" , color = :green) 194 | xaxis!("N_Iterations") 195 | yaxis!("Accuracy") 196 | plot(plt , plt_2 , layout = (2,1)) 197 | savefig("cost_plot_rand.png") 198 | return params , costs 199 | 200 | end 201 | 202 | 203 | 204 | 205 | 206 | 207 | --------------------------------------------------------------------------------