├── GWO
    ├── main.m
    ├── mape_calc.m
    ├── testset.mat
    ├── trainset.mat
    └── weight_mat.mat
├── ICA
    ├── AssimilateColonies.m
    ├── DoRevolution.m
    ├── ICA.m
    ├── InterEmpireCompetition.m
    ├── IntraEmpireCompetition.m
    ├── RouletteWheelSelection.m
    ├── UpdateTotalCost.m
    ├── mape_calc.m
    ├── testset.mat
    ├── trainset.mat
    └── weight_mat.mat
├── NN
    ├── data.mat
    ├── leaky_grad.m
    ├── leaky_relu.m
    ├── main.m
    ├── predict.m
    ├── relu.m
    ├── relu_grad.m
    └── rsme.m
├── PSO
    ├── main.m
    ├── mape_calc.m
    ├── testset.mat
    ├── trainset.mat
    └── weight_mat.mat
├── README.md
└── dataset_pre.m


/GWO/main.m:
--------------------------------------------------------------------------------
  1 | clc;
  2 | clear;
  3 | close all;
  4 | %% Problem Definition
  5 | load('trainset.mat')
  6 | load('testset.mat')
  7 | load('weight_mat.mat')
  8 | 
  9 | var_num=71;            
 10 | VarSize=[1 var_num];   
 11 | VarMin=-5;         
 12 | VarMax= 5;         
 13 | %% GWO Parameters
 14 | max_epoch=1500;      
 15 | ini_pop=50;      
 16 | empty_wolf.Position=[];
 17 | empty_wolf.Cost=[];
 18 | wolf=repmat(empty_wolf,ini_pop,1);
 19 | Cost_Test= zeros(50,1);
 20 | for i=1:ini_pop
 21 |     % Initialize Position
 22 |     wolf(i).Position= WEIGHTS(i ,:); 
 23 |     % Evaluation
 24 |     wolf(i).Cost=mape_calc(wolf(i).Position,trainset);
 25 |     Cost_Test(i)=mape_calc(wolf(i).Position,testset);
 26 | end
 27 | 
 28 | BestCost_Train=zeros(max_epoch,1);
 29 | BestCost_Test=zeros(max_epoch,1);
 30 | costs=[wolf.Cost];
 31 | [~, SortOrder]=sort(costs);
 32 | wolf=wolf(SortOrder);
 33 | 
 34 | [~, SortOrder]=sort(Cost_Test);
 35 | Cost_Test =Cost_Test(SortOrder);
 36 | 
 37 | % nfe=zeros(max_epoch,1);
 38 | 
 39 | %% GWO Main Loop
 40 | for epoch=1:max_epoch
 41 |     alfa = wolf(1);
 42 |     beta = wolf(2);
 43 |     delta = wolf(3);
 44 |     omegas = wolf(4:end);
 45 | 
 46 |     a = 2-(epoch/max_epoch);
 47 |     for j=1 :47
 48 |     A1 = 2*a*rand(1,71)-a;
 49 |     A2 = 2*a*rand(1,71)-a;
 50 |     A3 = 2*a*rand(1,71)-a;
 51 |     
 52 |     c1 = 2*rand(1,71);
 53 |     c2 = 2*rand(1,71);
 54 |     c3 = 2*rand(1,71);
 55 |     d_alfa = abs(c1.*(alfa.Position) - omegas(j).Position);
 56 |     d_beta = abs(c2.*(beta.Position) - omegas(j).Position);
 57 |     d_delta = abs(c3.*(delta.Position) - omegas(j).Position);
 58 |     
 59 |     new_omegas_1 = alfa.Position - A1.*d_alfa;
 60 |     new_omegas_2 = beta.Position - A2.*d_beta;
 61 |     new_omegas_3 = delta.Position - A3.*d_delta;
 62 |     omegas(j).Position = (new_omegas_1+new_omegas_2+new_omegas_3)/3;
 63 |     omegas(j).Cost = mape_calc(omegas(j).Position,trainset);
 64 |     end
 65 |     
 66 |       wolf(1)=alfa;
 67 |       wolf(2) = beta;
 68 |       wolf(3)= delta;
 69 |       wolf(4:end) = omegas;
 70 |       for l= 1:ini_pop
 71 |           Cost_Test(l)=mape_calc(wolf(l).Position,testset);
 72 |       end
 73 |       [~, SortOrder]=sort(Cost_Test);
 74 |       Cost_Test =Cost_Test(SortOrder);
 75 |       BestCost_Test(epoch) = Cost_Test(1);
 76 |       
 77 |       costs=[wolf.Cost];
 78 |       [~, SortOrder]=sort(costs);
 79 |       wolf=wolf(SortOrder); 
 80 |       BestCost_Train(epoch) = wolf(1).Cost;
 81 |       
 82 | %       nfe(epoch)=NFE;
 83 |       disp(['Iteration ' num2str(epoch)  ', cost on train data = ' num2str(BestCost_Train(epoch)) ', cost on test data = ' num2str(BestCost_Test(epoch))]);
 84 | 
 85 | end
 86 | 
 87 | %%  diagrams %%
 88 | figure;
 89 | semilogx(BestCost_Train,'LineWidth',2);
 90 | xlabel('epochs');
 91 | ylabel('train costs');
 92 | grid on;
 93 | 
 94 | figure;
 95 | semilogx(BestCost_Test,'LineWidth',2);
 96 | xlabel('epochs');
 97 | ylabel('test costs');
 98 | grid on;
 99 | 
100 | 
101 | 
102 | 
103 | 
104 | 


--------------------------------------------------------------------------------
/GWO/mape_calc.m:
--------------------------------------------------------------------------------
 1 | function MAPE_train = mape_calc(weight,trainset)
 2 |         W1 = reshape(weight(1:50),10,5);
 3 |         B1 = reshape(weight(51:60),10,1);
 4 |         W2 = weight(61:70);
 5 |         B2 = weight(71);
 6 |         target = trainset(:,6);
 7 |         output = zeros(size(trainset,1),1);
 8 |         for j=1:size(trainset,1)
 9 |             x = trainset(j,1:5);
10 |             z_in = x*W1'+B1';
11 |             z = max(0,z_in);
12 |             y = W2*z' + B2;
13 |             output(j) = max(0,y);
14 |         end
15 |         MAPE_train = sum(abs((output-target))./target)/size(trainset,1);
16 |     
17 | end


--------------------------------------------------------------------------------
/GWO/testset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/GWO/testset.mat


--------------------------------------------------------------------------------
/GWO/trainset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/GWO/trainset.mat


--------------------------------------------------------------------------------
/GWO/weight_mat.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/GWO/weight_mat.mat


--------------------------------------------------------------------------------
/ICA/AssimilateColonies.m:
--------------------------------------------------------------------------------
 1 | 
 2 | function emp=AssimilateColonies(emp)
 3 |     global trainset;
 4 |     global ProblemSettings;
 5 |     CostFunction=ProblemSettings.CostFunction;
 6 |     VarSize=ProblemSettings.VarSize;
 7 |     VarMin=ProblemSettings.VarMin;
 8 |     VarMax=ProblemSettings.VarMax;
 9 |     
10 |     global ICASettings;
11 |     beta=ICASettings.beta;
12 |     
13 |     nEmp=numel(emp);
14 |     for k=1:nEmp
15 |         for i=1:emp(k).nCol
16 |             
17 |             emp(k).Col(i).Position = emp(k).Col(i).Position ...
18 |                 + beta*rand(VarSize).*(emp(k).Imp.Position-emp(k).Col(i).Position);
19 |             
20 |             emp(k).Col(i).Position = max(emp(k).Col(i).Position,VarMin);
21 |             emp(k).Col(i).Position = min(emp(k).Col(i).Position,VarMax);
22 |             
23 |             emp(k).Col(i).Cost = CostFunction(emp(k).Col(i).Position,trainset);
24 |             
25 |         end
26 |     end
27 | end


--------------------------------------------------------------------------------
/ICA/DoRevolution.m:
--------------------------------------------------------------------------------
 1 | 
 2 | function emp=DoRevolution(emp)
 3 |     global ProblemSettings;
 4 |     global trainset;
 5 |     CostFunction=ProblemSettings.CostFunction;
 6 |     nVar=ProblemSettings.nVar;
 7 |     VarSize=ProblemSettings.VarSize;
 8 |     VarMin=ProblemSettings.VarMin;
 9 |     VarMax=ProblemSettings.VarMax;
10 |     
11 |     global ICASettings;
12 |     pRevolution=ICASettings.pRevolution;
13 |     mu=ICASettings.mu;
14 |     
15 |     nmu=ceil(mu*nVar);
16 |     
17 |     sigma=0.1*(VarMax-VarMin);
18 |     
19 |     nEmp=numel(emp);
20 |     for k=1:nEmp
21 |         
22 |         NewPos = emp(k).Imp.Position + sigma*randn(VarSize);
23 |         
24 |         jj=randsample(nVar,nmu)';
25 |         NewImp=emp(k).Imp;
26 |         NewImp.Position(jj)=NewPos(jj);
27 |         NewImp.Cost=CostFunction(NewImp.Position,trainset);
28 |         if NewImp.Cost<emp(k).Imp.Cost
29 |             emp(k).Imp = NewImp;
30 |         end
31 |         
32 |         for i=1:emp(k).nCol
33 |             if rand<=pRevolution
34 |                 NewPos = emp(k).Col(i).Position + sigma*randn(VarSize);
35 |                 jj=randsample(nVar,nmu)';
36 |                 emp(k).Col(i).Position(jj) = NewPos(jj);
37 |                 emp(k).Col(i).Position = max(emp(k).Col(i).Position,VarMin);
38 |                 emp(k).Col(i).Position = min(emp(k).Col(i).Position,VarMax);
39 |                 emp(k).Col(i).Cost = CostFunction(emp(k).Col(i).Position,trainset);
40 |             end
41 |         end
42 |     end
43 | end


--------------------------------------------------------------------------------
/ICA/ICA.m:
--------------------------------------------------------------------------------
  1 | 
  2 | clc;
  3 | clear;
  4 | close all;
  5 | load('weight_mat.mat')
  6 | load('trainset.mat')
  7 | load('testset.mat')
  8 | global trainset;
  9 | %% Problem Definition  
 10 | CostFunction=@(voroodi,trainset) mape_calc(voroodi,trainset);       
 11 | var_num=71;             
 12 | VarSize=[1 var_num];   
 13 | VarMin=-5;        
 14 | VarMax= 5;         
 15 | %% ICA Parameters
 16 | max_epoch=2000;        
 17 | ini_pop=50;           
 18 | em_num=4;            
 19 | nCol=ini_pop-em_num;
 20 | alpha=1;            % Selection Pressure
 21 | beta=1.5;           % Assimilation Coefficient
 22 | pRevolution=0.05;   % Revolution Probability
 23 | mu=0.1;             % Revolution Rate
 24 | zeta=0.2;           % Colonies Mean Cost Coefficient
 25 | %% Globalization of Parameters and Settings
 26 | global ProblemSettings;
 27 | ProblemSettings.CostFunction=CostFunction;
 28 | ProblemSettings.nVar=var_num;
 29 | ProblemSettings.VarSize=VarSize;
 30 | ProblemSettings.VarMin=VarMin;
 31 | ProblemSettings.VarMax=VarMax;
 32 | global ICASettings;
 33 | ICASettings.MaxIt=max_epoch;
 34 | ICASettings.nPop=ini_pop;
 35 | ICASettings.nEmp=em_num;
 36 | ICASettings.alpha=alpha;
 37 | ICASettings.beta=beta;
 38 | ICASettings.pRevolution=pRevolution;
 39 | ICASettings.mu=mu;
 40 | ICASettings.zeta=zeta;
 41 | %% Initialization
 42 | % Initialize Empires
 43 | empty_country.Position=[];
 44 | empty_country.Cost=[];
 45 | Cost_Test= zeros(50,1);
 46 | country=repmat(empty_country,ini_pop,1);
 47 | for i=1:ini_pop
 48 |         country(i).Position=WEIGHTS(i ,:);
 49 |         country(i).Cost=CostFunction(country(i).Position,trainset);
 50 |         Cost_Test(i)=CostFunction(country(i).Position,testset);
 51 | end
 52 |     
 53 |     costs=[country.Cost];
 54 |     [~, SortOrder]=sort(costs);  
 55 |     country=country(SortOrder);
 56 |     [~, SortOrder]=sort(Cost_Test);
 57 |     Cost_Test =Cost_Test(SortOrder);
 58 |     
 59 |     imp=country(1:em_num);
 60 |     col=country(em_num+1:end); 
 61 |     empty_empire.Imp=[];
 62 |     empty_empire.Col=repmat(empty_country,0,1);
 63 |     empty_empire.nCol=0;
 64 |     empty_empire.TotalCost=[];
 65 |     emp=repmat(empty_empire,em_num,1);
 66 |     
 67 |     % Assign Imperialists
 68 |     for k=1:em_num
 69 |         emp(k).Imp=imp(k);
 70 |     end
 71 |     
 72 |     % Assign Colonies
 73 |     P=exp(-alpha*[imp.Cost]/max([imp.Cost]));
 74 |     P=P/sum(P);
 75 |     for j=1:nCol
 76 |         
 77 |         k=RouletteWheelSelection(P);
 78 |         emp(k).Col=[emp(k).Col
 79 |                     col(j)];
 80 |         emp(k).nCol=emp(k).nCol+1;
 81 |     end
 82 |     
 83 |     emp=UpdateTotalCost(emp);
 84 | 
 85 | % Array to Hold Best Cost Values
 86 | BestCost_Train=zeros(max_epoch,1);
 87 | BestCost_Test=zeros(max_epoch,1);
 88 | nfe=zeros(max_epoch,1);
 89 | %% ICA Main Loop
 90 | for it=1:max_epoch
 91 | 
 92 |     % Assimilation
 93 |     emp=AssimilateColonies(emp);
 94 |     
 95 |     % Revolution
 96 |     emp=DoRevolution(emp);
 97 |     
 98 |     % Intra-Empire Competition
 99 |     emp=IntraEmpireCompetition(emp);
100 |     
101 |     % Update Total Cost of Empires
102 |     emp=UpdateTotalCost(emp);
103 |     
104 |     % Inter-Empire Competition
105 |     emp=InterEmpireCompetition(emp);
106 |     
107 |     % Update Best Solution Ever Found
108 |     imp=[emp.Imp];
109 |     [~, BestImpIndex]=min([imp.Cost]);
110 |     BestSol=imp(BestImpIndex);
111 |     % Update Best Cost
112 |     BestCost_Train(it)=BestSol.Cost;
113 |     switch numel(emp)
114 |         case 4
115 |             M = [1,emp(1).nCol+1,1+emp(1).nCol+emp(2).nCol,1+emp(1).nCol+emp(2).nCol+emp(3).nCol];
116 |             N=[emp(1).nCol,emp(1).nCol+emp(2).nCol,emp(1).nCol+emp(2).nCol+emp(3).nCol,emp(1).nCol+emp(2).nCol+emp(3).nCol+emp(4).nCol];
117 |         case 3
118 |             M = [1,emp(1).nCol+1,1+emp(1).nCol+emp(2).nCol];
119 |             N=[emp(1).nCol,emp(1).nCol+emp(2).nCol,emp(1).nCol+emp(2).nCol+emp(3).nCol];
120 |         case 2
121 |             M = [1,emp(1).nCol+1];
122 |             N=[emp(1).nCol,emp(1).nCol+emp(2).nCol];
123 |         case 1
124 |             M = 1;
125 |             N=[emp(1).nCol];
126 |     end
127 |     o=1;
128 |     for m=1 : numel(emp)
129 |         for n = M(m) : N(m)
130 |            Cost_Test(n)=CostFunction(emp(m).Col(o).Position,testset); 
131 |            o=o+1;
132 |         end
133 |         o=1;
134 |     end
135 |     for l=1 :numel(emp)
136 |         Cost_Test(l + N(end))=CostFunction(imp(l).Position,testset);
137 |     end
138 |     [~, SortOrder]=sort(Cost_Test);
139 |     Cost_Test =Cost_Test(SortOrder);
140 |     BestCost_Test(it) = Cost_Test(1);
141 |     % Show Iteration Information
142 |     disp(['epoch ' num2str(it) ', cost on train data = ' num2str(BestCost_Train(it)) ', cost on test data = ' num2str(BestCost_Test(it))]);
143 |     
144 | end
145 | %% Results
146 | figure;
147 | semilogx(BestCost_Train,'LineWidth',2);
148 | xlabel('epochs');
149 | ylabel('train costs');
150 | grid on;
151 | 
152 | figure;
153 | semilogx(BestCost_Test,'LineWidth',2);
154 | xlabel('epochs');
155 | ylabel('test costs');
156 | grid on;
157 | 


--------------------------------------------------------------------------------
/ICA/InterEmpireCompetition.m:
--------------------------------------------------------------------------------
 1 | function emp=InterEmpireCompetition(emp)
 2 |     if numel(emp)==1
 3 |         return;
 4 |     end
 5 |     global ICASettings;
 6 |     alpha=ICASettings.alpha;
 7 |     TotalCost=[emp.TotalCost];
 8 |     
 9 |     [~, WeakestEmpIndex]=max(TotalCost);
10 |     WeakestEmp=emp(WeakestEmpIndex);
11 |     
12 |     P=exp(-alpha*TotalCost/max(TotalCost));
13 |     P(WeakestEmpIndex)=0;
14 |     P=P/sum(P);
15 |     if any(isnan(P))
16 |         P(isnan(P))=0;
17 |         if all(P==0)
18 |             P(:)=1;
19 |         end
20 |         P=P/sum(P);
21 |     end
22 |         
23 |     if WeakestEmp.nCol>0
24 |         [~, WeakestColIndex]=max([WeakestEmp.Col.Cost]);
25 |         WeakestCol=WeakestEmp.Col(WeakestColIndex);
26 |         WinnerEmpIndex=RouletteWheelSelection(P);
27 |         WinnerEmp=emp(WinnerEmpIndex);
28 |         WinnerEmp.Col(end+1)=WeakestCol;
29 |         WinnerEmp.nCol=WinnerEmp.nCol+1;
30 |         emp(WinnerEmpIndex)=WinnerEmp;
31 |         WeakestEmp.Col(WeakestColIndex)=[];
32 |         WeakestEmp.nCol=WeakestEmp.nCol-1;
33 |         emp(WeakestEmpIndex)=WeakestEmp;
34 |     end
35 |     
36 |     if WeakestEmp.nCol==0
37 |         
38 |         WinnerEmpIndex2=RouletteWheelSelection(P);
39 |         WinnerEmp2=emp(WinnerEmpIndex2);
40 |         
41 |         WinnerEmp2.Col(end+1)=WeakestEmp.Imp;
42 |         WinnerEmp2.nCol=WinnerEmp2.nCol+1;
43 |         emp(WinnerEmpIndex2)=WinnerEmp2;
44 |         
45 |         emp(WeakestEmpIndex)=[];
46 |     end
47 |     
48 | end


--------------------------------------------------------------------------------
/ICA/IntraEmpireCompetition.m:
--------------------------------------------------------------------------------
 1 | function emp=IntraEmpireCompetition(emp)
 2 |     nEmp=numel(emp);
 3 |     
 4 |     for k=1:nEmp
 5 |         for i=1:emp(k).nCol
 6 |             if emp(k).Col(i).Cost<emp(k).Imp.Cost
 7 |                 imp=emp(k).Imp;
 8 |                 col=emp(k).Col(i);
 9 |                 
10 |                 emp(k).Imp=col;
11 |                 emp(k).Col(i)=imp;
12 |             end
13 |         end
14 |     end
15 | end


--------------------------------------------------------------------------------
/ICA/RouletteWheelSelection.m:
--------------------------------------------------------------------------------
1 | function i=RouletteWheelSelection(P)
2 |     r=rand;
3 |     
4 |     C=cumsum(P);
5 |     
6 |     i=find(r<=C,1,'first');
7 | end


--------------------------------------------------------------------------------
/ICA/UpdateTotalCost.m:
--------------------------------------------------------------------------------
 1 | function emp=UpdateTotalCost(emp)
 2 |     global ICASettings;
 3 |     zeta=ICASettings.zeta;
 4 |     
 5 |     nEmp=numel(emp);
 6 |     
 7 |     for k=1:nEmp
 8 |         if emp(k).nCol>0
 9 |             emp(k).TotalCost=emp(k).Imp.Cost+zeta*mean([emp(k).Col.Cost]);
10 |         else
11 |             emp(k).TotalCost=emp(k).Imp.Cost;
12 |         end
13 |     end
14 | end


--------------------------------------------------------------------------------
/ICA/mape_calc.m:
--------------------------------------------------------------------------------
 1 | function MAPE_train = mape_calc(voroodi,trainset)
 2 |         v = voroodi(1:50);
 3 |         v0 = voroodi(51:60);
 4 |         v = reshape(v,10,5);
 5 |         v0 = reshape(v0,10,1);
 6 |         w = voroodi(61:70);
 7 |         w0 = voroodi(71);
 8 |         target = trainset(:,6);
 9 |         output = zeros(size(trainset,1),1);
10 |         for j=1:size(trainset,1)
11 |             x = trainset(j,1:5);
12 |             z_in = x*v'+v0';
13 |             z = max(0,z_in);
14 |             y = w*z' + w0;
15 |             output(j) = max(0,y);
16 |         end
17 |         MAPE_train = sum(abs((output-target))./target)/size(trainset,1);
18 |     
19 | end


--------------------------------------------------------------------------------
/ICA/testset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/ICA/testset.mat


--------------------------------------------------------------------------------
/ICA/trainset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/ICA/trainset.mat


--------------------------------------------------------------------------------
/ICA/weight_mat.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/ICA/weight_mat.mat


--------------------------------------------------------------------------------
/NN/data.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/NN/data.mat


--------------------------------------------------------------------------------
/NN/leaky_grad.m:
--------------------------------------------------------------------------------
 1 | function out = leaky_grad(X)
 2 | 
 3 | for i=1:size(X,1)
 4 |     for j = 1:10
 5 |         if X(i,j) >= 0
 6 |             out(i,j) = 1;
 7 |         else
 8 |             out(i,j) = 1/20;
 9 |         end
10 |     end
11 | end
12 | end


--------------------------------------------------------------------------------
/NN/leaky_relu.m:
--------------------------------------------------------------------------------
 1 | function out=leaky_relu(X)
 2 | 
 3 | for i=1:size(X,1)
 4 |     for j = 1:10
 5 |         if X(i,j) >= 0
 6 |             out(i,j) = X(i,1);
 7 |         else
 8 |             out(i,j) = 1/20*X(i,1);
 9 |         end
10 |     end
11 | end
12 | end
13 | 
14 | 
15 | 


--------------------------------------------------------------------------------
/NN/main.m:
--------------------------------------------------------------------------------
 1 | clc;
 2 | close all
 3 | clear all
 4 | %% preparing dataset %%
 5 | load('data.mat');
 6 | dataset = dataset1./100;
 7 | train_mat = dataset(1:floor(0.8*size(dataset,1)) , :);
 8 | test_mat = dataset(floor(0.8*size(dataset,1))+1 : size(dataset,1)  , :);
 9 | 
10 | train_data = train_mat(: , 1:5);
11 | X = train_data(1:5700,:);
12 | validX = train_data(5701:6341 ,:);
13 | train_tar = train_mat(: , 6);
14 | T = train_tar(1:5700,:);
15 | validtar = train_tar(5701:6341 ,:);
16 | test_data = test_mat(: , 1:5);
17 | test_tar = test_mat(: , 6);
18 | %% parameters %%
19 | max_epoch = 50;
20 | alpha = 0.000001;
21 | %% initial values %%
22 | m = size(X , 1);
23 | n = size(X , 2);
24 | h = 10; %% # of hidden units
25 | 
26 | %% training section %%
27 | for q = 1:50
28 |     q
29 |     W1 = 0.1*rand(n , h); %% 5*10
30 |     W2 = 0.1*rand(h , 1); %% 10*1
31 |     B1 = 0.1*rand(1 , h); %% 1*10
32 |     B2 = 0.1*rand(1 , 1); %% 1*1
33 |     for i = 1:max_epoch
34 |         i
35 |         %% feedforward %%
36 |         U = train_data*W1;
37 |         Z_in = bsxfun(@plus, U' , B1');
38 |         Z_in = Z_in';
39 |         Z = leaky_relu(Z_in);
40 |         UU = Z*W2;
41 |         Y_in = bsxfun(@plus, UU' , B2);
42 |         Y_in = Y_in';
43 |         Y = relu(Y_in);
44 |         %% backpropagation %%
45 |         delta_out = train_tar - Y;
46 |         dif_Y = relu_grad(Y_in);
47 |         delta_output = delta_out.*dif_Y;
48 |         deltaW2 = alpha*delta_output'*Z;
49 |         deltaB2 = alpha*1/m * sum(delta_out);
50 |         W2 = W2 + deltaW2';
51 |         B2 = B2 + deltaB2;
52 |         
53 |         A = delta_output*W2';
54 |         dif_Z = leaky_grad(Z_in);
55 |         B = bsxfun(@times, A ,dif_Z);
56 |         deltaW1 = alpha*train_data'*B;
57 |         deltaB1 = alpha*1/m * sum(B);
58 |         W1 = W1 + deltaW1;
59 |         B1 = B1 + deltaB1;
60 |         %% evaluation
61 |         yhat = predict(train_data,W1,W2,B1,B2);
62 |         yhatt = predict(validX,W1,W2,B1,B2);
63 |         RMSE(1,i) = rsme(yhat , train_tar);
64 |         RMSE2(1,i) = rsme(yhatt , validtar);
65 |         r1 = reshape(W1,1,50);
66 |         r2 = reshape(W2,1,10);
67 |     end
68 |     Weights(q,:) = [r1,r2,B1,B2];
69 |     epo = [1:max_epoch];
70 |     figure
71 |     plot(epo,RMSE,'-b')
72 |     hold on
73 |     plot(epo,RMSE2 , '--r')
74 | end
75 | 


--------------------------------------------------------------------------------
/NN/predict.m:
--------------------------------------------------------------------------------
 1 | function out = predict(X,W1,W2,B1,B2)
 2 | 
 3 | U = X*W1;
 4 | Z1 = bsxfun(@plus , U' , B1');
 5 | Z1 = Z1';
 6 | Z = leaky_relu(Z1);
 7 | UU = Z*W2;
 8 | Y1 = bsxfun(@plus, UU' , B2);
 9 | Y1 = Y1';
10 | out = Y1;
11 | 
12 | end
13 | 


--------------------------------------------------------------------------------
/NN/relu.m:
--------------------------------------------------------------------------------
1 | function out = relu(X)
2 | 
3 | for i=1:size(X,1) 
4 |     a(i,1) = max(0,X(i,1));
5 |     out = a(:,1);
6 | end
7 | end


--------------------------------------------------------------------------------
/NN/relu_grad.m:
--------------------------------------------------------------------------------
 1 | function out = relu_grad(X)
 2 | 
 3 | for i=1:size(X,1) 
 4 |     if X(i,1) >= 0
 5 |         out(i,1) = 1;
 6 |     else
 7 |         out(i,1) = 0;
 8 |     end
 9 | end
10 | end


--------------------------------------------------------------------------------
/NN/rsme.m:
--------------------------------------------------------------------------------
 1 | function out = rsme(yhat , train_tar)
 2 | 
 3 | a = 0;
 4 | for i = 1:size(train_tar,1)
 5 |     a = a + (yhat(i,1) - train_tar(i,1))^2;
 6 | end
 7 | b = a/size(train_tar,1);
 8 | out = sqrt(b); 
 9 | end
10 |     


--------------------------------------------------------------------------------
/PSO/main.m:
--------------------------------------------------------------------------------
  1 | clc;
  2 | clear;
  3 | close all;
  4 | 
  5 | %% Problem Definition
  6 | %% loading dataset %%
  7 | load('Weight_mat.mat')
  8 | load('trainset.mat')
  9 | load('testset.mat')
 10 | 
 11 | var_num=71;            
 12 | VarSize=[1 var_num];  
 13 | VarMin=-5;        
 14 | VarMax= 5;       
 15 | %% PSO Parameters
 16 | max_epoch=100;      
 17 | ini_pop=50;        
 18 | 
 19 | % Constriction Coefficients
 20 | phi1=2.1;
 21 | phi2=2.1;
 22 | phi=phi1+phi2;
 23 | khi=2/(phi-2+sqrt(phi^2-4*phi));
 24 | w=khi;          % Inertia Weight
 25 | wdamp=0.99;        % Inertia Weight Damping Ratio
 26 | c1=khi*phi1;    % Personal Learning Coefficient
 27 | c2=khi*phi2;    % Global Learning Coefficient
 28 | 
 29 | % Velocity Limits
 30 | VelMax=0.1*(VarMax-VarMin);
 31 | VelMin=-VelMax;
 32 | %% Initialization
 33 | 
 34 | empty_particle.Position=[];
 35 | empty_particle.Cost=[];
 36 | empty_particle.Velocity=[];
 37 | empty_particle.Best.Position=[];
 38 | empty_particle.Best.Cost=[];
 39 | 
 40 | particle=repmat(empty_particle,ini_pop,1);  
 41 | GlobalBest.Cost=inf;
 42 | Cost_Test= zeros(50,1);
 43 | for i=1:ini_pop
 44 |     
 45 |     % Initialize Position
 46 |     particle(i).Position= WEIGHTS(i ,:);
 47 |     
 48 |     % Initialize Velocity
 49 |     particle(i).Velocity=zeros(VarSize);
 50 |     
 51 |     % Evaluation
 52 |     particle(i).Cost=mape_calc(particle(i).Position,trainset);
 53 |     Cost_Test(i)=mape_calc(particle(i).Position,testset);
 54 |     % Update Personal Best
 55 |     particle(i).Best.Position=particle(i).Position;
 56 |     particle(i).Best.Cost=particle(i).Cost;
 57 |     
 58 |     % Update Global Best
 59 |     if particle(i).Best.Cost<GlobalBest.Cost
 60 |         
 61 |         GlobalBest=particle(i).Best;
 62 |         
 63 |     end
 64 |     
 65 | end
 66 | 
 67 | BestCost_Train=zeros(max_epoch,1);
 68 | BestCost_Test=zeros(max_epoch,1);
 69 | [~, SortOrder]=sort(Cost_Test);
 70 | Cost_Test =Cost_Test(SortOrder);
 71 | %% PSO Main Loop
 72 | for it=1:max_epoch
 73 |     for i=1:ini_pop
 74 |         % Update Velocity
 75 |         particle(i).Velocity = w*particle(i).Velocity ...
 76 |             +c1*rand(VarSize).*(particle(i).Best.Position-particle(i).Position) ...
 77 |             +c2*rand(VarSize).*(GlobalBest.Position-particle(i).Position);
 78 |         
 79 |         % Apply Velocity Limits
 80 |         particle(i).Velocity = max(particle(i).Velocity,VelMin);
 81 |         particle(i).Velocity = min(particle(i).Velocity,VelMax);
 82 |         
 83 |         % Update Position
 84 |         particle(i).Position = particle(i).Position + particle(i).Velocity;
 85 |         IsOutside=(particle(i).Position<VarMin | particle(i).Position>VarMax);
 86 |         particle(i).Velocity(IsOutside)=-particle(i).Velocity(IsOutside);
 87 |         
 88 |         % Apply Position Limits
 89 |         particle(i).Position = max(particle(i).Position,VarMin);
 90 |         particle(i).Position = min(particle(i).Position,VarMax);
 91 |         
 92 |         % Evaluation
 93 |         particle(i).Cost = mape_calc(particle(i).Position,trainset);
 94 |         for l= 1:ini_pop
 95 |           Cost_Test(l)=mape_calc(particle(l).Position,testset);
 96 |         end
 97 |         [~, SortOrder]=sort(Cost_Test);
 98 |         Cost_Test =Cost_Test(SortOrder);
 99 |         BestCost_Test(it) = Cost_Test(1);
100 |         % Update Personal Best
101 |         if particle(i).Cost<particle(i).Best.Cost
102 |             
103 |             particle(i).Best.Position=particle(i).Position;
104 |             particle(i).Best.Cost=particle(i).Cost;
105 |             
106 |             % Update Global Best
107 |             if particle(i).Best.Cost<GlobalBest.Cost
108 |                 
109 |                 GlobalBest=particle(i).Best;
110 |                 
111 |             end
112 |             
113 |         end
114 |         
115 |     end
116 |     
117 |     BestCost_Train(it)=GlobalBest.Cost;
118 |     disp(['epochs ' num2str(it) ', cost on train data = ' num2str(BestCost_Train(it)) ', cost on test data = ' num2str(BestCost_Test(it))]);
119 |     w=w*wdamp;
120 |     
121 | end
122 | 
123 | %% Results
124 | figure;
125 | semilogx(BestCost_Train,'LineWidth',2);
126 | xlabel('epochs');
127 | ylabel('train costs');
128 | grid on;
129 | 
130 | figure;
131 | semilogx(BestCost_Test,'LineWidth',2);
132 | xlabel('epochs');
133 | ylabel('test costs');
134 | grid on;


--------------------------------------------------------------------------------
/PSO/mape_calc.m:
--------------------------------------------------------------------------------
 1 | function MAPE_train = mape_calc(weight,trainset)
 2 |         W1 = reshape(weight(1:50),10,5);
 3 |         B1 = reshape(weight(51:60),10,1);
 4 |         W2 = weight(61:70);
 5 |         B2 = weight(71);
 6 |         target = trainset(:,6);
 7 |         output = zeros(size(trainset,1),1);
 8 |         for j=1:size(trainset,1)
 9 |             x = trainset(j,1:5);
10 |             z_in = x*W1'+B1';
11 |             z = max(0,z_in);
12 |             y = W2*z' + B2;
13 |             output(j) = max(0,y);
14 |         end
15 |         MAPE_train = 100*sum(abs((output-target))./target)/size(trainset,1);
16 |     
17 | end


--------------------------------------------------------------------------------
/PSO/testset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/PSO/testset.mat


--------------------------------------------------------------------------------
/PSO/trainset.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/PSO/trainset.mat


--------------------------------------------------------------------------------
/PSO/weight_mat.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/mohammad-AJP/Neural-Network-weight-improvement-using-optimization-algorithms/fa7b95d1c8f10f63caa4579ac92ec906016c4f12/PSO/weight_mat.mat


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Neural-Network-weight-improvement-using-optimization-algorithms
2 | Gray Wolf Optimization (GWO), Imperialist Competitive Algorithm (ICA) and Particle Swarm Optimization (PSO) are used to improve the weights achieved by a Neural Network trained with Gradient Descent method. 
3 | the data used in this repo is NASA trace network gathered in july. the data is processed using "data_pre.m" which from adds the number of requests and responses of every 5 minutes so that we have a number for every five minutes from the day 1 to the day 28 of july. then 6 of these numbers creates a row of the data in which the first 5 are inputs and the last number is treated as the target data.  
4 | a Neural network is used to predict the achieved time series which has 7927 rows of data which exploits gradient descent method.  
5 | then the weights are optimized through GWO, ICA and PSO algorithms to improve the total performance of the system. the data and codes are categorized into 4 separate directories. 
6 | 


--------------------------------------------------------------------------------
/dataset_pre.m:
--------------------------------------------------------------------------------
 1 | clc
 2 | clear all
 3 | close all
 4 | 
 5 | load('thetable.mat');
 6 | x = table2cell(accesslogJul95);
 7 | u = cell2mat(x);
 8 | u(:,1) = [];
 9 | u(:,12) = ' ';
10 | u(: , [3:12,15,18]) = [];
11 | m1 = u(:,[1,2]);
12 | m2 = u(:,[3,4]);
13 | m3 = u(:,[5,6]);
14 | for i = 1:size(u,1)
15 |     n1(i,1) = str2num(m1(i,[1,2]));
16 |     n2(i,1) = str2num(m2(i,[1,2]));
17 |     n3(i,1) = str2num(m3(i,[1,2]));
18 | end
19 | data = [n1,n2,n3];
20 | 
21 | req = [];
22 | day =1;
23 | hour =0;
24 | ii = 1;
25 | rows = size(data,1);
26 | t1=0;t2=0;t3=0;t4=0;t5=0;t6=0;t7=0;t8=0;t9=0;t10=0;t11=0;t12=0;
27 | while ii <= rows
28 |     while data(ii,1) == day
29 |         while data(ii,2) == hour
30 |             if (0<=data(ii,3))&&(data(ii,3)<=4)
31 |                 t1 = t1+1;
32 |             elseif (5<=data(ii,3))&&(data(ii,3)<=9)
33 |                 t2 = t2+1;
34 |             elseif (10<=data(ii,3))&&(data(ii,3)<=14)
35 |                 t3 = t3+1;
36 |             elseif (15<=data(ii,3))&&(data(ii,3)<=19)
37 |                 t4 = t4+1;
38 |             elseif (20<=data(ii,3))&&(data(ii,3)<=24)
39 |                 t5 = t5+1;
40 |             elseif (25<=data(ii,3))&&(data(ii,3)<=29)
41 |                 t6 = t6+1;
42 |             elseif (30<=data(ii,3))&&(data(ii,3)<=34)
43 |                 t7 = t7+1;
44 |             elseif (35<=data(ii,3))&&(data(ii,3)<=39)
45 |                 t8 = t8+1;
46 |             elseif (40<=data(ii,3))&&(data(ii,3)<=44)
47 |                 t9 = t9+1;
48 |             elseif (45<=data(ii,3))&&(data(ii,3)<=49)
49 |                 t10 = t10+1;
50 |             elseif (50<=data(ii,3))&&(data(ii,3)<=54)
51 |                 t11 = t11+1;
52 |             else
53 |                 t12 = t12+1;
54 |             end
55 |             ii = ii+1;
56 |         end
57 |         hour = hour+1;
58 |         req = [req;t1,t2,t3,t4,t5,t6,t7,t8,t9,t10,t11,t12];
59 |         t1=0;t2=0;t3=0;t4=0;t5=0;t6=0;t7=0;t8=0;t9=0;t10=0;t11=0;t12=0;
60 |     end
61 |     hour = 0;
62 |     day = day+1;
63 | end
64 | 
65 | req1 = req';
66 | req_list = reshape(req1,1,[]);
67 | r=1;
68 | while r+5 <= 7932
69 |     pp_data(r,1:6) = req_list(1,r:r+5);
70 |     r = r+1;
71 | end
72 |      
73 | 
74 | vec = randperm(size(pp_data, 1));
75 | for i= 1:size(pp_data,1)
76 |     dataset1(i,:) = pp_data(vec(i) , :);
77 | end
78 | 


--------------------------------------------------------------------------------