├── README.md ├── datasets └── DB_normalized │ └── mnist.mat ├── acc.py ├── utils.py ├── main.py └── danmf_crfr.py /README.md: -------------------------------------------------------------------------------- 1 | # DANMF_CRFR 2 | Deep Autoencoder-like NMF with Contrastive Regularization and Feature Relationship preservation 3 | -------------------------------------------------------------------------------- /datasets/DB_normalized/mnist.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/navidsalahian/DANMF_CRFR/HEAD/datasets/DB_normalized/mnist.mat -------------------------------------------------------------------------------- /acc.py: -------------------------------------------------------------------------------- 1 | import scipy 2 | import numpy as np 3 | from sklearn.metrics import accuracy_score 4 | 5 | 6 | def cluster_acc(real_labels, labels): 7 | permutation = [] 8 | n_clusters = len(np.unique(real_labels)) 9 | labels = np.unique(labels, return_inverse=True)[1] 10 | for i in range(n_clusters): 11 | idx = labels == i 12 | if np.sum(idx) != 0: 13 | new_label = scipy.stats.mode(real_labels[idx])[0][0] 14 | permutation.append(new_label) 15 | new_labels = [permutation[label] for label in labels] 16 | return accuracy_score(real_labels, new_labels) 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | 2 | import numpy as np 3 | from scipy import io 4 | from sklearn.model_selection import train_test_split 5 | from sklearn.neighbors import NearestNeighbors 6 | from sklearn.metrics import mean_squared_error, pairwise_distances 7 | import pandas as pd 8 | 9 | def normalize(X, n): 10 | 11 | Xmin = np.min(X, axis=1) 12 | Xmax = np.max(X, axis=1) 13 | mmin = np.matlib.repmat(Xmin.T, n, 1) 14 | mmax = np.matlib.repmat(Xmax.T, n, 1) 15 | Xu = X - mmin.T 16 | Xd = mmax.T - mmin.T 17 | X = Xu / np.maximum(Xd, 10**-10) 18 | return X.T 19 | 20 | def preproccessing(args): 21 | 22 | print("\nPre-processing started.\n") 23 | data = io.loadmat(args.dataset_path) 24 | X = data['X'] 25 | # get real labels 26 | Y = data['Y'] 27 | d, n = X.shape 28 | near_n = NearestNeighbors(n_neighbors=args.k_neigh, metric='euclidean').fit(X.T) 29 | S = np.zeros((n, n)) 30 | dist, indices = near_n.kneighbors(X.T) 31 | indices = indices[:, 1:] 32 | dist = dist[:, 1:] 33 | for i in range(indices.shape[0]): 34 | for j in range(indices.shape[1]): 35 | S[i, indices[i, j]] = np.exp(-(dist[i, j] ** 2) / args.delta) # Heat kernel similarity 36 | 37 | D = np.sum(S, axis=1) 38 | D = np.diag(D) 39 | L = D - S 40 | data = {'X': X, 'Y': Y, 'D': D, 'S': S, 'L': L} 41 | return data -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | from danmf_crfr import DANMF_CRFR 3 | from tqdm import tqdm 4 | from utils import preproccessing 5 | 6 | 7 | class Namespace: 8 | def __init__(self, **kwargs): 9 | self.__dict__.update(kwargs) 10 | 11 | 12 | def main(lambda0, lambda1, lambda2): 13 | 14 | layer = [120, 100, 80] 15 | dataset = "mnist" 16 | lambda0 = lambda0 # DIS SIMILARITY 17 | lambda1 = lambda1 # SIMILARITY 18 | lambda2 = lambda2 # FRP 19 | 20 | args = Namespace( 21 | k_neigh=5, 22 | k_kmeans=10, 23 | delta=1000, 24 | dataset=dataset, 25 | calculate_loss=False, 26 | dataset_path="datasets/DB_normalized/"+dataset+".mat", 27 | iterations=500, 28 | pre_iterations=500, 29 | lamb0=lambda0, 30 | lamb1=lambda1, 31 | lamb2=lambda2, 32 | layers=layer, 33 | seed=None) 34 | 35 | data = preproccessing(args) 36 | model = DANMF_CRFR(data, args) 37 | model.pre_training() 38 | return model.training() 39 | 40 | 41 | # range for obtaining values as grid search strategy 42 | lambda_0_arr = [0, 0.000001, 0.00001, 0.0001, 0.001] # dis 43 | lambda_1_arr = [0, 0.01, 0.1, 1, 10, 100, 1000] # sim 44 | lambda_2_arr = [0, 0.5, 0.75, 1, 1.25, 1.5, 2] # FRP 45 | 46 | nmi_arr, ari_arr, acc_arr = [], [], [] 47 | 48 | if __name__ == "__main__": 49 | # run the code n times for achieving standard deviation and average measurements 50 | for i in tqdm(range(3)): 51 | NMI, ARI, ACC = main(0.000001, 5, 0.1) 52 | nmi_arr.append(NMI) 53 | ari_arr.append(ARI) 54 | acc_arr.append(ACC) 55 | 56 | print(f"Average NMI: {np.average(np.array(nmi_arr))}, ARI: {np.average(np.array(ari_arr))}, ACC: {np.average(np.array(acc_arr))}") 57 | print(f"STD NMI: {np.std(np.array(nmi_arr))}, ARI: {np.std(np.array(ari_arr))}, ACC: {np.std(np.array(acc_arr))}") 58 | -------------------------------------------------------------------------------- /danmf_crfr.py: -------------------------------------------------------------------------------- 1 | from tqdm import tqdm 2 | from sklearn.decomposition import NMF as NMF_SKL 3 | from sklearn.metrics import normalized_mutual_info_score, adjusted_rand_score 4 | from acc import cluster_acc 5 | import numpy as np 6 | from sklearn.metrics import mean_squared_error, pairwise_distances 7 | from sklearn.cluster import KMeans 8 | 9 | 10 | 11 | 12 | class DANMF_CRFR(): 13 | 14 | def __init__(self, data, args): 15 | self.X = data['X'] 16 | self.Y = data['Y'][0] 17 | self.XXT = self.X @ self.X.T 18 | self.D = data['D'] 19 | self.S = data['S'] 20 | self.L = data['L'] 21 | self.args = args 22 | self.pre_iter = args.pre_iterations 23 | self.lamb0 = args.lamb0 24 | self.lamb1 = args.lamb1 25 | self.lamb2 = args.lamb2 26 | self.U_s = [] 27 | self.V_s = [] 28 | self.Z = [] 29 | self.p = len(args.layers) 30 | self.Dis = pairwise_distances(self.X.T, self.X.T) 31 | self.k_kmeans = args.k_kmeans 32 | 33 | # Target matrix 34 | def setup_z(self, i): 35 | if i == 0: 36 | self.Z = self.X 37 | else: 38 | self.Z = self.V_s[i-1] 39 | 40 | # Shallow Autoencoder-like NMF for pre-training 41 | def auto_encoder_pretrain(self, i): 42 | U = np.random.rand(self.Z.shape[0], self.args.layers[i]) 43 | V = np.random.rand(self.args.layers[i], self.Z.shape[1]) 44 | for i in range(self.args.pre_iterations): 45 | u_u = 2 * self.Z @ V.T 46 | u_d = U @ V @ V.T + self.Z @ self.Z.T @ U 47 | U = U * (u_u / np.maximum(u_d, 10 ** -10)) 48 | 49 | v_u = 2 * U.T @ self.Z 50 | v_d = U.T @ U @ V + V 51 | V = V * (v_u / np.maximum(v_d, 10 ** -10)) 52 | return U, V 53 | 54 | # Pre-training process 55 | def pre_training(self): 56 | print("\nLayer pre-training started. \n") 57 | for i in tqdm(range(self.p), desc="Layers trained: ", leave=True): 58 | self.setup_z(i) 59 | U, V = self.auto_encoder_pretrain(i) 60 | self.U_s.append(U) 61 | self.V_s.append(V) 62 | 63 | # update Basis Matrix (U or W) according to Paper 64 | def update_U(self, i): 65 | 66 | if i == 0: 67 | # the next line is just for simplifying U in following lines 68 | U = self.U_s[i] @ self.PHI[i + 1] 69 | bottom = self.U_s[i] @ self.PHI[i+1] @ self.Vp @ self.Vp.T @ self.PHI[i+1].T 70 | bottom = bottom + self.X @ self.X.T @ self.U_s[i] @ self.PHI[i+1] @ self.PHI[i+1].T + 2 * self.lamb2 ** 2 * U @ U.T @ U @ self.PHI[i+1].T 71 | top = 2 * self.X @ self.Vp.T @ self.PHI[i+1].T + 2 * self.lamb2 * self.XXT @ U @ self.PHI[i+1].T 72 | self.U_s[0] = self.U_s[0] * (top / np.maximum(bottom, 10 ** -10)) 73 | else: 74 | # the next line is just for simplifying U in following lines 75 | U = self.SII @ self.U_s[i] @ self.PHI[i + 1] 76 | bottom = self.SII.T @ self.SII @ self.U_s[i] @ self.PHI[i + 1] @ self.Vp @ self.Vp.T @ self.PHI[i + 1].T 77 | bottom = bottom + self.SII.T @ self.X @ self.X.T @ self.SII @ self.U_s[i] @ self.PHI[i + 1] @ self.PHI[i + 1].T \ 78 | + 2 * self.lamb2**2 * self.SII.T @ U @ U.T @ U @ self.PHI[i+1].T 79 | top = 2 * self.SII.T @ self.X @ self.Vp.T @ self.PHI[i + 1].T + 2 * self.lamb2 * self.SII.T @ self.XXT @ U @ self.PHI[i+1].T 80 | self.U_s[i] = self.U_s[i] * (top / np.maximum(bottom, 10 ** -10)) 81 | return 82 | 83 | # Update coefficient Matrix (H or V) according with paper 84 | def update_V(self, i): 85 | if i == self.p - 1: 86 | top = 2 * self.SII.T @ self.X + 2 * self.lamb1 * self.V_s[-1] @ self.S 87 | bottom = self.SII.T @ self.SII @ self.V_s[-1] + self.V_s[-1] + 2 * self.lamb1 * self.V_s[-1] @ self.D + self.lamb0 * self.V_s[-1] @ self.Dis 88 | self.V_s[i] = np.multiply(self.V_s[i], top) / np.maximum(bottom, 10 ** -10) 89 | return 90 | 91 | # Set PHI ( Φ ) Matrix based on Ui+1 to Up 92 | def set_PHI(self): 93 | self.PHI = [None for _ in range(self.p + 1)] 94 | self.PHI[self.p] = np.eye(self.args.layers[self.p - 1]) 95 | for i in range(self.p - 1, -1, -1): # Question 96 | self.PHI[i] = np.dot(self.U_s[i], self.PHI[i + 1]) 97 | 98 | # Set PSI ( Ψ ) Matrix based on U1 to Ui-1 99 | def set_SII(self, i): 100 | if i == 0: 101 | self.SII = self.U_s[0] 102 | else: 103 | self.SII = self.SII.dot(self.U_s[i]) 104 | 105 | 106 | # Main training process after pre-training 107 | def training(self): 108 | 109 | print("\n\nTraining started. \n") 110 | self.loss = np.zeros(self.args.iterations) 111 | for iteration in tqdm(range(self.args.iterations), desc="Training pass: ", leave=True): 112 | 113 | self.Vp = self.V_s[self.p-1] 114 | for i in range(self.p): 115 | self.set_PHI() 116 | self.update_U(i) 117 | self.set_PHI() 118 | self.set_SII(i) 119 | self.update_V(i) 120 | if self.args.calculate_loss: 121 | self.calculate_cost(iteration) 122 | 123 | # normalizing Vp 124 | sum = np.sum(self.V_s[-1], axis=0) 125 | normalized_Vp = self.V_s[-1] / sum 126 | 127 | kmeans = KMeans(self.k_kmeans, random_state=20).fit(normalized_Vp.T) 128 | print(normalized_mutual_info_score(self.Y, kmeans.labels_), adjusted_rand_score(self.Y, kmeans.labels_), cluster_acc(self.Y, kmeans.labels_)) 129 | # return NMI, ARI, ACC 130 | return normalized_mutual_info_score(self.Y, kmeans.labels_), adjusted_rand_score(self.Y, kmeans.labels_), cluster_acc(self.Y, kmeans.labels_) 131 | 132 | 133 | --------------------------------------------------------------------------------