├── JDA └── JDA.py ├── README.md ├── TCA └── TCA.py └── TrAdaboost └── TrAaboost.py /JDA/JDA.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @author: Infaraway 4 | @time: 2018/3/17 20:40 5 | @Function: 6 | """ 7 | from Demos.dde.ddeclient import sl 8 | from sklearn.tree import DecisionTreeClassifier 9 | import numpy as np 10 | 11 | 12 | class JDA: 13 | 14 | def __init__(self, base_classifier=DecisionTreeClassifier(), dim=5, N=10, k=200, lmbda=1.0, kernel='liner', kernel_param=1.0): 15 | self.base_classifier = base_classifier 16 | self.dim = dim 17 | self.N = N 18 | self.k = k 19 | self.lmbda = lmbda 20 | self.kernel = kernel 21 | self.kernel_param = kernel_param 22 | self.Zs = None 23 | self.Zt = None 24 | 25 | def fit_transform(self, x_source, x_target, y_source): 26 | self.base_classifier.fit(x_source, y_source) 27 | # 伪target加入伪标签 28 | y_target0 = self.base_classifier.predict(x_target) 29 | ns = x_source.shape[-1] 30 | nt = x_target.shape[-1] 31 | for i in range(self.N): 32 | 33 | Z, A = self._get_JDA_matrix(x_source, x_target, y_source, y_target0) 34 | Z = np.matmul(Z, np.diag(1 / np.sqrt(np.sum(np.square(Z), 0)))) 35 | self.Zs = Z[:, :ns] 36 | self.Zt = Z[:, ns:] 37 | 38 | self.base_classifier.fit(self.Zs.T, y_source) 39 | y_target0 = self.base_classifier.predict(self.Zt.T) 40 | return self.Zs.T, self.Zt.T 41 | 42 | def transform(self): 43 | pass 44 | 45 | def _kernel_func(self, x1, x2=None): 46 | n1, dim = x1.shape 47 | K = None 48 | if self.kernel == 'linear': 49 | if x2 is not None: 50 | K = np.dot(x2, x1.T) 51 | else: 52 | K = np.dot(x1, x1.T) 53 | elif self.kernel == 'rbf': 54 | if x2 is not None: 55 | n2 = np.shape(x2)[0] 56 | sum_x2 = np.sum(np.multiply(x2, x2), axis=1) 57 | sum_x2 = sum_x2.reshape((len(sum_x2), 1)) 58 | K = np.exp(-1 * (np.tile(np.sum(np.multiply(x1, x1), axis=1).T, (n2, 1)) + np.tile(sum_x2, (1, n1)) 59 | - 2 * np.dot(x2, x1.T)) / (dim * 2 * self.kernel_param)) 60 | else: 61 | P = np.sum(np.multiply(x1, x1), axis=1) 62 | P = P.reshape((len(P), 1)) 63 | K = np.exp(-1 * (np.tile(P.T, (n1, 1)) + np.tile(P, (1, n1)) - 2 * np.dot(x1, x1.T)) / (dim * 2 * self.kernel_param)) 64 | # more kernels can be added 65 | return K 66 | 67 | def _get_JDA_matrix(self, x_source, x_target, y_source, y_target0): 68 | X = np.vstack((x_source, x_target)) 69 | X = np.matmul(X, np.diag(1 / np.sqrt(np.sum(np.square(X), 0)))) 70 | 71 | m, n = np.shape(X) 72 | n_source = np.shape(x_source)[0] 73 | n_target = np.shape(x_target)[0] 74 | 75 | C = len(np.unique(y_source)) 76 | 77 | a = 1 / n_source * np.ones([n_source, 1]) 78 | b = -1 / n_target * np.ones([n_target, 1]) 79 | 80 | e = np.hstack((a, b)) 81 | M = np.matmul(e, e.T) * C 82 | N = 0 83 | if len(y_target0) != 0 and len(y_target0) == n_target: 84 | for c in np.reshape(np.unique(y_target0), 1): 85 | e = np.zeros([n, -1]) 86 | idx = [i for i, y in enumerate(y_source) if y == c] 87 | e[idx] = 1 / len(idx) 88 | 89 | e[np.where(np.isinf(e))[0]] = 0 90 | N = N + np.matmul(e, e.T) 91 | M = M + N 92 | M = M / np.sqrt(np.sum(np.diag(np.matmul(M.T, M)))) 93 | H = np.eye(n) - 1 / n * np.ones([n, n]) 94 | 95 | if "primal" == self.kernel: 96 | a = np.matmul(np.matmul(X, M), X.T) + self.lmbda * np.eye(m) 97 | b = np.matmul(np.matmul(X, H), X.T) 98 | eigenvalue, eigenvector = sl.eig(a, b) 99 | av = np.array(list(map(lambda item: np.abs(item), eigenvalue))) 100 | idx = np.argsort(av)[:self] 101 | _ = eigenvalue[idx] 102 | 103 | A = eigenvector[:, idx] 104 | Z = np.matmul(A.T, X) 105 | 106 | else: 107 | K = self._kernel_func(X) 108 | a = np.matmul(np.matmul(K, M), K.T) + self.lmbda * np.eye(n) 109 | b = np.matmul(np.matmul(K, H), K.T) 110 | eigenvalue, eigenvector = sl.eig(a, b) 111 | av = np.array(list(map(lambda item: np.abs(item), eigenvalue))) 112 | idx = np.argsort(av)[:self] 113 | _ = eigenvalue[idx] 114 | 115 | A = eigenvector[:, idx] 116 | Z = np.matmul(A.T, K) 117 | 118 | return Z, A 119 | 120 | 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # TransferLearning 2 | TransferLearning JDA and TrAdaboost 3 | 4 | - 有关于迁移学习的知识可以阅读知乎专栏的文章：https://zhuanlan.zhihu.com/p/30685086 5 | 6 | - 以及作者免费总结的非常好的综述：https://zhuanlan.zhihu.com/p/35352154 7 | -------------------------------------------------------------------------------- /TCA/TCA.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @author: Infaraway 4 | @time: 2018/4/22 13:58 5 | @Function: 6 | """ 7 | 8 | import numpy as np 9 | 10 | class TCA: 11 | 12 | def __init__(self, dim=5, kernel='rbf', kernel_param=1, mu=1): 13 | """ 14 | 15 | :param dim: 16 | :param kernel: 'rbf' | 'linear' | 'poly' 17 | :param kernel_param: 18 | :param mu: 19 | """ 20 | self.dim = dim 21 | self.kernel = kernel 22 | self.kernel_param = kernel_param 23 | self.mu = mu 24 | self.X = None 25 | self.V = None 26 | 27 | def _get_L(self, n_source, n_targetget): 28 | """ 29 | Get MMD matrix 30 | :param n_source: number of source samples 31 | :param n_targetget: number of target samples 32 | :return: 33 | """ 34 | L_ss = (1. / (n_source * n_source)) * np.full((n_source, n_source), 1) 35 | L_st = (-1. / (n_source * n_targetget)) * np.full((n_source, n_targetget), 1) 36 | L_ts = (-1. / (n_targetget * n_source)) * np.full((n_targetget, n_source), 1) 37 | L_tt = (1. / (n_targetget * n_targetget)) * np.full((n_targetget, n_targetget), 1) 38 | 39 | L_up = np.hstack((L_ss, L_st)) 40 | L_down = np.hstack((L_ts, L_tt)) 41 | L = np.vstack((L_up, L_down)) 42 | 43 | return L 44 | 45 | def _kernel_func(self, x1, x2=None): 46 | """ 47 | Calculate kernel for TCA 48 | :param x1: 49 | :param x2: 50 | :return: 51 | """ 52 | n1, dim = x1.shape 53 | K = None 54 | if x2 is not None: 55 | n2 = x2.shape[0] 56 | if self.kernel == 'linear': 57 | if x2 is not None: 58 | K = np.dot(x2, x1.T) 59 | else: 60 | K = np.dot(x1, x1.T) 61 | elif self.kernel == 'poly': 62 | if x2 is not None: 63 | K = np.power(np.dot(x1, x2.T), self.kernel_param) 64 | else: 65 | K = np.power(np.dot(x1, x1.T), self.kernel_param) 66 | elif self.kernel == 'rbf': 67 | if x2 is not None: 68 | sum_x2 = np.sum(np.multiply(x2, x2), axis=1) 69 | sum_x2 = sum_x2.reshape((len(sum_x2), 1)) 70 | K = np.exp(-1 * (np.tile(np.sum(np.multiply(x1, x1), axis=1).T, (n2, 1)) + np.tile(sum_x2, (1, n1)) 71 | - 2 * np.dot(x2, x1.T)) / (dim * 2 * self.kernel_param)) 72 | else: 73 | P = np.sum(np.multiply(x1, x1), axis=1) 74 | P = P.reshape((len(P), 1)) 75 | K = np.exp(-1 * (np.tile(P.T, (n1, 1)) + np.tile(P, (1, n1)) - 2 * np.dot(x1, x1.T)) / (dim * 2 * self.kernel_param)) 76 | # more kernels can be added 77 | return K 78 | 79 | def fit_transform(self, x_source, x_target): 80 | """ 81 | TCA main method. Wrapped from Sinno J. Pan and Qiang Yang's "Domain adaptation via 82 | transfer component ayalysis. IEEE TNN 2011" 83 | :param x_source: Source domain data feature matrix. 84 | :param x_target: Target domain data feature matrix. 85 | :return: 86 | """ 87 | n_source = x_source.shape[0] 88 | n_target = x_target.shape[0] 89 | self.X = np.vstack((x_source, x_target)) 90 | L = self._get_L(n_source, n_target) 91 | L[np.isnan(L)] = 0 92 | K = self._kernel_func(self.X) 93 | K[np.isnan(K)] = 0 94 | H = np.identity(n_source + n_target) - 1. / (n_source + n_target) * np.ones(shape=(n_source + n_target, 1)) * np.ones( 95 | shape=(n_source + n_target, 1)).T 96 | forPinv = self.mu * np.identity(n_source + n_target) + np.dot(np.dot(K, L), K) 97 | forPinv[np.isnan(forPinv)] = 0 98 | Kc = np.dot(np.dot(np.dot(np.linalg.pinv(forPinv), K), H), K) 99 | Kc[np.isnan(Kc)] = 0 100 | D, V = np.linalg.eig(Kc) 101 | eig_values = D.reshape(len(D), 1) 102 | eig_values_sorted = np.sort(eig_values[::-1], axis=0) 103 | index_sorted = np.argsort(-eig_values, axis=0) 104 | V = V[:, index_sorted] 105 | self.V = V.reshape((V.shape[0], V.shape[1])) 106 | x_source_tca = np.dot(K[:n_source, :], V) 107 | x_target_tca = np.dot(K[n_source:, :], V) 108 | 109 | x_source_tca = np.asarray(x_source_tca[:, :self.dim], dtype=float) 110 | x_target_tca = np.asarray(x_target_tca[:, :self.dim], dtype=float) 111 | 112 | return x_source_tca, x_target_tca 113 | 114 | def fit(self, x_target): 115 | """ 116 | change the other target data 117 | :param x_target: Out-of-sample target data feature matrix. 118 | :return: 119 | """ 120 | K_target = self._kernel_func(self.X, x_target) 121 | x_target_tca = np.dot(K_target, self.V) 122 | x_target_tca = x_target_tca[:, :self.dim] 123 | 124 | return x_target_tca 125 | 126 | 127 | if __name__ == '__main__': 128 | pass -------------------------------------------------------------------------------- /TrAdaboost/TrAaboost.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """ 3 | @author: Infaraway 4 | @time: 2018/4/16 10:47 5 | @Function: 6 | """ 7 | from sklearn.tree import DecisionTreeClassifier 8 | import numpy as np 9 | 10 | 11 | class TrAdaboost: 12 | def __init__(self, base_classifier=DecisionTreeClassifier(), N=10): 13 | self.base_classifier = base_classifier 14 | self.N = N 15 | self.beta_all = np.zeros([1, self.N]) 16 | self.classifiers = [] 17 | 18 | def fit(self, x_source, x_target, y_source, y_target): 19 | x_train = np.concatenate((x_source, x_target), axis=0) 20 | y_train = np.concatenate((y_source, y_target), axis=0) 21 | x_train = np.asarray(x_train, order='C') 22 | y_train = np.asarray(y_train, order='C') 23 | y_source = np.asarray(y_source, order='C') 24 | y_target = np.asarray(y_target, order='C') 25 | 26 | row_source = x_source.shape[0] 27 | row_target = x_target.shape[0] 28 | 29 | # 初始化权重 30 | weight_source = np.ones([row_source, 1]) / row_source 31 | weight_target = np.ones([row_target, 1]) / row_target 32 | weights = np.concatenate((weight_source, weight_target), axis=0) 33 | 34 | beta = 1 / (1 + np.sqrt(2 * np.log(row_source / self.N))) 35 | 36 | result = np.ones([row_source + row_target, self.N]) 37 | for i in range(self.N): 38 | weights = self._calculate_weight(weights) 39 | self.base_classifier.fit(x_train, y_train, sample_weight=weights[:, 0]) 40 | self.classifiers.append(self.base_classifier) 41 | 42 | result[:, i] = self.base_classifier.predict(x_train) 43 | error_rate = self._calculate_error_rate(y_target, 44 | result[row_source:, i], 45 | weights[row_source:, :]) 46 | 47 | print("Error Rate in target data: ", error_rate, 'round:', i, 'all_round:', self.N) 48 | 49 | if error_rate > 0.5: 50 | error_rate = 0.5 51 | if error_rate == 0: 52 | self.N = i 53 | print("Early stopping...") 54 | break 55 | self.beta_all[0, i] = error_rate / (1 - error_rate) 56 | 57 | # 调整 target 样本权重 正确样本权重变大 58 | for t in range(row_target): 59 | weights[row_source + t] = weights[row_source + t] * np.power(self.beta_all[0, i], -np.abs(result[row_source + t, i] - y_target[t])) 60 | # 调整 source 样本 错分样本变大 61 | for s in range(row_source): 62 | weights[s] = weights[s] * np.power(beta, np.abs(result[s, i] - y_source[s])) 63 | 64 | def predict(self, x_test): 65 | result = np.ones([x_test.shape[0], self.N + 1]) 66 | predict = [] 67 | 68 | i = 0 69 | for classifier in self.classifiers: 70 | y_pred = classifier.predict(x_test) 71 | result[:, i] = y_pred 72 | i += 1 73 | 74 | for i in range(x_test.shape[0]): 75 | left = np.sum(result[i, int(np.ceil(self.N / 2)): self.N] * 76 | np.log(1 / self.beta_all[0, int(np.ceil(self.N / 2)):self.N])) 77 | 78 | right = 0.5 * np.sum(np.log(1 / self.beta_all[0, int(np.ceil(self.N / 2)): self.N])) 79 | 80 | if left >= right: 81 | predict.append(1) 82 | else: 83 | predict.append(0) 84 | return predict 85 | 86 | def predict_prob(self, x_test): 87 | result = np.ones([x_test.shape[0], self.N + 1]) 88 | predict = [] 89 | 90 | i = 0 91 | for classifier in self.classifiers: 92 | y_pred = classifier.predict(x_test) 93 | result[:, i] = y_pred 94 | i += 1 95 | 96 | for i in range(x_test.shape[0]): 97 | left = np.sum(result[i, int(np.ceil(self.N / 2)): self.N] * 98 | np.log(1 / self.beta_all[0, int(np.ceil(self.N / 2)):self.N])) 99 | 100 | right = 0.5 * np.sum(np.log(1 / self.beta_all[0, int(np.ceil(self.N / 2)): self.N])) 101 | predict.append([left, right]) 102 | return predict 103 | 104 | def _calculate_weight(self, weights): 105 | sum_weight = np.sum(weights) 106 | return np.asarray(weights / sum_weight, order='C') 107 | 108 | def _calculate_error_rate(self, y_target, y_predict, weight_target): 109 | sum_weight = np.sum(weight_target) 110 | return np.sum(weight_target[:, 0] / sum_weight * np.abs(y_target - y_predict)) 111 | --------------------------------------------------------------------------------