├── .gitattributes ├── .idea ├── BaselineRemoval.iml ├── misc.xml ├── modules.xml ├── vcs.xml └── workspace.xml ├── __pycache__ ├── simulate.cpython-35.pyc └── simulate.cpython-36.pyc ├── baseline removal.pdf ├── baseline_removal.py └── simulate.py /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | -------------------------------------------------------------------------------- /.idea/BaselineRemoval.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 12 | -------------------------------------------------------------------------------- /.idea/misc.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 18 | 19 | 20 | 21 | 22 | 32 | 33 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 136 | 137 | 138 | 139 | 140 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 1527753497278 170 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 207 | 208 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | -------------------------------------------------------------------------------- /__pycache__/simulate.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/__pycache__/simulate.cpython-35.pyc -------------------------------------------------------------------------------- /__pycache__/simulate.cpython-36.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/__pycache__/simulate.cpython-36.pyc -------------------------------------------------------------------------------- /baseline removal.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/baseline removal.pdf -------------------------------------------------------------------------------- /baseline_removal.py: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | 3 | import numpy as np 4 | from scipy import sparse 5 | from scipy.sparse.linalg import spsolve 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | class BaselineRemoval: 10 | def __init__(self): 11 | self.lam = 1e5 # 1e2 - 1e9 12 | self.p = 1e-3 # 1e-3 - 1e-1 13 | 14 | @staticmethod 15 | def baseline_als(y, lam, p, niter=10): 16 | L = len(y) 17 | D = sparse.csc_matrix(np.diff(np.eye(L), 2)) 18 | w = np.ones(L) 19 | z = 0 20 | for i in range(niter): 21 | W = sparse.spdiags(w, 0, L, L) 22 | Z = W + lam * D.dot(D.transpose()) 23 | z = spsolve(Z, w * y) 24 | w = p * (y > z) + (1 - p) * (y < z) 25 | return z 26 | 27 | def baseline_removing(self, data): 28 | return np.apply_along_axis(lambda x: self.baseline_als(x, self.lam, self.p), 0, data) 29 | 30 | 31 | if __name__ == '__main__': 32 | # x = np.arange(-10, 10, 0.1) 33 | # y = np.exp(-x ** 2 / 2) 34 | # br = BaselineRemoval() 35 | # base_y = br.baseline_removing(y) 36 | # plt.plot(x, y) 37 | # plt.plot(x, base_y) 38 | # plt.show() 39 | import simulate 40 | 41 | rawdata = simulate.generate_fake_signal(5, 0.1, 0.8)[:, np.newaxis] 42 | plt.plot(rawdata) 43 | br = BaselineRemoval() 44 | baseline = br.baseline_removing(rawdata) 45 | plt.plot(baseline) 46 | plt.show() 47 | -------------------------------------------------------------------------------- /simulate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # coding: utf-8 3 | # @Time : 2018/8/14 16:11 4 | # @Author : Ye Wang (Wane) 5 | # @Email : y.wang@newdegreetech.com 6 | # @File : simulate.py 7 | # @Software: PyCharm 8 | 9 | 10 | import numpy as np 11 | import pandas as pd 12 | import matplotlib.pyplot as plt 13 | import time 14 | 15 | 16 | def generate_fake_signal(sigma, drift_max, drift_momentum): 17 | m = 1000 18 | # sigma = 3 19 | # drift_max = 0.1 20 | # drift_momentum = 0.8 21 | noise = np.squeeze(np.random.randn(m, 1) * sigma) 22 | drift_slp = np.squeeze(np.random.rand(m, 1) * 2 * drift_max - drift_max) 23 | for ii in range(1, len(drift_slp)): 24 | drift_slp[ii] += (drift_slp[ii - 1] - drift_slp[ii]) * drift_momentum 25 | drift_integral = np.cumsum(drift_slp) 26 | # plt.subplot(311) 27 | # plt.plot(noise) 28 | # plt.ylabel('noise: ADC') 29 | # plt.subplot(312) 30 | # plt.plot(drift_integral) 31 | # plt.ylabel('drift: ADC') 32 | # plt.subplot(313) 33 | plt.plot(drift_integral + noise) 34 | plt.ylabel('signal: ADC') 35 | # plt.title('Signal Decomposition') 36 | # plt.show() 37 | return drift_integral + noise 38 | 39 | 40 | def calc_upper_slp_and_integral(rawdata, slp_order, integral_order): 41 | raw_slp_history = rawdata[:-slp_order] 42 | raw_slp_new = rawdata[slp_order:] 43 | raw_integral_history = rawdata[:-integral_order] 44 | raw_integral_new = rawdata[integral_order:] 45 | slp_arr = np.abs(raw_slp_new - raw_slp_history) 46 | integral_arr = np.abs(raw_integral_new - raw_integral_history) 47 | # plt.subplot(211) 48 | # plt.hist(slp_arr, 100) 49 | # plt.vlines(np.percentile(slp_arr, 99), 0, 400) 50 | # plt.ylabel('slp') 51 | # plt.subplot(212) 52 | # plt.hist(integral_arr, 100) 53 | # plt.vlines(np.percentile(integral_arr, 99), 0, 400) 54 | # plt.ylabel('integral') 55 | # plt.suptitle('Slp and Integral almost i.i.d.') 56 | # plt.show() 57 | return np.percentile(integral_arr, 99) 58 | 59 | 60 | def generate_fake_model(lf, rf, decay, gap, wn): 61 | # lf = 60 62 | # rf = 200 63 | # decay = 5 64 | # gap = 8 65 | # wn = 5 # std 66 | gaussian = lambda x, p, wn: \ 67 | p[0] * np.exp(-(x - p[1]) ** 2 / (2 * p[2] ** 2)) + \ 68 | np.squeeze(np.random.randn(len(x), 1) * wn) 69 | xx = np.arange(-gap, 4 * gap, 0.1) 70 | sensor1 = gaussian(xx, (lf, 0, decay), wn) 71 | sensor2 = gaussian(xx, (rf, gap, decay), wn) 72 | sensor3 = gaussian(xx, (rf, 2 * gap, decay), wn) 73 | sensor4 = gaussian(xx, (lf, 3 * gap, decay), wn) 74 | plt.plot(xx, sensor1) 75 | plt.plot(xx, sensor2) 76 | plt.plot(xx, sensor3) 77 | plt.plot(xx, sensor4) 78 | plt.vlines(0, 0, np.max((sensor1, sensor2)), linestyles='--') 79 | plt.vlines(3 * gap, 0, np.max((sensor1, sensor2)), linestyles='--') 80 | plt.title('Signal vs. Position') 81 | plt.xlabel('position: mm') 82 | plt.ylabel('signal: ADC') 83 | # plt.show() 84 | model_xx = xx 85 | # model_xx = np.arange(0, 3 * gap, 0.1) 86 | model = np.hstack((gaussian(model_xx, (lf, 0, decay), 0)[:, np.newaxis], 87 | gaussian(model_xx, (rf, gap, decay), 0)[:, np.newaxis], 88 | gaussian(model_xx, (rf, 2 * gap, decay), 0)[:, np.newaxis], 89 | gaussian(model_xx, (lf, 3 * gap, decay), 0)[:, np.newaxis], 90 | )) 91 | real_sensor = np.hstack((gaussian(model_xx, (lf, 0, decay), wn)[:, np.newaxis], 92 | gaussian(model_xx, (rf, gap, decay), wn)[:, np.newaxis], 93 | gaussian(model_xx, (rf, 2 * gap, decay), wn)[:, np.newaxis], 94 | gaussian(model_xx, (lf, 3 * gap, decay), wn)[:, np.newaxis], 95 | )) 96 | normalization = lambda y: np.apply_along_axis(lambda x: x / np.max(np.abs(x)), 1, y) 97 | return model_xx, \ 98 | normalization(model), \ 99 | normalization(real_sensor) 100 | 101 | 102 | def calc_pos(model, data): 103 | m = np.shape(model)[0] 104 | dist_mat = np.tile(data, (m, 1)) 105 | dist_mat = (dist_mat - model) 106 | dist_mat_square_sum = np.sum(dist_mat ** 2, axis=1) 107 | min_dist_10_val = np.sort(dist_mat_square_sum)[:10] 108 | min_dist_10_val = min_dist_10_val / (np.min(min_dist_10_val) + 1) 109 | W_min_10 = np.exp(-200 * (min_dist_10_val - 1) ** 2) 110 | min_dist_10_ind = np.argsort(dist_mat_square_sum)[:10] 111 | weighted_pos = np.sum(min_dist_10_ind * W_min_10) / np.sum(W_min_10) 112 | return weighted_pos 113 | 114 | 115 | def map_pos(rgA_l, rgA_r, rgB_l, rgB_r, pos): 116 | return rgB_l + pos * (rgB_r - rgB_l) / (rgA_r - rgA_l) 117 | 118 | 119 | def calc_positions(real_x, model, data): 120 | # plt.subplot(211) 121 | # plt.plot(model) 122 | # plt.subplot(212) 123 | # plt.plot(data) 124 | # plt.show() 125 | postions = np.apply_along_axis( 126 | lambda x: calc_pos(model, x), 127 | 1, data) 128 | postions = map_pos(0, np.shape(data)[0], 129 | np.min(real_x), np.max(real_x), 130 | postions) 131 | plt.plot(real_x, postions) 132 | plt.grid(True) 133 | plt.axis('equal') 134 | plt.xlabel('real position') 135 | plt.ylabel('calc position') 136 | plt.title('Real position vs. Calculated position') 137 | # plt.show() 138 | 139 | 140 | if __name__ == '__main__': 141 | plt.rcParams['font.family'] = 'Consolas' 142 | plt.rcParams['font.size'] = 20 143 | fig = plt.figure() 144 | fig.set_size_inches(60, 10) 145 | while True: 146 | params = pd.read_table('hyper_params.txt', delimiter='\t').ix[:, :-1] 147 | module_noise_sigma = np.squeeze(params['module_noise_sigma']) 148 | module_drift_max = np.squeeze(params['module_drift_max']) 149 | module_drift_momentum = np.squeeze(params['module_drift_momentum']) 150 | structure_little_finger = np.squeeze(params['structure_little_finger']) 151 | structure_ring_finger = np.squeeze(params['structure_ring_finger']) 152 | structure_decay = np.squeeze(params['structure_decay']) 153 | structure_gap = np.squeeze(params['structure_gap']) 154 | structure_wn_of_str = np.squeeze(params['structure_noise_WN']) 155 | 156 | plt.subplot(221) 157 | rawdata = generate_fake_signal(module_noise_sigma, module_drift_max, module_drift_momentum) 158 | min_max_integral = calc_upper_slp_and_integral(rawdata, 3, 15) 159 | plt.title('Signal Decomposition ' + '%.2f' % min_max_integral) 160 | # print(min_max_integral) 161 | plt.subplot(223) 162 | model_x, model, real_data = generate_fake_model( 163 | structure_little_finger, 164 | structure_ring_finger, 165 | structure_decay, 166 | structure_gap, 167 | structure_wn_of_str 168 | ) 169 | plt.subplot(122) 170 | calc_positions(model_x, model, real_data) 171 | plt.suptitle('Intuition on how hyper parameters influence performance') 172 | plt.pause(0.2) 173 | plt.clf() 174 | # time.sleep(0.1) 175 | --------------------------------------------------------------------------------