├── .gitattributes
├── .idea
├── BaselineRemoval.iml
├── misc.xml
├── modules.xml
├── vcs.xml
└── workspace.xml
├── __pycache__
├── simulate.cpython-35.pyc
└── simulate.cpython-36.pyc
├── baseline removal.pdf
├── baseline_removal.py
└── simulate.py
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
--------------------------------------------------------------------------------
/.idea/BaselineRemoval.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
--------------------------------------------------------------------------------
/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
69 |
70 |
71 |
72 |
73 |
74 |
80 |
81 |
82 |
83 |
84 | true
85 | DEFINITION_ORDER
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
159 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 | 1527753497278
170 |
171 |
172 | 1527753497278
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
--------------------------------------------------------------------------------
/__pycache__/simulate.cpython-35.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/__pycache__/simulate.cpython-35.pyc
--------------------------------------------------------------------------------
/__pycache__/simulate.cpython-36.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/__pycache__/simulate.cpython-36.pyc
--------------------------------------------------------------------------------
/baseline removal.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wangyendt/BaselineRemoval/1bb9adc8e5a994afef043e86227ef3d4ff6beb42/baseline removal.pdf
--------------------------------------------------------------------------------
/baseline_removal.py:
--------------------------------------------------------------------------------
1 | # coding: utf-8
2 |
3 | import numpy as np
4 | from scipy import sparse
5 | from scipy.sparse.linalg import spsolve
6 | import matplotlib.pyplot as plt
7 |
8 |
9 | class BaselineRemoval:
10 | def __init__(self):
11 | self.lam = 1e5 # 1e2 - 1e9
12 | self.p = 1e-3 # 1e-3 - 1e-1
13 |
14 | @staticmethod
15 | def baseline_als(y, lam, p, niter=10):
16 | L = len(y)
17 | D = sparse.csc_matrix(np.diff(np.eye(L), 2))
18 | w = np.ones(L)
19 | z = 0
20 | for i in range(niter):
21 | W = sparse.spdiags(w, 0, L, L)
22 | Z = W + lam * D.dot(D.transpose())
23 | z = spsolve(Z, w * y)
24 | w = p * (y > z) + (1 - p) * (y < z)
25 | return z
26 |
27 | def baseline_removing(self, data):
28 | return np.apply_along_axis(lambda x: self.baseline_als(x, self.lam, self.p), 0, data)
29 |
30 |
31 | if __name__ == '__main__':
32 | # x = np.arange(-10, 10, 0.1)
33 | # y = np.exp(-x ** 2 / 2)
34 | # br = BaselineRemoval()
35 | # base_y = br.baseline_removing(y)
36 | # plt.plot(x, y)
37 | # plt.plot(x, base_y)
38 | # plt.show()
39 | import simulate
40 |
41 | rawdata = simulate.generate_fake_signal(5, 0.1, 0.8)[:, np.newaxis]
42 | plt.plot(rawdata)
43 | br = BaselineRemoval()
44 | baseline = br.baseline_removing(rawdata)
45 | plt.plot(baseline)
46 | plt.show()
47 |
--------------------------------------------------------------------------------
/simulate.py:
--------------------------------------------------------------------------------
1 | #!/usr/bin/python
2 | # coding: utf-8
3 | # @Time : 2018/8/14 16:11
4 | # @Author : Ye Wang (Wane)
5 | # @Email : y.wang@newdegreetech.com
6 | # @File : simulate.py
7 | # @Software: PyCharm
8 |
9 |
10 | import numpy as np
11 | import pandas as pd
12 | import matplotlib.pyplot as plt
13 | import time
14 |
15 |
16 | def generate_fake_signal(sigma, drift_max, drift_momentum):
17 | m = 1000
18 | # sigma = 3
19 | # drift_max = 0.1
20 | # drift_momentum = 0.8
21 | noise = np.squeeze(np.random.randn(m, 1) * sigma)
22 | drift_slp = np.squeeze(np.random.rand(m, 1) * 2 * drift_max - drift_max)
23 | for ii in range(1, len(drift_slp)):
24 | drift_slp[ii] += (drift_slp[ii - 1] - drift_slp[ii]) * drift_momentum
25 | drift_integral = np.cumsum(drift_slp)
26 | # plt.subplot(311)
27 | # plt.plot(noise)
28 | # plt.ylabel('noise: ADC')
29 | # plt.subplot(312)
30 | # plt.plot(drift_integral)
31 | # plt.ylabel('drift: ADC')
32 | # plt.subplot(313)
33 | plt.plot(drift_integral + noise)
34 | plt.ylabel('signal: ADC')
35 | # plt.title('Signal Decomposition')
36 | # plt.show()
37 | return drift_integral + noise
38 |
39 |
40 | def calc_upper_slp_and_integral(rawdata, slp_order, integral_order):
41 | raw_slp_history = rawdata[:-slp_order]
42 | raw_slp_new = rawdata[slp_order:]
43 | raw_integral_history = rawdata[:-integral_order]
44 | raw_integral_new = rawdata[integral_order:]
45 | slp_arr = np.abs(raw_slp_new - raw_slp_history)
46 | integral_arr = np.abs(raw_integral_new - raw_integral_history)
47 | # plt.subplot(211)
48 | # plt.hist(slp_arr, 100)
49 | # plt.vlines(np.percentile(slp_arr, 99), 0, 400)
50 | # plt.ylabel('slp')
51 | # plt.subplot(212)
52 | # plt.hist(integral_arr, 100)
53 | # plt.vlines(np.percentile(integral_arr, 99), 0, 400)
54 | # plt.ylabel('integral')
55 | # plt.suptitle('Slp and Integral almost i.i.d.')
56 | # plt.show()
57 | return np.percentile(integral_arr, 99)
58 |
59 |
60 | def generate_fake_model(lf, rf, decay, gap, wn):
61 | # lf = 60
62 | # rf = 200
63 | # decay = 5
64 | # gap = 8
65 | # wn = 5 # std
66 | gaussian = lambda x, p, wn: \
67 | p[0] * np.exp(-(x - p[1]) ** 2 / (2 * p[2] ** 2)) + \
68 | np.squeeze(np.random.randn(len(x), 1) * wn)
69 | xx = np.arange(-gap, 4 * gap, 0.1)
70 | sensor1 = gaussian(xx, (lf, 0, decay), wn)
71 | sensor2 = gaussian(xx, (rf, gap, decay), wn)
72 | sensor3 = gaussian(xx, (rf, 2 * gap, decay), wn)
73 | sensor4 = gaussian(xx, (lf, 3 * gap, decay), wn)
74 | plt.plot(xx, sensor1)
75 | plt.plot(xx, sensor2)
76 | plt.plot(xx, sensor3)
77 | plt.plot(xx, sensor4)
78 | plt.vlines(0, 0, np.max((sensor1, sensor2)), linestyles='--')
79 | plt.vlines(3 * gap, 0, np.max((sensor1, sensor2)), linestyles='--')
80 | plt.title('Signal vs. Position')
81 | plt.xlabel('position: mm')
82 | plt.ylabel('signal: ADC')
83 | # plt.show()
84 | model_xx = xx
85 | # model_xx = np.arange(0, 3 * gap, 0.1)
86 | model = np.hstack((gaussian(model_xx, (lf, 0, decay), 0)[:, np.newaxis],
87 | gaussian(model_xx, (rf, gap, decay), 0)[:, np.newaxis],
88 | gaussian(model_xx, (rf, 2 * gap, decay), 0)[:, np.newaxis],
89 | gaussian(model_xx, (lf, 3 * gap, decay), 0)[:, np.newaxis],
90 | ))
91 | real_sensor = np.hstack((gaussian(model_xx, (lf, 0, decay), wn)[:, np.newaxis],
92 | gaussian(model_xx, (rf, gap, decay), wn)[:, np.newaxis],
93 | gaussian(model_xx, (rf, 2 * gap, decay), wn)[:, np.newaxis],
94 | gaussian(model_xx, (lf, 3 * gap, decay), wn)[:, np.newaxis],
95 | ))
96 | normalization = lambda y: np.apply_along_axis(lambda x: x / np.max(np.abs(x)), 1, y)
97 | return model_xx, \
98 | normalization(model), \
99 | normalization(real_sensor)
100 |
101 |
102 | def calc_pos(model, data):
103 | m = np.shape(model)[0]
104 | dist_mat = np.tile(data, (m, 1))
105 | dist_mat = (dist_mat - model)
106 | dist_mat_square_sum = np.sum(dist_mat ** 2, axis=1)
107 | min_dist_10_val = np.sort(dist_mat_square_sum)[:10]
108 | min_dist_10_val = min_dist_10_val / (np.min(min_dist_10_val) + 1)
109 | W_min_10 = np.exp(-200 * (min_dist_10_val - 1) ** 2)
110 | min_dist_10_ind = np.argsort(dist_mat_square_sum)[:10]
111 | weighted_pos = np.sum(min_dist_10_ind * W_min_10) / np.sum(W_min_10)
112 | return weighted_pos
113 |
114 |
115 | def map_pos(rgA_l, rgA_r, rgB_l, rgB_r, pos):
116 | return rgB_l + pos * (rgB_r - rgB_l) / (rgA_r - rgA_l)
117 |
118 |
119 | def calc_positions(real_x, model, data):
120 | # plt.subplot(211)
121 | # plt.plot(model)
122 | # plt.subplot(212)
123 | # plt.plot(data)
124 | # plt.show()
125 | postions = np.apply_along_axis(
126 | lambda x: calc_pos(model, x),
127 | 1, data)
128 | postions = map_pos(0, np.shape(data)[0],
129 | np.min(real_x), np.max(real_x),
130 | postions)
131 | plt.plot(real_x, postions)
132 | plt.grid(True)
133 | plt.axis('equal')
134 | plt.xlabel('real position')
135 | plt.ylabel('calc position')
136 | plt.title('Real position vs. Calculated position')
137 | # plt.show()
138 |
139 |
140 | if __name__ == '__main__':
141 | plt.rcParams['font.family'] = 'Consolas'
142 | plt.rcParams['font.size'] = 20
143 | fig = plt.figure()
144 | fig.set_size_inches(60, 10)
145 | while True:
146 | params = pd.read_table('hyper_params.txt', delimiter='\t').ix[:, :-1]
147 | module_noise_sigma = np.squeeze(params['module_noise_sigma'])
148 | module_drift_max = np.squeeze(params['module_drift_max'])
149 | module_drift_momentum = np.squeeze(params['module_drift_momentum'])
150 | structure_little_finger = np.squeeze(params['structure_little_finger'])
151 | structure_ring_finger = np.squeeze(params['structure_ring_finger'])
152 | structure_decay = np.squeeze(params['structure_decay'])
153 | structure_gap = np.squeeze(params['structure_gap'])
154 | structure_wn_of_str = np.squeeze(params['structure_noise_WN'])
155 |
156 | plt.subplot(221)
157 | rawdata = generate_fake_signal(module_noise_sigma, module_drift_max, module_drift_momentum)
158 | min_max_integral = calc_upper_slp_and_integral(rawdata, 3, 15)
159 | plt.title('Signal Decomposition ' + '%.2f' % min_max_integral)
160 | # print(min_max_integral)
161 | plt.subplot(223)
162 | model_x, model, real_data = generate_fake_model(
163 | structure_little_finger,
164 | structure_ring_finger,
165 | structure_decay,
166 | structure_gap,
167 | structure_wn_of_str
168 | )
169 | plt.subplot(122)
170 | calc_positions(model_x, model, real_data)
171 | plt.suptitle('Intuition on how hyper parameters influence performance')
172 | plt.pause(0.2)
173 | plt.clf()
174 | # time.sleep(0.1)
175 |
--------------------------------------------------------------------------------