├── .gitattributes
├── AI_Lecture1.pdf
├── AI_Lecture10.pdf
├── AI_Lecture11.pdf
├── AI_Lecture12.pdf
├── AI_Lecture13.pdf
├── AI_Lecture14.pdf
├── AI_Lecture2.pdf
├── AI_Lecture3.pdf
├── AI_Lecture4.pdf
├── AI_Lecture5.pdf
├── AI_Lecture6.pdf
├── AI_Lecture7.pdf
├── AI_Lecture8.pdf
├── AI_Lecture9.pdf
├── README.md
├── homework
├── ex1
│ ├── .idea
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex1.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ └── workspace.xml
│ ├── __pycache__
│ │ └── linear_regression.cpython-37.pyc
│ ├── ex1.pdf
│ ├── ex1data1.txt
│ ├── ex1data2.txt
│ ├── explanation.jpg
│ ├── linear_regression.py
│ ├── linear_regression_with_multiple_variables.py
│ └── test.py
├── ex2
│ ├── .idea
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex2.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ └── workspace.xml
│ ├── Logistic_Regression.py
│ ├── Logistic_Regression_Regularized.py
│ ├── Logistic_Regression_Regularlized_Correct.py
│ ├── __pycache__
│ │ └── Logistic_Regression.cpython-37.pyc
│ ├── ex2.pdf
│ ├── ex2data1.txt
│ ├── ex2data2.txt
│ └── test.py
├── ex3
│ ├── .idea
│ │ ├── GrepConsole.xml
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex3.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ ├── vcs.xml
│ │ └── workspace.xml
│ ├── Multi-class Classification and Neural Networks.py
│ ├── ex3.pdf
│ ├── ex3data1.mat
│ └── ex3weights.mat
├── ex4
│ ├── .idea
│ │ ├── .gitignore
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex4.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ └── vcs.xml
│ ├── ex4.pdf
│ ├── ex4.py
│ ├── ex4data1.mat
│ ├── ex4weights.mat
│ ├── gradient_check.png
│ └── part1.png
├── ex5
│ ├── .idea
│ │ ├── .gitignore
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex5.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ └── vcs.xml
│ ├── ex5.pdf
│ ├── ex5.py
│ └── ex5data1.mat
├── ex6
│ ├── .idea
│ │ ├── .gitignore
│ │ ├── dictionaries
│ │ │ └── .xml
│ │ ├── ex6.iml
│ │ ├── inspectionProfiles
│ │ │ └── profiles_settings.xml
│ │ ├── misc.xml
│ │ ├── modules.xml
│ │ └── vcs.xml
│ ├── SpamClassification.py
│ ├── emailSample1.txt
│ ├── emailSample2.txt
│ ├── ex6.pdf
│ ├── ex6.py
│ ├── ex6data1.mat
│ ├── ex6data2.mat
│ ├── ex6data3.mat
│ ├── spamSample1.txt
│ ├── spamSample2.txt
│ ├── spamTest.mat
│ ├── spamTrain.mat
│ ├── test.py
│ └── vocab.txt
└── ex7
│ ├── .idea
│ ├── .gitignore
│ ├── ex7.iml
│ ├── inspectionProfiles
│ │ └── profiles_settings.xml
│ ├── misc.xml
│ ├── modules.xml
│ └── vcs.xml
│ ├── Kmeans.py
│ ├── bird_small.mat
│ ├── bird_small.png
│ ├── ex7.pdf
│ ├── ex7data1.mat
│ ├── ex7data2.mat
│ └── ex7faces.mat
└── 算法.txt
/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto
3 |
--------------------------------------------------------------------------------
/AI_Lecture1.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture1.pdf
--------------------------------------------------------------------------------
/AI_Lecture10.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture10.pdf
--------------------------------------------------------------------------------
/AI_Lecture11.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture11.pdf
--------------------------------------------------------------------------------
/AI_Lecture12.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture12.pdf
--------------------------------------------------------------------------------
/AI_Lecture13.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture13.pdf
--------------------------------------------------------------------------------
/AI_Lecture14.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture14.pdf
--------------------------------------------------------------------------------
/AI_Lecture2.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture2.pdf
--------------------------------------------------------------------------------
/AI_Lecture3.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture3.pdf
--------------------------------------------------------------------------------
/AI_Lecture4.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture4.pdf
--------------------------------------------------------------------------------
/AI_Lecture5.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture5.pdf
--------------------------------------------------------------------------------
/AI_Lecture6.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture6.pdf
--------------------------------------------------------------------------------
/AI_Lecture7.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture7.pdf
--------------------------------------------------------------------------------
/AI_Lecture8.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture8.pdf
--------------------------------------------------------------------------------
/AI_Lecture9.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/AI_Lecture9.pdf
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Machine Learning WuEnda
2 |
3 | 主页AI_LectureX是对应章节的pdf笔记
4 | 子文件夹homework下是对应的练习题以及python代码实现，欢迎pull request
5 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | enda
5 | iters
6 | xlim
7 |
8 |
9 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/ex1.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex1/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
21 |
22 |
23 |
24 |
25 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
142 |
143 |
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 | 1586268354226
170 |
171 |
172 | 1586268354226
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
223 |
224 |
225 |
226 |
227 |
228 |
229 |
230 |
231 |
232 |
233 |
234 |
235 |
236 |
237 |
238 |
239 |
240 |
241 |
242 |
243 |
244 |
245 |
246 |
247 |
248 |
249 |
250 |
251 |
--------------------------------------------------------------------------------
/homework/ex1/__pycache__/linear_regression.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex1/__pycache__/linear_regression.cpython-37.pyc
--------------------------------------------------------------------------------
/homework/ex1/ex1.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex1/ex1.pdf
--------------------------------------------------------------------------------
/homework/ex1/ex1data1.txt:
--------------------------------------------------------------------------------
1 | 6.1101,17.592
2 | 5.5277,9.1302
3 | 8.5186,13.662
4 | 7.0032,11.854
5 | 5.8598,6.8233
6 | 8.3829,11.886
7 | 7.4764,4.3483
8 | 8.5781,12
9 | 6.4862,6.5987
10 | 5.0546,3.8166
11 | 5.7107,3.2522
12 | 14.164,15.505
13 | 5.734,3.1551
14 | 8.4084,7.2258
15 | 5.6407,0.71618
16 | 5.3794,3.5129
17 | 6.3654,5.3048
18 | 5.1301,0.56077
19 | 6.4296,3.6518
20 | 7.0708,5.3893
21 | 6.1891,3.1386
22 | 20.27,21.767
23 | 5.4901,4.263
24 | 6.3261,5.1875
25 | 5.5649,3.0825
26 | 18.945,22.638
27 | 12.828,13.501
28 | 10.957,7.0467
29 | 13.176,14.692
30 | 22.203,24.147
31 | 5.2524,-1.22
32 | 6.5894,5.9966
33 | 9.2482,12.134
34 | 5.8918,1.8495
35 | 8.2111,6.5426
36 | 7.9334,4.5623
37 | 8.0959,4.1164
38 | 5.6063,3.3928
39 | 12.836,10.117
40 | 6.3534,5.4974
41 | 5.4069,0.55657
42 | 6.8825,3.9115
43 | 11.708,5.3854
44 | 5.7737,2.4406
45 | 7.8247,6.7318
46 | 7.0931,1.0463
47 | 5.0702,5.1337
48 | 5.8014,1.844
49 | 11.7,8.0043
50 | 5.5416,1.0179
51 | 7.5402,6.7504
52 | 5.3077,1.8396
53 | 7.4239,4.2885
54 | 7.6031,4.9981
55 | 6.3328,1.4233
56 | 6.3589,-1.4211
57 | 6.2742,2.4756
58 | 5.6397,4.6042
59 | 9.3102,3.9624
60 | 9.4536,5.4141
61 | 8.8254,5.1694
62 | 5.1793,-0.74279
63 | 21.279,17.929
64 | 14.908,12.054
65 | 18.959,17.054
66 | 7.2182,4.8852
67 | 8.2951,5.7442
68 | 10.236,7.7754
69 | 5.4994,1.0173
70 | 20.341,20.992
71 | 10.136,6.6799
72 | 7.3345,4.0259
73 | 6.0062,1.2784
74 | 7.2259,3.3411
75 | 5.0269,-2.6807
76 | 6.5479,0.29678
77 | 7.5386,3.8845
78 | 5.0365,5.7014
79 | 10.274,6.7526
80 | 5.1077,2.0576
81 | 5.7292,0.47953
82 | 5.1884,0.20421
83 | 6.3557,0.67861
84 | 9.7687,7.5435
85 | 6.5159,5.3436
86 | 8.5172,4.2415
87 | 9.1802,6.7981
88 | 6.002,0.92695
89 | 5.5204,0.152
90 | 5.0594,2.8214
91 | 5.7077,1.8451
92 | 7.6366,4.2959
93 | 5.8707,7.2029
94 | 5.3054,1.9869
95 | 8.2934,0.14454
96 | 13.394,9.0551
97 | 5.4369,0.61705
98 |
--------------------------------------------------------------------------------
/homework/ex1/ex1data2.txt:
--------------------------------------------------------------------------------
1 | 2104,3,399900
2 | 1600,3,329900
3 | 2400,3,369000
4 | 1416,2,232000
5 | 3000,4,539900
6 | 1985,4,299900
7 | 1534,3,314900
8 | 1427,3,198999
9 | 1380,3,212000
10 | 1494,3,242500
11 | 1940,4,239999
12 | 2000,3,347000
13 | 1890,3,329999
14 | 4478,5,699900
15 | 1268,3,259900
16 | 2300,4,449900
17 | 1320,2,299900
18 | 1236,3,199900
19 | 2609,4,499998
20 | 3031,4,599000
21 | 1767,3,252900
22 | 1888,2,255000
23 | 1604,3,242900
24 | 1962,4,259900
25 | 3890,3,573900
26 | 1100,3,249900
27 | 1458,3,464500
28 | 2526,3,469000
29 | 2200,3,475000
30 | 2637,3,299900
31 | 1839,2,349900
32 | 1000,1,169900
33 | 2040,4,314900
34 | 3137,3,579900
35 | 1811,4,285900
36 | 1437,3,249900
37 | 1239,3,229900
38 | 2132,4,345000
39 | 4215,4,549000
40 | 2162,4,287000
41 | 1664,2,368500
42 | 2238,3,329900
43 | 2567,4,314000
44 | 1200,3,299000
45 | 852,2,179900
46 | 1852,4,299900
47 | 1203,3,239500
48 |
--------------------------------------------------------------------------------
/homework/ex1/explanation.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex1/explanation.jpg
--------------------------------------------------------------------------------
/homework/ex1/linear_regression.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 | from mpl_toolkits.mplot3d import Axes3D
5 |
6 |
7 | # 这个部分计算J(Ѳ)，X是矩阵
8 | def compute_cost(X, y, theta):
9 | inner = np.power((X*theta.T - y), 2)
10 | return np.sum(inner, axis=0) / (2*len(X))
11 |
12 |
13 | def gradient_descent(X, y, _theta, alpha, iters):
14 | # temp用来暂存theta的值，因为要同时改变theta1和2
15 | temp = np.mat(np.zeros(_theta.shape))
16 |
17 | # parameters用来控制循环次数
18 | parameters = int(_theta.ravel().shape[1])
19 |
20 | # cost用来存放每次迭代后的代价函数
21 | cost = np.zeros(iters)
22 |
23 | # 外循环控制迭代次数
24 | for i in range(iters):
25 | error = (X * _theta.T) - y
26 |
27 | # 内循环对每个theta进行处理，即有几个theta就循环几次
28 | for j in range(parameters):
29 |
30 | term = np.multiply(error, X[:, j])
31 | temp[0, j] = _theta[0, j] - ((alpha / len(X)) * np.sum(term, axis=0))
32 |
33 | _theta = temp
34 | cost[i] = compute_cost(X, y, _theta)
35 |
36 | # print('这是第{}次循环, 代价函数的值为{:.4f}' .format(i+1, cost[i]), end=' ')
37 | # for k in range(parameters):
38 | # print(', Ѳ{:d}为{:.4f}' .format(k, _theta[0, k]), end=' ')
39 | # print('\n')
40 | return _theta, cost
41 |
42 |
43 | def normal_equation(X, y):
44 | _theta = (X.T*X).I*X.T*y
45 | return _theta
46 |
47 |
48 | if __name__ == '__main__':
49 | # 几维矩阵一般是指有多少列的矩阵而不是行数
50 | # 因为做数据分析的时候有时样本行数是几万几万的来
51 | # 但是降维分析不是缩减样本体积而是去掉不必要的列数来构建特征向量
52 |
53 | # 读取数据
54 | show_predict_result = False
55 | show_3d_plot = True
56 | show_2d_plot = True
57 |
58 | path = 'D:\Study\Machine Learning WuEnda\homework\ex1\ex1data1.txt'
59 | data = pd.read_csv(path, header=None, names=['Population', 'Profits'])
60 |
61 | plt.figure(figsize=(8, 6))
62 | dot = plt.scatter(x=data['Population'], y=data['Profits'], s=40, alpha=0.6, label='Training data')
63 | # plt.xlim((0, 23))
64 | plt.ylim((-5, 25))
65 | plt.xlabel('Population')
66 | plt.ylabel('Profits')
67 |
68 | ax = plt.gca()
69 | ax.spines['top'].set_color('None')
70 | ax.spines['right'].set_color('None')
71 | # plt.show()
72 |
73 | # 加入一列x，用于更新Ѳ
74 | data.insert(0, 'Ones', 1)
75 | # print(data)
76 |
77 | # 初始化X和y
78 | # data.shape返回(97, 3)即97行3列
79 | cols = data.shape[1]
80 | X = data.iloc[:, :-1] # X是data里的除最后列
81 | y = data.iloc[:, cols-1:cols] # y是data最后一列
82 | # print(X.head(), '\n', y.head())
83 |
84 | X = np.mat(X.values)
85 | y = np.mat(y.values)
86 | theta = np.mat(np.array([0, 0]))
87 | # print(normal_equation(X, y))
88 | # print('----------------------------------------------------------------')
89 | # print(X.shape, '\n', y.shape, '\n', theta.shape, theta.ravel())
90 |
91 | # 计算J(Ѳ)
92 | # print(compute_cost(X, y, theta))
93 |
94 | # np.shape显示(矩阵的行数, 矩阵的列数)
95 | # np.zeros((x, y))输出一个x行y列的都是0的矩阵
96 | # np.ravel(np.array([1, 2], [3, 4]))输出[1 2 3 4]实现多维度矩阵降维
97 |
98 | # 开始进行梯度下降
99 | # 这个部分实现了Ѳ的更新
100 | # alpha为学习速率, iters为迭代次数
101 | alpha = 0.01
102 | iters = 1500
103 | g, cost = gradient_descent(X, y, theta, alpha, iters)
104 | # print(g)
105 |
106 | # 构造预测数据,即人口为3.5w和7w时餐馆的利润
107 | predict1 = np.mat([1, 3.5]) * g.T
108 | print('predict1: {:.3f}' .format(predict1[0, 0])) # 保留三位小数
109 | predict2 = np.mat([1, 7]) * g.T
110 | print('predict2: {:.3f}' .format(predict2[0, 0]))
111 |
112 | # 可视化处理
113 | # 产生数据,图例在plot,scatter里面画
114 | x = np.linspace(2, data.Population.max(), 100)
115 | f_predict = g[0, 0] + (g[0, 1] * x)
116 | predict_line = plt.plot(x, f_predict, linewidth=1.7, linestyle='-', label='Predict line')
117 | plt.scatter(x=[3.5, 7], y=[predict1, predict2], s=35, alpha=0.9, color='red', label='Predict profits') # 产生预测点
118 | plt.title('Predicted Profit vs. Population Size')
119 | plt.legend(loc='upper left')
120 | plt.show()
121 |
122 | # 可视化J(θ) 暂且不会
123 |
124 |
125 |
126 |
--------------------------------------------------------------------------------
/homework/ex1/linear_regression_with_multiple_variables.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import matplotlib.pyplot as plt
4 | from linear_regression import gradient_descent
5 | from linear_regression import compute_cost
6 |
7 | path = 'D:\Study\Coding\Machine Learning WuEnda\homework\ex1\ex1data2.txt'
8 | data = pd.read_csv(path, header=None, names=['Size', 'Bedroom_nums', 'Price'])
9 | # print(data.head())
10 |
11 | # 归一化处理
12 | # data.mean()处理平均值,data.std() (short for 'standard')处理标准偏差(max-min)
13 | data = (data - data.mean()) / data.std()
14 | data.insert(0, 'Ones', 1)
15 | print(data.head())
16 |
17 | # 初始化x,y
18 | cols = data.shape[1] # 返回一个元组(行, 列)
19 | x = data.iloc[:, :cols-1]
20 | y = data.iloc[:, cols-1:cols]
21 |
22 | # 转换成矩阵
23 | x = np.mat(x.values)
24 | y = np.mat(y.values)
25 | theta = np.mat(np.array([0, 0, 0]))
26 |
27 | alpha = 0.01
28 | iters = 1500
29 | g2, cost = gradient_descent(x, y, theta, alpha, iters)
30 | print(g2)
31 |
--------------------------------------------------------------------------------
/homework/ex1/test.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 | from mpl_toolkits.mplot3d import Axes3D
5 |
6 | fig = plt.figure()
7 | ax = Axes3D(fig) # 在窗口上添加3D坐标轴
8 |
9 |
10 |
11 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | enda
5 | lamda
6 | linestyle
7 |
8 |
9 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/ex2.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex2/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 |
141 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 | 1588168811236
156 |
157 |
158 | 1588168811236
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
211 |
212 |
213 |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 |
222 |
--------------------------------------------------------------------------------
/homework/ex2/Logistic_Regression.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | import numpy as np
4 | import scipy.optimize as opt
5 |
6 |
7 | # define sigmoid function
8 | def sigmoid(z):
9 | return 1 / (1 + np.exp(-z))
10 |
11 |
12 | def gradient_descent(X, y, theta, alpha, iters, vistualize=True):
13 | temp = np.mat(np.zeros(theta.shape)) # temp暂存theta的值
14 | parameters = int(theta.shape[1]) # parameters控制循环次数
15 | cost = np.zeros(iters) # cost用来存放每次迭代后的代价函数
16 |
17 | for i in range(iters):
18 | subtraction = sigmoid(X * theta.T) - y
19 |
20 | for j in range(parameters):
21 | term = np.multiply(subtraction, X[:, j])
22 | temp[0, j] = theta[0, j] - ((alpha / len(X)) * np.sum(term, axis=0)) # len()返回矩阵的行数
23 |
24 | theta = temp
25 | cost[i] = cost_function(X, y, theta)
26 |
27 | if vistualize:
28 | print('这是第{}次循环, 代价函数的值为{:.4f}' .format(i+1, cost[i]), end=' ')
29 | for k in range(parameters):
30 | print(', Ѳ{:d}为{:.4f}' .format(k, theta[0, k]), end=' ')
31 | print('\n')
32 |
33 | if i != 0 and cost[i] > cost[i-1]: # 检查梯度下降算法是否正常工作
34 | raise ValueError('The value of cost_function becomes larger')
35 |
36 | return theta, cost
37 |
38 |
39 | def cost_function(theta, X, y): # correct √
40 | hypothesis_function = sigmoid(X * theta.T)
41 | each = np.multiply(-y, np.log(hypothesis_function)) - np.multiply((1 - y), np.log(1 - hypothesis_function))
42 |
43 | return np.sum(each, axis=0) / len(X)
44 |
45 |
46 | def ensure_alpha(X, y, theta, alpha, iters): # 选择学习速率
47 | alpha_first = alpha
48 | subtraction = 0
49 | while alpha / alpha_first >= subtraction:
50 | try:
51 | gradient_descent(X, y, theta, alpha, iters, vistualize=False)
52 | alpha *= 1.2
53 | subtraction = 0.2 * alpha
54 |
55 | except:
56 | alpha /= 1.1
57 | subtraction = 0.1 * alpha
58 | print(alpha)
59 | return alpha
60 |
61 |
62 | # 评价录取结果
63 | def func1(theta, X):
64 | return sigmoid(X * theta.T)
65 |
66 |
67 | def predict(theta, X): # 预测模型对训练集的数据准确率
68 | result = []
69 | a = sigmoid(X * theta.T)
70 | for each in a:
71 | if each >= 0.5:
72 | result.append(1)
73 | else:
74 | result.append(0)
75 |
76 | result = np.mat(result)
77 | result = result.reshape(result.shape[1], 1)
78 | return result
79 |
80 |
81 | def compare(matrix, y):
82 | temp = (matrix == y)
83 | correct_nums = 0
84 | rows = temp.shape[0]
85 | for each in range(rows):
86 | if temp[each, 0]:
87 | correct_nums += 1
88 | else:
89 | pass
90 |
91 | accuracy = correct_nums / temp.shape[0]
92 | print('accuracy = {:.2f}%' .format(accuracy*100))
93 | return accuracy
94 |
95 |
96 | if __name__ == '__main__':
97 | # 首先进行数据可视化(Visualizing the data)
98 | # 读取数据
99 | path = 'D:\Study\Coding\Machine Learning WuEnda\homework\ex2\ex2data1.txt'
100 | data = pd.read_csv(path, header=None, names=['exam1', 'exam2', 'result'])
101 | # print(data)
102 | data.insert(0, 'Ones', 1)
103 | yes = data[data['result'] == 1]
104 | no = data[data['result'] == 0]
105 | # print(yes, '\n', no)
106 |
107 | # 开始画图
108 | plt.figure(figsize=(8, 6))
109 | dot_yes = plt.scatter(x=yes['exam1'], y=yes['exam2'], s=40, alpha=0.5, label='Admitted', color='blue')
110 | dot_no = plt.scatter(x=no['exam1'], y=no['exam2'], s=40, alpha=0.5, label='Not admitted', color='red', marker='x')
111 | plt.xlabel('exam1 score')
112 | plt.ylabel('exam2 score')
113 |
114 | ax = plt.gca()
115 | ax.spines['top'].set_color('none')
116 | ax.spines['right'].set_color('none')
117 |
118 | plt.legend(loc='upper right')
119 | # plt.show()
120 |
121 | # 初始化X, y, theta
122 | # data.shape返回一个元组(行, 列)
123 | # rows = data.shape[0]
124 | cols = data.shape[1]
125 | X = data.iloc[:, :-1]
126 | y = data.iloc[:, cols-1:cols]
127 | print(X.head(), '\n', y.head())
128 |
129 | X = np.mat(X.values)
130 | y = np.mat(y.values)
131 | theta = np.mat(np.array([-25.1537, 0.2062, 0.2014])) # 将theta初始化为[0, 0, 0]即将Ѳ0，Ѳ1，Ѳ2均置零
132 | # print(X.shape, theta.shape, y.shape)
133 | #
134 | # print(cost_function(X, y, theta))
135 |
136 | alpha = 0.0009817
137 | iters = 1000
138 | theta_new, cost = gradient_descent(X, y, theta, alpha, iters)
139 | print(theta_new)
140 | print(func1(theta_new, np.mat([1, 45, 85])))
141 | # print(ensure_alpha(X, y, theta, alpha, iters))
142 |
143 | # 可视化拟合结果
144 | theta_0 = theta_new[0, 0]
145 | theta_1 = theta_new[0, 1]
146 | theta_2 = theta_new[0, 2]
147 | x_data = np.linspace(25, 100, 1024)
148 | y_data = -(theta_0 + np.multiply(theta_1, x_data)) / theta_2
149 | plt.plot(x_data, y_data, color='black', linewidth=1.0)
150 | plt.show()
151 |
152 | expected_result = predict(theta_new, X)
153 | compare(expected_result, y)
154 |
155 |
--------------------------------------------------------------------------------
/homework/ex2/Logistic_Regression_Regularized.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | import numpy as np
4 | import Logistic_Regression as lr
5 | import scipy.optimize as opt
6 |
7 | # 数据可视化
8 | path = 'D:\Study\Coding\Machine Learning WuEnda\homework\ex2\ex2data2.txt'
9 | data_init = pd.read_csv(path, header=None, names=['Microchip Test 1', 'Microchip Test 2', 'Accepted'])
10 |
11 | positive = data_init[data_init['Accepted'].isin([1])]
12 | negative = data_init[data_init['Accepted'].isin([0])]
13 | fig, ax = plt.subplots(figsize=(12,8))
14 | ax.scatter(positive['Microchip Test 1'], positive['Microchip Test 2'], alpha=0.5, s=50, c='b', marker='o', label='Accepted')
15 | ax.scatter(negative['Microchip Test 1'], negative['Microchip Test 2'], alpha=0.5, s=50, c='r', marker='x', label='Rejected')
16 | ax.legend()
17 | ax.set_xlabel('Test 1 Score')
18 | ax.set_ylabel('Test 2 Score')
19 | plt.show()
20 |
21 | # 发现不适用于直线拟合，所以我们要创造更多的特征
22 | data_init.insert(0, 'Ones', 1)
23 | print(data_init.head())
24 |
25 | loc_nums = 3
26 | for i in range(6): # 控制x1的幂
27 | for j in range(6): # 控制x2的幂
28 | cols_name = 'F' + str(i+1) + str(j+1)
29 | X1 = data_init.loc[:, ['Microchip Test 1']]
30 | X2 = data_init.loc[:, ['Microchip Test 2']]
31 | F_ij = np.multiply(np.power(X1, i+1), np.power(X2, j+1))
32 | data_init.insert(loc=loc_nums, column=cols_name, value=F_ij)
33 | loc_nums += 1
34 | # print(data_init.head())
35 |
36 | # 初始化数据
37 | cols = data_init.shape[1]
38 | X = data_init.iloc[:, :-1]
39 | y = data_init.iloc[:, cols-1:cols]
40 | # print(X.head(), '\n', y.head())
41 |
42 | X = np.mat(X.values)
43 | y = np.mat(y.values)
44 | theta = np.mat(np.array([0.0509, -0.1138, 0.1981, -0.4741, -0.2382, 0.0609, -0.1437, 0.1020, -0.0419, -0.3915, -1.1065, -0.4496, -0.6370, -0.3913, -0.4575, 0.1050, -0.1164, 0.1048, -0.0310, 0.0660, 0.0071, -0.2492, -0.5434, -0.1493, -0.2261, -0.1088, -0.1304, 0.0727, -0.1306, 0.0462, -0.0369, 0.0198, -0.0085, -0.1664, -0.3108, -0.0552, -0.1013, -0.0361, -0.0482]))
45 | # theta = np.mat(np.zeros(39))
46 |
47 | alpha = 0.0001
48 | iters = 1000000
49 | lamda = 1
50 |
51 |
52 | def cost_function_regularized(theta, X, y, lamda): # correct √
53 | plus = (lamda / (2 * X.shape[0])) * np.sum(np.power(theta, 2))
54 | return lr.cost_function(X=X, y=y, theta=theta) + plus
55 |
56 |
57 | print(cost_function_regularized(theta, X, y, lamda)) # correct √
58 |
59 |
60 | def gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=True):
61 | temp = np.mat(np.zeros(theta.shape)) # temp暂存theta的值
62 | parameters = int(theta.shape[1]) # parameters控制循环次数
63 | cost = np.zeros(iters) # cost用来存放每次迭代后的代价函数
64 |
65 | for i in range(iters):
66 | subtraction = lr.sigmoid(X * theta.T) - y
67 |
68 | for j in range(parameters):
69 | term = np.multiply(subtraction, X[:, j])
70 | if not j:
71 | temp[0, j] = theta[0, j] - ((alpha / len(X)) * np.sum(term, axis=0)) # len()返回矩阵的行数
72 | else:
73 | temp[0, j] = np.multiply(theta[0, j], 1 - alpha * lamda / len(X)) - ((alpha / len(X)) * np.sum(term, axis=0)) # len()返回矩阵的行数
74 |
75 | theta = temp
76 | cost[i] = cost_function_regularized(X, y, theta, lamda)
77 |
78 | if vistualize:
79 | print('这是第{}次循环, 代价函数的值为{:.4f}' .format(i+1, cost[i]), end=' ')
80 | for k in range(parameters):
81 | print(', Ѳ{:d}为{:.4f}' .format(k, theta[0, k]), end=' ')
82 | print('\n')
83 | for k in range(parameters):
84 | print('{:.4f}, '.format(theta[0, k]), end=' ')
85 | print('\n')
86 | if i != 0 and cost[i] > cost[i-1]: # 检查梯度下降算法是否正常工作
87 | raise ValueError('The value of cost_function becomes larger')
88 |
89 | return cost
90 |
91 | # print(lr.ensure_alpha(X, y, theta, alpha, iters)) # 确定学习速率
92 | # print(gradient_decent_regularized(X, y, theta, lamda))
93 |
94 |
95 | # theta_new, cost = gradient_decent_regularized(theta, X, y, alpha, lamda, vistualize=False)
96 | # expected_result = lr.predict(theta_new, X)
97 | expected_result = opt.fmin_tnc(func=cost_function_regularized, x0=theta, fprime=gradient_decent_regularized, args=(X, y, lamda))
98 | print(expected_result)
99 | lr.compare(expected_result, y)
100 |
101 |
--------------------------------------------------------------------------------
/homework/ex2/Logistic_Regression_Regularlized_Correct.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | import numpy as np
4 | import Logistic_Regression as lr
5 | import scipy.optimize as opt
6 |
7 | # 数据可视化
8 | path = 'D:\Study\Coding\Machine Learning WuEnda\homework\ex2\ex2data2.txt'
9 | data_init = pd.read_csv(path, header=None, names=['Microchip Test 1', 'Microchip Test 2', 'Accepted'])
10 |
11 | positive = data_init[data_init['Accepted'].isin([1])]
12 | negative = data_init[data_init['Accepted'].isin([0])]
13 | fig, ax = plt.subplots(figsize=(12,8))
14 | ax.scatter(positive['Microchip Test 1'], positive['Microchip Test 2'], alpha=0.5, s=50, c='b', marker='o', label='Accepted')
15 | ax.scatter(negative['Microchip Test 1'], negative['Microchip Test 2'], alpha=0.5, s=50, c='r', marker='x', label='Rejected')
16 | ax.legend()
17 | ax.set_xlabel('Test 1 Score')
18 | ax.set_ylabel('Test 2 Score')
19 | # plt.show()
20 |
21 | # 发现不适用于直线拟合，所以我们要创造更多的特征
22 | data_init.insert(0, 'Ones', 1)
23 | print(data_init.head())
24 |
25 | degree = 20
26 | loc_nums = 3
27 | for i in range(degree): # 控制x1的幂
28 | for j in range(degree): # 控制x2的幂
29 | cols_name = 'F' + '('+str(i+1) + ')'+'('+str(j+1)+')'
30 | X1 = data_init.loc[:, ['Microchip Test 1']]
31 | X2 = data_init.loc[:, ['Microchip Test 2']]
32 | F_ij = np.multiply(np.power(X1, i+1), np.power(X2, j+1))
33 | data_init.insert(loc=loc_nums, column=cols_name, value=F_ij)
34 | loc_nums += 1
35 | # print(data_init.head())
36 |
37 | # 初始化数据
38 | cols = data_init.shape[1]
39 | X = data_init.iloc[:, :-1]
40 | y = data_init.iloc[:, cols-1:cols]
41 | print(X.head(), '\n', y.head())
42 |
43 | X = np.mat(X.values)
44 | y = np.mat(y.values)
45 | theta = np.zeros(cols-1)
46 |
47 | learningRate = 1.5
48 |
49 |
50 | def cost_function_regularized(theta, X, y, learningRate): # correct √
51 | theta = np.matrix(theta)
52 | X = np.matrix(X)
53 | y = np.matrix(y)
54 | first = np.multiply(-y, np.log(lr.sigmoid(X * theta.T)))
55 | second = np.multiply((1 - y), np.log(1 - lr.sigmoid(X * theta.T)))
56 | reg = (learningRate / (2 * len(X))) * np.sum(np.power(theta[:, 1:theta.shape[1]], 2))
57 | return np.sum(first - second) / len(X) + reg
58 |
59 |
60 | print(cost_function_regularized(theta, X, y, learningRate)) # correct √
61 |
62 |
63 | def gradient_decent_regularized(theta, X, y, learningRate):
64 | theta = np.matrix(theta)
65 | X = np.matrix(X)
66 | y = np.matrix(y)
67 |
68 | parameters = int(theta.ravel().shape[1])
69 | grad = np.zeros(parameters)
70 |
71 | error = lr.sigmoid(X * theta.T) - y
72 |
73 | for i in range(parameters):
74 | term = np.multiply(error, X[:, i])
75 |
76 | if (i == 0):
77 | grad[i] = np.sum(term) / len(X)
78 | else:
79 | grad[i] = (np.sum(term) / len(X)) + ((learningRate / len(X)) * theta[:, i])
80 |
81 | return grad
82 |
83 |
84 | def predict(theta, X):
85 | probability = lr.sigmoid(X * theta.T)
86 | return [1 if x >= 0.5 else 0 for x in probability]
87 |
88 |
89 | expected_result = opt.fmin_tnc(func=cost_function_regularized, x0=theta, fprime=gradient_decent_regularized, args=(X, y, learningRate))
90 | print(expected_result)
91 |
92 | theta_min = np.matrix(expected_result[0])
93 | predictions = predict(theta_min, X)
94 | correct = [1 if ((a == 1 and b == 1) or (a == 0 and b == 0)) else 0 for (a, b) in zip(predictions, y)]
95 | accuracy = (sum(map(int, correct)) % len(correct))
96 | print('accuracy = {0}%'.format(accuracy))
97 |
98 |
99 |
100 | def hfunc2(theta, x1, x2):
101 | temp = theta[0][0]
102 | place = 0
103 | for i in range(1, degree+1):
104 | for j in range(0, i+1):
105 | temp+= np.power(x1, i-j) * np.power(x2, j) * theta[0][place+1]
106 | place+=1
107 | return temp
108 |
109 |
110 | def find_decision_boundary(theta):
111 | t1 = np.linspace(-1, 1.5, 1000)
112 | t2 = np.linspace(-1, 1.5, 1000)
113 |
114 | cordinates = [(x, y) for x in t1 for y in t2]
115 | x_cord, y_cord = zip(*cordinates)
116 | h_val = pd.DataFrame({'x1':x_cord, 'x2':y_cord})
117 | h_val['hval'] = hfunc2(theta, h_val['x1'], h_val['x2'])
118 |
119 | decision = h_val[np.abs(h_val['hval']) < 2 * 10**-3]
120 | return decision.x1, decision.x2
121 |
122 |
123 | fig, ax = plt.subplots(figsize=(12,8))
124 | ax.scatter(positive['Microchip Test 1'], positive['Microchip Test 2'], s=50, c='b', marker='o', label='Accepted')
125 | ax.scatter(negative['Microchip Test 1'], negative['Microchip Test 2'], s=50, c='r', marker='x', label='Rejected')
126 | ax.set_xlabel('Test 1 Score')
127 | ax.set_ylabel('Test 2 Score')
128 |
129 | x, y = find_decision_boundary(expected_result)
130 | plt.scatter(x, y, c='y', s=10, label='Prediction')
131 | ax.legend()
132 | plt.show()
133 |
--------------------------------------------------------------------------------
/homework/ex2/__pycache__/Logistic_Regression.cpython-37.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex2/__pycache__/Logistic_Regression.cpython-37.pyc
--------------------------------------------------------------------------------
/homework/ex2/ex2.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex2/ex2.pdf
--------------------------------------------------------------------------------
/homework/ex2/ex2data1.txt:
--------------------------------------------------------------------------------
1 | 34.62365962451697,78.0246928153624,0
2 | 30.28671076822607,43.89499752400101,0
3 | 35.84740876993872,72.90219802708364,0
4 | 60.18259938620976,86.30855209546826,1
5 | 79.0327360507101,75.3443764369103,1
6 | 45.08327747668339,56.3163717815305,0
7 | 61.10666453684766,96.51142588489624,1
8 | 75.02474556738889,46.55401354116538,1
9 | 76.09878670226257,87.42056971926803,1
10 | 84.43281996120035,43.53339331072109,1
11 | 95.86155507093572,38.22527805795094,0
12 | 75.01365838958247,30.60326323428011,0
13 | 82.30705337399482,76.48196330235604,1
14 | 69.36458875970939,97.71869196188608,1
15 | 39.53833914367223,76.03681085115882,0
16 | 53.9710521485623,89.20735013750205,1
17 | 69.07014406283025,52.74046973016765,1
18 | 67.94685547711617,46.67857410673128,0
19 | 70.66150955499435,92.92713789364831,1
20 | 76.97878372747498,47.57596364975532,1
21 | 67.37202754570876,42.83843832029179,0
22 | 89.67677575072079,65.79936592745237,1
23 | 50.534788289883,48.85581152764205,0
24 | 34.21206097786789,44.20952859866288,0
25 | 77.9240914545704,68.9723599933059,1
26 | 62.27101367004632,69.95445795447587,1
27 | 80.1901807509566,44.82162893218353,1
28 | 93.114388797442,38.80067033713209,0
29 | 61.83020602312595,50.25610789244621,0
30 | 38.78580379679423,64.99568095539578,0
31 | 61.379289447425,72.80788731317097,1
32 | 85.40451939411645,57.05198397627122,1
33 | 52.10797973193984,63.12762376881715,0
34 | 52.04540476831827,69.43286012045222,1
35 | 40.23689373545111,71.16774802184875,0
36 | 54.63510555424817,52.21388588061123,0
37 | 33.91550010906887,98.86943574220611,0
38 | 64.17698887494485,80.90806058670817,1
39 | 74.78925295941542,41.57341522824434,0
40 | 34.1836400264419,75.2377203360134,0
41 | 83.90239366249155,56.30804621605327,1
42 | 51.54772026906181,46.85629026349976,0
43 | 94.44336776917852,65.56892160559052,1
44 | 82.36875375713919,40.61825515970618,0
45 | 51.04775177128865,45.82270145776001,0
46 | 62.22267576120188,52.06099194836679,0
47 | 77.19303492601364,70.45820000180959,1
48 | 97.77159928000232,86.7278223300282,1
49 | 62.07306379667647,96.76882412413983,1
50 | 91.56497449807442,88.69629254546599,1
51 | 79.94481794066932,74.16311935043758,1
52 | 99.2725269292572,60.99903099844988,1
53 | 90.54671411399852,43.39060180650027,1
54 | 34.52451385320009,60.39634245837173,0
55 | 50.2864961189907,49.80453881323059,0
56 | 49.58667721632031,59.80895099453265,0
57 | 97.64563396007767,68.86157272420604,1
58 | 32.57720016809309,95.59854761387875,0
59 | 74.24869136721598,69.82457122657193,1
60 | 71.79646205863379,78.45356224515052,1
61 | 75.3956114656803,85.75993667331619,1
62 | 35.28611281526193,47.02051394723416,0
63 | 56.25381749711624,39.26147251058019,0
64 | 30.05882244669796,49.59297386723685,0
65 | 44.66826172480893,66.45008614558913,0
66 | 66.56089447242954,41.09209807936973,0
67 | 40.45755098375164,97.53518548909936,1
68 | 49.07256321908844,51.88321182073966,0
69 | 80.27957401466998,92.11606081344084,1
70 | 66.74671856944039,60.99139402740988,1
71 | 32.72283304060323,43.30717306430063,0
72 | 64.0393204150601,78.03168802018232,1
73 | 72.34649422579923,96.22759296761404,1
74 | 60.45788573918959,73.09499809758037,1
75 | 58.84095621726802,75.85844831279042,1
76 | 99.82785779692128,72.36925193383885,1
77 | 47.26426910848174,88.47586499559782,1
78 | 50.45815980285988,75.80985952982456,1
79 | 60.45555629271532,42.50840943572217,0
80 | 82.22666157785568,42.71987853716458,0
81 | 88.9138964166533,69.80378889835472,1
82 | 94.83450672430196,45.69430680250754,1
83 | 67.31925746917527,66.58935317747915,1
84 | 57.23870631569862,59.51428198012956,1
85 | 80.36675600171273,90.96014789746954,1
86 | 68.46852178591112,85.59430710452014,1
87 | 42.0754545384731,78.84478600148043,0
88 | 75.47770200533905,90.42453899753964,1
89 | 78.63542434898018,96.64742716885644,1
90 | 52.34800398794107,60.76950525602592,0
91 | 94.09433112516793,77.15910509073893,1
92 | 90.44855097096364,87.50879176484702,1
93 | 55.48216114069585,35.57070347228866,0
94 | 74.49269241843041,84.84513684930135,1
95 | 89.84580670720979,45.35828361091658,1
96 | 83.48916274498238,48.38028579728175,1
97 | 42.2617008099817,87.10385094025457,1
98 | 99.31500880510394,68.77540947206617,1
99 | 55.34001756003703,64.9319380069486,1
100 | 74.77589300092767,89.52981289513276,1
101 |
--------------------------------------------------------------------------------
/homework/ex2/ex2data2.txt:
--------------------------------------------------------------------------------
1 | 0.051267,0.69956,1
2 | -0.092742,0.68494,1
3 | -0.21371,0.69225,1
4 | -0.375,0.50219,1
5 | -0.51325,0.46564,1
6 | -0.52477,0.2098,1
7 | -0.39804,0.034357,1
8 | -0.30588,-0.19225,1
9 | 0.016705,-0.40424,1
10 | 0.13191,-0.51389,1
11 | 0.38537,-0.56506,1
12 | 0.52938,-0.5212,1
13 | 0.63882,-0.24342,1
14 | 0.73675,-0.18494,1
15 | 0.54666,0.48757,1
16 | 0.322,0.5826,1
17 | 0.16647,0.53874,1
18 | -0.046659,0.81652,1
19 | -0.17339,0.69956,1
20 | -0.47869,0.63377,1
21 | -0.60541,0.59722,1
22 | -0.62846,0.33406,1
23 | -0.59389,0.005117,1
24 | -0.42108,-0.27266,1
25 | -0.11578,-0.39693,1
26 | 0.20104,-0.60161,1
27 | 0.46601,-0.53582,1
28 | 0.67339,-0.53582,1
29 | -0.13882,0.54605,1
30 | -0.29435,0.77997,1
31 | -0.26555,0.96272,1
32 | -0.16187,0.8019,1
33 | -0.17339,0.64839,1
34 | -0.28283,0.47295,1
35 | -0.36348,0.31213,1
36 | -0.30012,0.027047,1
37 | -0.23675,-0.21418,1
38 | -0.06394,-0.18494,1
39 | 0.062788,-0.16301,1
40 | 0.22984,-0.41155,1
41 | 0.2932,-0.2288,1
42 | 0.48329,-0.18494,1
43 | 0.64459,-0.14108,1
44 | 0.46025,0.012427,1
45 | 0.6273,0.15863,1
46 | 0.57546,0.26827,1
47 | 0.72523,0.44371,1
48 | 0.22408,0.52412,1
49 | 0.44297,0.67032,1
50 | 0.322,0.69225,1
51 | 0.13767,0.57529,1
52 | -0.0063364,0.39985,1
53 | -0.092742,0.55336,1
54 | -0.20795,0.35599,1
55 | -0.20795,0.17325,1
56 | -0.43836,0.21711,1
57 | -0.21947,-0.016813,1
58 | -0.13882,-0.27266,1
59 | 0.18376,0.93348,0
60 | 0.22408,0.77997,0
61 | 0.29896,0.61915,0
62 | 0.50634,0.75804,0
63 | 0.61578,0.7288,0
64 | 0.60426,0.59722,0
65 | 0.76555,0.50219,0
66 | 0.92684,0.3633,0
67 | 0.82316,0.27558,0
68 | 0.96141,0.085526,0
69 | 0.93836,0.012427,0
70 | 0.86348,-0.082602,0
71 | 0.89804,-0.20687,0
72 | 0.85196,-0.36769,0
73 | 0.82892,-0.5212,0
74 | 0.79435,-0.55775,0
75 | 0.59274,-0.7405,0
76 | 0.51786,-0.5943,0
77 | 0.46601,-0.41886,0
78 | 0.35081,-0.57968,0
79 | 0.28744,-0.76974,0
80 | 0.085829,-0.75512,0
81 | 0.14919,-0.57968,0
82 | -0.13306,-0.4481,0
83 | -0.40956,-0.41155,0
84 | -0.39228,-0.25804,0
85 | -0.74366,-0.25804,0
86 | -0.69758,0.041667,0
87 | -0.75518,0.2902,0
88 | -0.69758,0.68494,0
89 | -0.4038,0.70687,0
90 | -0.38076,0.91886,0
91 | -0.50749,0.90424,0
92 | -0.54781,0.70687,0
93 | 0.10311,0.77997,0
94 | 0.057028,0.91886,0
95 | -0.10426,0.99196,0
96 | -0.081221,1.1089,0
97 | 0.28744,1.087,0
98 | 0.39689,0.82383,0
99 | 0.63882,0.88962,0
100 | 0.82316,0.66301,0
101 | 0.67339,0.64108,0
102 | 1.0709,0.10015,0
103 | -0.046659,-0.57968,0
104 | -0.23675,-0.63816,0
105 | -0.15035,-0.36769,0
106 | -0.49021,-0.3019,0
107 | -0.46717,-0.13377,0
108 | -0.28859,-0.060673,0
109 | -0.61118,-0.067982,0
110 | -0.66302,-0.21418,0
111 | -0.59965,-0.41886,0
112 | -0.72638,-0.082602,0
113 | -0.83007,0.31213,0
114 | -0.72062,0.53874,0
115 | -0.59389,0.49488,0
116 | -0.48445,0.99927,0
117 | -0.0063364,0.99927,0
118 | 0.63265,-0.030612,0
119 |
--------------------------------------------------------------------------------
/homework/ex2/test.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import matplotlib.pyplot as plt
3 | import numpy as np
4 | import scipy.optimize as opt
5 |
6 |
7 | for i in range(6): # 控制x1的幂
8 | for j in range(6): # 控制x2的幂
9 | print('i={},j={}' .format(i, j))
10 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/GrepConsole.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
38 |
39 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | enda
5 | fmin
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/ex3.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex3/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
86 |
87 |
88 |
89 |
90 |
91 |
92 |
93 |
94 |
95 |
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
134 |
135 |
136 |
137 | C:\Users\钱先明\AppData\Roaming\Subversion
138 |
139 |
140 |
141 |
142 | 1589285025678
143 |
144 |
145 | 1589285025678
146 |
147 |
148 |
149 |
150 |
151 |
152 |
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
165 |
166 |
167 |
168 |
169 |
170 |
171 |
172 |
173 |
174 |
175 |
176 |
177 |
178 |
179 |
180 |
181 |
182 |
183 |
184 |
185 |
186 |
187 |
188 |
189 |
190 |
191 |
192 |
193 |
194 |
195 |
196 |
197 |
198 |
199 |
200 |
201 |
202 |
203 |
204 |
205 |
206 |
207 |
208 |
209 |
210 |
--------------------------------------------------------------------------------
/homework/ex3/Multi-class Classification and Neural Networks.py:
--------------------------------------------------------------------------------
1 | import pandas as pd
2 | import numpy as np
3 | import matplotlib.pyplot as plt
4 | import matplotlib
5 | from scipy.io import loadmat
6 | from scipy.optimize import minimize
7 | from sklearn.metrics import classification_report # 这个是评价报告
8 |
9 |
10 | def sigmoid(z):
11 | return 1 / (1 + np.exp(-z))
12 |
13 |
14 | def cost_function(theta, X, y, learning_rate):
15 | theta = np.mat(theta)
16 | X = np.mat(X)
17 | y = np.mat(y)
18 | h_theta = sigmoid(X * theta.T)
19 |
20 | first = np.multiply(-y, np.log(h_theta))
21 | second = np.multiply(-(1 - y), np.log(1 - h_theta))
22 |
23 | reg = (learning_rate / (2 * len(X))) * np.sum(np.power(theta[:, 1:theta.shape[1]], 2)) # 注意这里不包含θ0
24 | return np.sum(first + second) / len(X) + reg
25 |
26 |
27 | def gradient(theta, X, y, learning_rate):
28 | theta = np.mat(theta)
29 | X = np.mat(X)
30 | y = np.mat(y)
31 |
32 | parameters = int(theta.ravel().shape[1])
33 | error = sigmoid(X @ theta.T) - y
34 |
35 | grad = ((X.T @ error) / len(X)).T + ((learning_rate / len(X)) * theta)
36 |
37 | # intercept gradient is not regularized
38 | grad[0, 0] = np.sum(np.multiply(error, X[:, 0])) / len(X)
39 |
40 | return np.array(grad).ravel()
41 |
42 |
43 | def one_vs_all(X, y, num_labels, learning_rate):
44 | rows = X.shape[0]
45 | params = X.shape[1]
46 |
47 | # k * (n + 1) array for the parameters of each of the k classifiers
48 | all_theta = np.zeros((num_labels, params + 1))
49 |
50 | # insert a column of ones at the beginning for the intercept term
51 | X = np.insert(X, 0, values=np.ones(rows), axis=1)
52 |
53 | # labels are 1-indexed instead of 0-indexed
54 | for i in range(1, num_labels + 1):
55 | theta = np.zeros(params + 1)
56 | y_i = np.array([1 if label == i else 0 for label in y])
57 | y_i = np.reshape(y_i, (rows, 1))
58 |
59 | # minimize the objective function
60 | fmin = minimize(fun=cost_function, x0=theta, args=(X, y_i, learning_rate), method='TNC', jac=gradient)
61 | all_theta[i - 1, :] = fmin.x
62 |
63 | return all_theta
64 |
65 |
66 | def predict_all(X, all_theta):
67 | rows = X.shape[0]
68 | params = X.shape[1]
69 | num_labels = all_theta.shape[0]
70 |
71 | # same as before, insert ones to match the shape
72 | X = np.insert(X, 0, values=np.ones(rows), axis=1)
73 |
74 | # convert to matrices
75 | X = np.mat(X)
76 | all_theta = np.mat(all_theta)
77 |
78 | # compute the class probability for each class on each training instance
79 | h = sigmoid(X @ all_theta.T)
80 |
81 | # create array of the index with the maximum probability
82 | h_argmax = np.argmax(h, axis=1)
83 |
84 | # because our array was zero-indexed we need to add one for the true label prediction
85 | h_argmax = h_argmax + 1
86 |
87 | return h_argmax
88 |
89 |
90 |
91 | # 读取数据
92 | path = 'D:/Study/MachineLearning/Machine Learning WuEnda/homework/ex3/ex3data1.mat'
93 | data = loadmat(path)
94 | # print(data['X'], '\n', data['y'])
95 | # print(data['X'].shape, data['y'].shape)
96 |
97 | # 数据可视化
98 | sample_index = np.random.choice(np.arange(data['X'].shape[0]), 100) # 从5000行中选取100行作为样本
99 | sample_images = data['X'][sample_index, :]
100 | # print(sample_images)
101 |
102 | fig, ax_array = plt.subplots(nrows=10, ncols=10, sharex='all', sharey='all', figsize=(12, 12))
103 | for r in range(10):
104 | for c in range(10):
105 | ax_array[r, c].matshow(np.array(sample_images[10 * r + c].reshape(20, 20)).T, cmap=matplotlib.cm.binary)
106 | plt.xticks(np.array([]))
107 | plt.yticks(np.array([]))
108 | plt.show()
109 |
110 | rows = data['X'].shape[0]
111 | params = data['X'].shape[1]
112 |
113 | all_theta = np.zeros((10, params + 1))
114 |
115 | X = np.insert(data['X'], 0, values=np.ones(rows), axis=1)
116 |
117 | theta = np.zeros(params + 1)
118 |
119 | y_0 = np.array([1 if label == 0 else 0 for label in data['y']])
120 | y_0 = np.reshape(y_0, (rows, 1))
121 |
122 | # print(X.shape, y_0.shape, theta.shape, all_theta.shape)
123 | # 去除数组中的重复数字，并进行排序之后输出
124 | # print(np.unique(data['y']))
125 |
126 | # 训练开始
127 | # all_theta = one_vs_all(data['X'], data['y'], 10, 0.1)
128 | # print(all_theta)
129 |
130 | # y_pred = predict_all(data['X'], all_theta)
131 | # print(classification_report(data['y'], y_pred))
132 |
133 |
134 | # 实现前馈神经网络预测
135 | weight = loadmat('D:/Study/MachineLearning/Machine Learning WuEnda/homework/ex3/ex3weights.mat')
136 | theta1, theta2 = weight['Theta1'], weight['Theta2']
137 | # print(theta1.shape, theta2.shape)
138 |
139 | X2 = np.mat(np.insert(data['X'], 0, values=np.ones(X.shape[0]), axis=1))
140 | y2 = np.mat(data['y'])
141 |
142 | a1 = X2
143 | z2 = a1 * theta1.T
144 | # print(z2.shape)
145 |
146 | a2 = sigmoid(z2)
147 | # print(a2.shape)
148 |
149 | a2 = np.insert(a2, 0, values=np.ones(a2.shape[0]), axis=1)
150 | z3 = a2 * theta2.T
151 | # print(z3.shape)
152 |
153 | a3 = sigmoid(z3)
154 | # print(a3)
155 |
156 | y_pred2 = np.argmax(a3, axis=1) + 1 # 取出a3中元素最大值所对应的索引 (axis=0是行，1是列)
157 | # print(y_pred2.shape)
158 |
159 | print(classification_report(y2, y_pred2))
160 |
161 |
--------------------------------------------------------------------------------
/homework/ex3/ex3.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex3/ex3.pdf
--------------------------------------------------------------------------------
/homework/ex3/ex3data1.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex3/ex3data1.mat
--------------------------------------------------------------------------------
/homework/ex3/ex3weights.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex3/ex3weights.mat
--------------------------------------------------------------------------------
/homework/ex4/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml
--------------------------------------------------------------------------------
/homework/ex4/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | backpropagation
5 | fmin
6 | matshow
7 |
8 |
9 |
--------------------------------------------------------------------------------
/homework/ex4/.idea/ex4.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex4/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex4/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex4/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex4/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex4/ex4.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex4/ex4.pdf
--------------------------------------------------------------------------------
/homework/ex4/ex4.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import pandas as pd
3 | import matplotlib.pyplot as plt
4 | import matplotlib
5 | from scipy.io import loadmat
6 | from scipy.optimize import minimize
7 |
8 | from sklearn.preprocessing import OneHotEncoder
9 | from sklearn.metrics import classification_report # 这个包是评价报告
10 |
11 | from tqdm import tqdm
12 |
13 |
14 | def sigmoid(z):
15 | return 1 / (1 + np.exp(-z))
16 |
17 | def sigmoid_gradient(z):
18 | g_z = sigmoid(z)
19 | return np.multiply(g_z, 1-g_z)
20 |
21 | def forward_propagate(X, theta1, theta2): # only one hidden unit
22 | m = X.shape[0]
23 | a1 = np.insert(X, 0, values=np.ones(m), axis=1)
24 | z2 = np.dot(a1, theta1.T)
25 | a2 = np.insert(sigmoid(z2), 0, values=np.ones(m), axis=1)
26 | z3 = np.dot(a2, theta2.T)
27 | h = sigmoid(z3)
28 |
29 | return a1, z2, a2, z3, h
30 |
31 | def cost_function(X, y, theta1, theta2, learning_rate=1, reg=True):
32 | h = forward_propagate(X, theta1, theta2)[-1]
33 | m = X.shape[0]
34 |
35 | part_one = np.multiply(y, np.log(h)) + np.multiply((1-y), np.log(1-h))
36 | all_one = - np.sum(part_one) / m
37 |
38 | if reg: # 判断是否需要正则化
39 | part_two = np.power(theta1[:, 1:], 2).sum() + np.power(theta2[:, 1:], 2).sum() # 正则化项
40 | all_two = part_two * learning_rate / (2 * m)
41 | J = all_one + all_two
42 | else:
43 | J = all_one
44 |
45 | return J
46 |
47 | def back_propagation(params, input_size, hidden_size, num_labels, X, y, learning_rate=1, reg=True):
48 | m = X.shape[0]
49 |
50 | # reshape the parameter array into parameter matrices for each layer
51 | # 将输入的params分成theta11,2
52 | theta1 = np.matrix(np.reshape(params[:hidden_size * (input_size + 1)], (hidden_size, (input_size + 1))))
53 | theta2 = np.matrix(np.reshape(params[hidden_size * (input_size + 1):], (num_labels, (hidden_size + 1))))
54 |
55 | # run the feed-forward pass
56 | a1, z2, a2, z3, h = forward_propagate(X, theta1, theta2)
57 |
58 | # initializations
59 | J = 0
60 | delta1 = np.zeros(theta1.shape) # (25, 401)
61 | delta2 = np.zeros(theta2.shape) # (10, 26)
62 |
63 | # compute the cost
64 | for i in range(m):
65 | first_term = np.multiply(-y[i, :], np.log(h[i, :]))
66 | second_term = np.multiply((1 - y[i, :]), np.log(1 - h[i, :]))
67 | J += np.sum(first_term - second_term)
68 |
69 | J = J / m
70 |
71 | # add the cost regularization term
72 | if reg:
73 | J += (float(learning_rate) / (2 * m)) * (np.sum(np.power(theta1[:, 1:], 2)) + np.sum(np.power(theta2[:, 1:], 2)))
74 |
75 | # perform backpropagation
76 | for t in tqdm(range(m)):
77 | a1t = a1[t, :] # (1, 401)
78 | z2t = z2[t, :] # (1, 25)
79 | a2t = a2[t, :] # (1, 26)
80 | ht = h[t, :] # (1, 10)
81 | yt = y[t, :] # (1, 10)
82 |
83 | d3t = ht - yt # (1, 10)
84 |
85 | z2t = np.insert(z2t, 0, values=np.ones(1)) # (1, 26)
86 | d2t = np.multiply((theta2.T * d3t.T).T, sigmoid_gradient(z2t)) # (1, 26)
87 |
88 | delta1 = delta1 + (d2t[:, 1:]).T * a1t
89 | delta2 = delta2 + d3t.T * a2t
90 |
91 | delta1 = delta1 / m
92 | delta2 = delta2 / m
93 |
94 | # add the gradient regularization term
95 | # 除去bias项都加上正则化
96 | delta1[:, 1:] = delta1[:, 1:] + (theta1[:, 1:] * learning_rate) / m
97 | delta2[:, 1:] = delta2[:, 1:] + (theta2[:, 1:] * learning_rate) / m
98 |
99 | # unravel the gradient matrices into a single array
100 | grad = np.hstack((np.ravel(delta1), np.ravel(delta2)))
101 |
102 | # just a show
103 |
104 |
105 | return J, grad
106 |
107 | def gradient_check(X, y, theta1, theta2, e=0.001):
108 | print('Start checking gradient: ')
109 | theta_all = np.hstack((np.ravel(theta1), np.ravel(theta2))).T.reshape(-1, 1) # (10285, 1)
110 | f_theta = np.zeros(theta_all.shape).reshape(-1, 1)
111 |
112 | for i in tqdm(range(theta_all.shape[0])):
113 | e_add = np.zeros(theta_all.shape).reshape(-1, 1)
114 | e_add[i, 0] = e
115 | theta_minus = theta_all - e_add
116 | theta_plus = theta_all + e_add
117 |
118 | theta_minus_1 = theta_minus[:theta1.shape[0] * theta1.shape[1], 0].reshape(theta1.shape)
119 | theta_minus_2 = theta_minus[theta1.shape[0] * theta1.shape[1]:, 0].reshape(theta2.shape)
120 | theta_plus_1 = theta_plus[:theta1.shape[0] * theta1.shape[1], 0].reshape(theta1.shape)
121 | theta_plus_2 = theta_plus[theta1.shape[0] * theta1.shape[1]:, 0].reshape(theta2.shape)
122 |
123 | f_theta[i, 0] = (cost_function(X, y, theta_minus_1, theta_minus_2) - cost_function(X, y, theta_plus_1, theta_plus_2)) / (2 * e)
124 |
125 | return f_theta
126 |
127 | # 读取数据
128 | data = loadmat('D:/Study/MachineLearning/Machine Learning WuEnda/homework/ex4/ex4data1.mat')
129 |
130 | X = data['X']
131 | y = data['y']
132 | print(X.shape, y.shape)
133 |
134 | encoder = OneHotEncoder(sparse=False) # 对y进行独热编码
135 | y_onehot = encoder.fit_transform(y)
136 | print(y_onehot.shape)
137 |
138 | weight = loadmat('D:/Study/MachineLearning/Machine Learning WuEnda/homework/ex4/ex4weights.mat')
139 | theta1, theta2 = weight['Theta1'], weight['Theta2']
140 | print(theta1.shape, theta2.shape)
141 |
142 | # 数据可视化
143 | sample_index = np.random.choice(np.arange(data['X'].shape[0]), 100, replace=False) # 从5000行中选取100行作为样本,replace=False表示不能取相同的数字
144 | sample_images = data['X'][sample_index, :] # sample_index行,所有列
145 | # print(sample_images.shape)
146 |
147 | fig, ax_array = plt.subplots(nrows=10, ncols=10, sharex='all', sharey='all', figsize=(12, 12))
148 | for r in range(10):
149 | for c in range(10):
150 | # matshow是按色块画图
151 | ax_array[r, c].matshow(np.array(sample_images[10 * r + c].reshape(20, 20)).T, cmap=matplotlib.cm.binary)
152 | plt.xticks(np.array([]))
153 | plt.yticks(np.array([]))
154 | plt.show()
155 |
156 |
157 | # 对theta做随机初始化
158 | e1 = 0.12
159 | e2 = 0.414
160 |
161 | # 初始化设置
162 | input_size = 400
163 | hidden_size = 25
164 | num_labels = 10
165 | learning_rate = 1
166 |
167 | cost = cost_function(X, y_onehot, theta1, theta2, learning_rate=1)
168 | # print(cost)
169 |
170 | # np.random.random(size) 返回size大小的0-1随机浮点数, 初始化theta
171 | params = (np.random.random(size=hidden_size * (input_size + 1) + num_labels * (hidden_size + 1)) - 0.5) * 0.24
172 | print(params.shape) # (10285,)
173 |
174 | # print(back_propagation(params, input_size, hidden_size, num_labels, X, y_onehot, learning_rate=1))
175 |
176 | # 使用工具库计算参数最优解
177 | # minimize the objective function
178 | '''
179 | minimize里面参数解释:
180 | fun : 要计算的函数
181 | x0 : 函数里面传入的第一个参数,通常是自变量或者要优化的值的初始值
182 | args : 函数里面传入的其他的参数，要用tuple打包
183 | method : 优化方法
184 | jac :
185 | options : 其他设置
186 | '''
187 | fmin = minimize(fun=back_propagation, x0=params, args=(input_size, hidden_size, num_labels, X, y_onehot, learning_rate),
188 | method='TNC', jac=True, options={'maxiter': 250})
189 | # print(fmin)
190 |
191 | X = np.matrix(X)
192 | theta1_final = np.matrix(np.reshape(fmin.x[:hidden_size * (input_size + 1)], (hidden_size, (input_size + 1))))
193 | theta2_final = np.matrix(np.reshape(fmin.x[hidden_size * (input_size + 1):], (num_labels, (hidden_size + 1))))
194 |
195 |
196 | # 检测back propagation是否正常运行，记得检查一次后关闭gradient_check(),这里只计算第一次训练(套用5000个数据)的theta
197 | # params_check = np.hstack((theta1.ravel(), theta2.ravel()))
198 | # print(params_check.shape)
199 | # gradient_check = gradient_check(X, y_onehot, theta1, theta2)
200 | # gradient_train = back_propagation(params_check, input_size, hidden_size, num_labels, X, y_onehot)[1]
201 | # 求誤差
202 | # error = np.fabs(gradient_check - gradient_train).mean()
203 | # print('-------------------------------------------')
204 | # print(error)
205 | # print('-------------------------------------------')
206 |
207 |
208 | # 计算使用优化后的θ得出的预测
209 | a1, z2, a2, z3, h = forward_propagate(X, theta1_final, theta2_final)
210 | y_pred = np.array(np.argmax(h, axis=1) + 1)
211 | print(y_pred)
212 |
213 | # 预测值与实际值比较
214 | print(classification_report(y, y_pred)) # 完美 √
215 |
216 | # 可视化hidden layer
217 | hidden_layer = theta1_final[:, 1:]
218 | fig, ax_array = plt.subplots(nrows=5, ncols=5, sharey='all', sharex='all', figsize=(12, 12))
219 | for r in range(5):
220 | for c in range(5):
221 | ax_array[r, c].matshow(np.array(hidden_layer[5 * r + c].reshape((20, 20))), cmap=matplotlib.cm.binary)
222 | plt.xticks(np.array([]))
223 | plt.yticks(np.array([]))
224 | plt.show()
225 |
226 |
227 |
--------------------------------------------------------------------------------
/homework/ex4/ex4data1.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex4/ex4data1.mat
--------------------------------------------------------------------------------
/homework/ex4/ex4weights.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex4/ex4weights.mat
--------------------------------------------------------------------------------
/homework/ex4/gradient_check.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex4/gradient_check.png
--------------------------------------------------------------------------------
/homework/ex4/part1.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex4/part1.png
--------------------------------------------------------------------------------
/homework/ex5/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml
3 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | ndarry
5 | overfit
6 | underfit
7 | xtest
8 | xval
9 | ytest
10 | yval
11 |
12 |
13 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/ex5.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex5/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex5/ex5.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex5/ex5.pdf
--------------------------------------------------------------------------------
/homework/ex5/ex5.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import scipy.io as sio
3 | import scipy.optimize as opt
4 | import pandas as pd
5 | import matplotlib.pyplot as plt
6 | import seaborn as sns
7 |
8 | data = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex5\ex5data1.mat')
9 | # map() 会根据提供的函数对指定序列做映射
10 | X, y, X_val, y_val, X_test, y_test = map(lambda x: np.reshape(x, (-1, 1)), [data['X'], data['y'], data['Xval'], data['yval'], data['Xtest'], data['ytest']])
11 | print(X.shape, '\n', y.shape) # (12, 1) (12, 1)
12 | print(type(X))
13 | print(y)
14 |
15 | plt.figure(figsize=(12, 8))
16 | ax1 = plt.subplot2grid((2, 3), (0, 0), colspan=1, rowspan=1)
17 | ax1.scatter(X, y, label='Training data', alpha=0.7)
18 | ax1.set_xlabel('water_level')
19 | ax1.set_ylabel('flow')
20 |
21 | def gradient(theta, X, y, reg):
22 | theta = np.mat(theta)
23 | m = len(X)
24 | inner = np.dot(X, theta.T) - y
25 | grad = np.zeros(theta.shape)
26 |
27 | for j in range(theta.shape[1]):
28 | grad_first = np.sum(np.multiply(inner, X[:, [j]])) / m
29 | if j == 0:
30 | grad[0, j] = grad_first
31 | else:
32 | grad_reg = (reg * theta[0, j]) / m
33 | grad[0, j] = grad_first + grad_reg
34 |
35 | return grad
36 |
37 | def linear_regression(X, y, l=1.0):
38 | """linear regression
39 | args:
40 | X: feature matrix, (m, n+1) # with intercept x0=1
41 | y: target vector, (m, )
42 | l: lambda constant for regularization
43 |
44 | return: trained parameters
45 | """
46 | # init theta
47 | _theta = np.ones(X.shape[1])
48 |
49 | # train it
50 | result = opt.minimize(fun=cost_function, # target minimize function
51 | x0=_theta,
52 | args=(X, y, l),
53 | method='TNC',
54 | jac=gradient, # a function that calculate the gradient
55 | options={'disp': True})
56 | return result
57 |
58 | X, X_val, X_test = [np.insert(x, 0, np.ones(x.shape[0]), axis=1) for x in (X, X_val, X_test)]
59 |
60 | # 数据处理
61 | def cost_function(theta, X, y, reg=0):
62 | X = np.mat(X)
63 | y = np.mat(y)
64 | theta = np.mat(theta)
65 | m = len(X)
66 |
67 | h_theta = np.dot(X, theta.T) # (m, 2) @ (2, 1) -> (m, 1)
68 | J_first = np.sum(np.power(h_theta - y, 2)) / (2 * m)
69 | J_second = reg * np.sum(np.power(theta, 2)) / (2 * m)
70 |
71 | J = J_first + J_second
72 | return J
73 |
74 | theta = np.ones(X.shape[1])
75 | theta_final = opt.minimize(fun=cost_function, x0=theta, args=(X, y, 0), method='TNC', jac=gradient, options={'disp': True}).x
76 | print(theta_final)
77 | print(cost_function(theta_final, X, y, reg=0))
78 | # print(gradient(theta_final, X, y, reg=1))
79 |
80 | theta_0 = theta_final[0]
81 | theta_1 = theta_final[1]
82 | x_plot1 = np.linspace(-50, 50)
83 | y_plot1 = theta_0 + theta_1 * x_plot1
84 | ax1.plot(x_plot1, y_plot1, color='red', label='Prediction')
85 | ax1.legend()
86 |
87 |
88 | # plot the learning curves
89 | def cv_train_cost_compute(X, y, X_val, y_val, l=1.0):
90 | training_cost, cv_cost = [], []
91 | m = X.shape[0]
92 | # 用循环记录下不同training dataset下的cost，然后append到列表里，画出learning curve
93 | for i in range(1, m+1):
94 | # 用X, y训练模型, 然后分别算出训练集和交叉验证集上的J(θ)
95 | model = linear_regression(X[:i, :], y[:i, :], l)
96 |
97 | train_error = cost_function(model.x, X[:i, :], y[:i, :])
98 | cv_error = cost_function(model.x, X_val, y_val)
99 | training_cost.append(train_error)
100 | cv_cost.append(cv_error)
101 |
102 | return training_cost, cv_cost
103 |
104 |
105 | training_cost, cv_cost = cv_train_cost_compute(X, y, X_val, y_val)
106 | ax2 = plt.subplot2grid((2, 3), (0, 1), colspan=1, rowspan=1)
107 | ax2.plot(np.arange(1, len(X)+1), training_cost, label='training cost')
108 | ax2.plot(np.arange(1, len(X)+1), cv_cost, label='cross validation cost')
109 | ax2.legend()
110 | ax2.set_xlabel('numbers of training data')
111 | ax2.set_ylabel('J(θ)')
112 |
113 | # 此时发现模型underfit, 于是增加多项式特征
114 | def add_polynomial(X, num):
115 | """add polynomial features to X
116 | args:
117 | X: a matrix that to be add (*, 2) (already add cols x0=1)
118 | num: number of polynomial
119 | normal: whether to do normalization
120 | return: a new X
121 | """
122 | orgin = X[:, 1]
123 | for i in range(num):
124 | add_col = np.power(orgin, i+2)
125 | X = np.insert(X, obj=X.shape[1], values=add_col, axis=1)
126 |
127 | return X
128 |
129 | X_new, X_val, X_test = map(add_polynomial, [X, X_val, X_test], [5, 5, 5]) ####
130 |
131 | # 将X扩展到8阶并使用归一化
132 | def normalize_feature(X, method='Z'):
133 | """to normalize feature
134 | :param X: type() --> np.ndarry, X already have a col, namely x0 = 1
135 | :param method: how to normalize, include 'Z'(Z-score), '01'((0,1)normalize), 'sigmoid'
136 | :return: a new X
137 | """
138 | for i in range(1, X.shape[1]):
139 | each_col = X[:, i]
140 | if method == 'Z': # (X - mu) / sigma
141 | each_col = (each_col - np.average(each_col)) / np.std(each_col)
142 |
143 | elif method == '01': # (X - max) / (max - min)
144 | each_col = (each_col - np.max(each_col)) / (np.max(each_col) - np.min(each_col))
145 |
146 | elif method == 'sigmoid': # 1 / (1 + e^(-x))
147 | each_col = 1 / (1 + np.exp(-each_col))
148 |
149 | X[:, i] = each_col
150 |
151 | return X
152 |
153 | X_new, X_val, X_test = map(normalize_feature, [X_new, X_val, X_test]) # (12, 10)
154 | # print(X_new)
155 |
156 | # 下一步，画出learning curve，判断是underfit还是overfit
157 | poly_training_cost, poly_cv_cost = cv_train_cost_compute(X_new, y, X_val, y_val, l=0.55) ####
158 | ax3 = plt.subplot2grid((2, 3), (1, 1), colspan=1, rowspan=1)
159 | ax3.plot(np.arange(1, len(X_new)+1), poly_training_cost, label='training cost')
160 | ax3.plot(np.arange(1, len(X_new)+1), poly_cv_cost, label='cross validation cost')
161 | ax3.set_xlabel('numbers of training data')
162 | ax3.set_ylabel('J(θ)')
163 | ax3.legend()
164 |
165 | # 训练θ
166 | theta_new = linear_regression(X_new, y, l=0.55).x.reshape(-1, 1) ####
167 | print(theta_new)
168 |
169 | ax4 = plt.subplot2grid((2, 3), (1, 0), colspan=1, rowspan=1)
170 | x_temp = np.insert(np.linspace(-50, 50).reshape(-1, 1), 0, np.ones(np.linspace(-50, 50).reshape(-1, 1).shape[0]), axis=1)
171 | x_temp = normalize_feature(add_polynomial(x_temp, 5)) ####
172 | y_plot2 = np.dot(x_temp, theta_new)
173 |
174 | ax4.plot(np.linspace(-50, 50), y_plot2, color='red', label='Prediction')
175 | ax4.scatter(np.delete(X, 0, axis=1), y, label='Training data', alpha=0.7)
176 | ax4.set_xlabel('water_level')
177 | ax4.set_ylabel('flow')
178 | ax4.legend()
179 |
180 | # 选取最适合的λ
181 | l_candidate = [0, 0.01, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.8]
182 | training_cost, cv_cost = [], []
183 | for l in l_candidate:
184 | theta_plot = linear_regression(X_new, y, l).x
185 | training_error = cost_function(theta_plot, X_new, y, reg=l)
186 | cv_error = cost_function(theta_plot, X_val, y_val, reg=l)
187 |
188 | training_cost.append(training_error)
189 | cv_cost.append(cv_error)
190 |
191 | ax5 = plt.subplot2grid((2, 3), (0, 2), rowspan=1, colspan=1)
192 | ax5.plot(l_candidate, training_cost, label='training cost')
193 | ax5.plot(l_candidate, cv_cost, label='cross validation cost')
194 | ax5.set_xlabel('λ')
195 | ax5.set_ylabel('J(θ)')
196 | ax5.legend()
197 | plt.show()
198 |
199 | # 在测试集上进行测试
200 | # use test data to compute the cost
201 | for l in l_candidate:
202 | theta = linear_regression(X_new, y, l).x
203 | print('test cost(l={}) = {}'.format(l, cost_function(theta, X_test, y_test)))
204 |
205 | # 计算测试集上的准确率
206 | y_pred = np.dot(X_test, theta_new)
207 | mse = np.sum(np.square(y_test - y_pred)) / len(y_pred)
208 | print("MSE为: ", mse)
209 |
210 | '''
211 | 最终参数：
212 | λ=0.55
213 | 增添到x^6系数项
214 | 最终MSE: 14.86687
215 | '''
216 |
217 |
--------------------------------------------------------------------------------
/homework/ex5/ex5data1.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex5/ex5data1.mat
--------------------------------------------------------------------------------
/homework/ex6/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml
--------------------------------------------------------------------------------
/homework/ex6/.idea/dictionaries/.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | alist
5 | cordinates
6 | enda
7 | xval
8 | yval
9 |
10 |
11 |
--------------------------------------------------------------------------------
/homework/ex6/.idea/ex6.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex6/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex6/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex6/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex6/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex6/SpamClassification.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import pandas as pd
4 | from pprint import pprint
5 | from tqdm import tqdm
6 |
7 | import scipy.io as sio
8 | import scipy.optimize as opt
9 | from sklearn import svm
10 | from sklearn.model_selection import train_test_split, cross_val_score
11 |
12 | # 引用一些自定义的函数
13 | def divide_dataset(data, train_size=0.6, cv_size=0.3, test_size=0.1, nums=3):
14 | """
15 | 分为三类, 训练集, 交叉验证集, 测试集, 默认比例6: 3: 1
16 | """
17 | # 数据集的行数
18 | m = data.shape[0]
19 |
20 | # 打乱重排
21 | data = data.sample(frac=1.0)
22 |
23 | train_pro, cv_pro, test_pro = map(lambda x: round(m * x), [train_size, cv_size, test_size])
24 |
25 | # 检查是否有0出现，若有则替换为1
26 | alist = [train_pro, cv_pro, test_pro]
27 | for i in range(3):
28 | if not alist[i]:
29 | alist[i] = 1
30 |
31 | data_train = data[: train_pro].reset_index(drop=True)
32 | data_cv = data[train_pro: train_pro + cv_pro].reset_index(drop=True)
33 | data_test = data[-test_pro:].reset_index(drop=True)
34 |
35 | return data_train, data_cv, data_test
36 |
37 | # 进行参数筛选
38 | def select_params(data, gamma_list=None, c_list=None, method='SVC', max_iter=1000):
39 | best_score = 0
40 | best_parameters = {}
41 | for c in tqdm(c_list):
42 |
43 | if method == 'SVC':
44 | for gamma in gamma_list:
45 |
46 | # 对于每种参数可能的组合，进行一次训练
47 | svc = svm.SVC(gamma=gamma, C=c, max_iter=max_iter)
48 |
49 | # 5 折交叉验证
50 | scores = cross_val_score(svc, data.iloc[:, :-1], data[['y']].values.ravel(), cv=10)
51 | score = scores.mean()
52 | # print('gamma={}, C={}, score is {}' .format(gamma, c, score))
53 |
54 | # 找到表现最好的参数
55 | if score > best_score:
56 | best_parameters = {'gamma': gamma, "C": c}
57 | best_score = score
58 |
59 | elif method == 'LinearSVC':
60 |
61 | # 对于每种参数可能的组合，进行一次训练
62 | svc = svm.LinearSVC(C=c, max_iter=max_iter)
63 |
64 | # 5 折交叉验证
65 | scores = cross_val_score(svc, data.iloc[:, :-1], data[['y']].values.ravel(), cv=10)
66 | score = scores.mean()
67 | # print('C={}, score is {}' .format(gamma, c, score))
68 |
69 | # 找到表现最好的参数
70 | if score > best_score:
71 | best_parameters = {"C": c}
72 | best_score = score
73 |
74 | if method == 'SVC':
75 | print('THE BEST : gamma={}, C={}, score is {}' .format(best_parameters['gamma'], best_parameters['C'], best_score))
76 | elif method == 'LinearSVC':
77 | print('THE BEST : C={}, score is {}' .format(best_parameters['C'], best_score))
78 |
79 | return best_parameters
80 |
81 | # 读取数据
82 | raw_data = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\spamTrain.mat')
83 | # pprint(raw_data)
84 |
85 | data = pd.DataFrame(raw_data.get('X'), columns=['X' + str(x) for x in range(1, 1900)])
86 | data['y'] = raw_data.get('y')
87 | # print(data.shape)
88 |
89 | raw_data_test = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\spamTest.mat')
90 | data_test = pd.DataFrame(raw_data_test.get('Xtest'), columns=['X' + str(x) for x in range(1, 1900)])
91 | data_test['y'] = raw_data_test.get('ytest')
92 | # print(data_test.head())
93 | # print(data_test.shape)
94 |
95 | # 读取词频表
96 | word_list = []
97 | with open('vocab.txt', 'r') as f:
98 | for each_line in f.readlines():
99 | word_list.append(each_line.split('\t')[1][:-1])
100 | # print(word_list)
101 |
102 | # 选取训练参数
103 | # best_params = select_params(data=data, c_list=np.arange(0.017, 0.025, 0.001), method='LinearSVC', max_iter=30000)
104 |
105 | # 开始训练模型, 选用线性svm
106 | # svc = svm.LinearSVC(**best_params)
107 | svc = svm.LinearSVC(C=0.021)
108 | svc.fit(data.iloc[:, :-1], data[['y']].values.ravel())
109 |
110 | print('Training accuracy = {0}%'.format(np.round(svc.score(data.iloc[:, :-1], data[['y']].values.ravel()) * 100, 3)))
111 | print('Test accuracy = {0}%'.format(np.round(svc.score(data_test.iloc[:, :-1], data_test[['y']].values.ravel()) * 100, 3)))
112 |
113 | # 可视化拟合结果
114 | kw = np.eye(1899)
115 | spam_val = pd.DataFrame({'idx': range(1899)}) # 创建1899维的单位矩阵
116 | spam_val['is_spam'] = svc.decision_function(kw)
117 | # print(spam_val['is_spam'].describe())
118 | # print(spam_val.head())
119 |
120 | decision = spam_val[spam_val['is_spam'] > 0].sort_values(by=['is_spam'], ascending=False).reset_index(drop=True)
121 | decision['word'] = [word_list[i] for i in decision['idx']]
122 | # print(decision.head(10))
123 |
124 | # 读取输入的邮件
125 | path = 'D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\spamSample2.txt'
126 | with open(input('请输入邮件的本地地址: '), 'r') as f:
127 | all_words = []
128 | symbol = '!@#$%^&*(),.-?><;:|\"\\\'*`~'
129 | for each_line in f.readlines():
130 | for each_word in each_line.split():
131 | # print(each_word)
132 | each_word = each_word.casefold()
133 |
134 | if each_word[-1] in symbol:
135 | each_word = each_word[:-1]
136 | elif each_word[0] in symbol:
137 | if len(each_word) == 1:
138 | continue
139 | else:
140 | each_word = each_word[1:]
141 | elif each_word == '\n':
142 | continue
143 |
144 |
145 | if not each_word:
146 | continue
147 |
148 | elif each_word.isalpha() and each_word in word_list:
149 | all_words.append(each_word)
150 |
151 | elif each_word[-1] == 's' and each_word not in word_list:
152 | each_word = each_word[:-1]
153 | if each_word in word_list:
154 | all_words.append(each_word)
155 |
156 | elif each_word[-3:] == 'ing':
157 | each_word = each_word[:-3]
158 | if each_word in word_list:
159 | all_words.append(each_word)
160 |
161 | # print(all_words)
162 |
163 | all_words_loc = [word_list.index(each) for each in all_words]
164 | X_test = np.zeros((1, 1899))
165 |
166 | for each in all_words_loc:
167 | X_test[0, each] = 1
168 |
169 | predict = svc.predict(X_test)
170 | if predict:
171 | print('是垃圾邮件')
172 | else:
173 | print('不是垃圾邮件')
174 |
175 |
--------------------------------------------------------------------------------
/homework/ex6/emailSample1.txt:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/emailSample1.txt
--------------------------------------------------------------------------------
/homework/ex6/emailSample2.txt:
--------------------------------------------------------------------------------
1 | Folks,
2 |
3 | my first time posting - have a bit of Unix experience, but am new to Linux.
4 |
5 |
6 | Just got a new PC at home - Dell box with Windows XP. Added a second hard disk
7 | for Linux. Partitioned the disk and have installed Suse 7.2 from CD, which went
8 | fine except it didn't pick up my monitor.
9 |
10 | I have a Dell branded E151FPp 15" LCD flat panel monitor and a nVidia GeForce4
11 | Ti4200 video card, both of which are probably too new to feature in Suse's default
12 | set. I downloaded a driver from the nVidia website and installed it using RPM.
13 | Then I ran Sax2 (as was recommended in some postings I found on the net), but
14 | it still doesn't feature my video card in the available list. What next?
15 |
16 | Another problem. I have a Dell branded keyboard and if I hit Caps-Lock twice,
17 | the whole machine crashes (in Linux, not Windows) - even the on/off switch is
18 | inactive, leaving me to reach for the power cable instead.
19 |
20 | If anyone can help me in any way with these probs., I'd be really grateful -
21 | I've searched the 'net but have run out of ideas.
22 |
23 | Or should I be going for a different version of Linux such as RedHat? Opinions
24 | welcome.
25 |
26 | Thanks a lot,
27 | Peter
28 |
29 | --
30 | Irish Linux Users' Group: ilug@linux.ie
31 | http://www.linux.ie/mailman/listinfo/ilug for (un)subscription information.
32 | List maintainer: listmaster@linux.ie
33 |
34 |
35 |
--------------------------------------------------------------------------------
/homework/ex6/ex6.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/ex6.pdf
--------------------------------------------------------------------------------
/homework/ex6/ex6.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import pandas as pd
4 | from pprint import pprint
5 | from tqdm import tqdm
6 |
7 | import scipy.io as sio
8 | import scipy.optimize as opt
9 | from sklearn import svm
10 | from sklearn.model_selection import train_test_split, cross_val_score
11 |
12 | raw_data = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\ex6data1.mat')
13 | # pprint(raw_data)
14 | data = pd.DataFrame(raw_data.get('X'), columns=['X1', 'X2'])
15 | data['y'] = raw_data.get('y')
16 | # print(data.head())
17 |
18 | # 进行划分数据集，train, cross_validation, test
19 | def divide_dataset(data, train_size=0.6, cv_size=0.3, test_size=0.1, nums=3):
20 | """
21 | 分为三类, 训练集, 交叉验证集, 测试集, 默认比例6: 3: 1
22 | """
23 | # 数据集的行数
24 | m = data.shape[0]
25 |
26 | # 打乱重排
27 | data = data.sample(frac=1.0)
28 |
29 | train_pro, cv_pro, test_pro = map(lambda x: round(m * x), [train_size, cv_size, test_size])
30 |
31 | # 检查是否有0出现，若有则替换为1
32 | alist = [train_pro, cv_pro, test_pro]
33 | for i in range(3):
34 | if not alist[i]:
35 | alist[i] = 1
36 |
37 | data_train = data[: train_pro].reset_index(drop=True)
38 | data_cv = data[train_pro: train_pro + cv_pro].reset_index(drop=True)
39 | data_test = data[-test_pro:].reset_index(drop=True)
40 |
41 | return data_train, data_cv, data_test
42 |
43 | data_train, data_cv, data_test = divide_dataset(data)
44 |
45 | # 先进行可视化
46 | plt.figure(figsize=(12, 8))
47 | ax1 = plt.subplot2grid((2, 2), loc=(0, 0), rowspan=1, colspan=1)
48 |
49 | def plot_init_data(data, ax, title=None):
50 | positive = data[data['y'].isin([1])]
51 | negative = data[data['y'].isin([0])]
52 |
53 | ax.scatter(positive['X1'], positive['X2'], s=20, marker='x', label='Positive')
54 | ax.scatter(negative['X1'], negative['X2'], s=20, marker='o', label='Negative')
55 | ax.legend()
56 | ax.set_title(title)
57 |
58 | plot_init_data(data, ax1, title='SVM (C=1) Decision Boundary')
59 |
60 | # 开始训练
61 | svc = svm.LinearSVC(C=1, loss='hinge', max_iter=10000)
62 | svc.fit(data[['X1', 'X2']], data['y'])
63 | print(svc.score(data[['X1', 'X2']], data['y']))
64 |
65 | # 可视化分类边界
66 | x = np.linspace(0, 5)
67 |
68 | def find_decision_boundary(svc, x1min, x1max, x2min, x2max, diff):
69 | x1 = np.linspace(x1min, x1max, 1000)
70 | x2 = np.linspace(x2min, x2max, 1000)
71 |
72 | # 先绘制网格，也就是全平面的所有点
73 | cordinates = [(x, y) for x in x1 for y in x2]
74 | x_cord, y_cord = zip(*cordinates)
75 | c_val = pd.DataFrame({'x1': x_cord, 'x2': y_cord})
76 |
77 | # decision_function(X)返回样本到超平面的符号距离
78 | c_val['c_val'] = svc.decision_function(c_val[['x1', 'x2']])
79 |
80 | # 将到超平面距离小于diff的样本返回, 也就是调整线宽
81 | decision = c_val[np.abs(c_val['c_val']) < diff]
82 |
83 | return decision.x1, decision.x2
84 |
85 | x1, x2 = find_decision_boundary(svc, 0, 4, 1.5, 5, 1 * 10**-3)
86 | ax1.scatter(x1, x2, s=10, c='r', label='Boundary')
87 | ax1.legend()
88 |
89 | # 然后在第26行改变C的值, 观察曲线变化
90 |
91 |
92 |
93 |
94 | # 开始使用具有高斯核函数内核的SVM
95 | def gaussian_kernel(x1, x2, sigma):
96 | return np.exp(-(np.sum((x1 - x2) ** 2) / (2 * (sigma ** 2))))
97 |
98 | # 测试高斯函数
99 | x1 = np.array([1.0, 2.0, 1.0])
100 | x2 = np.array([0.0, 4.0, -1.0])
101 | sigma = 2
102 | print(gaussian_kernel(x1, x2, sigma))
103 |
104 |
105 | # 开始使用dataset2进行拟合，注意变量的替换
106 | raw_data = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\ex6data2.mat')
107 |
108 | data = pd.DataFrame(raw_data['X'], columns=['X1', 'X2'])
109 | data['y'] = raw_data['y']
110 |
111 | # 划分为三个数据集
112 | data_train, data_cv, data_test = divide_dataset(data)
113 |
114 | # 进行参数筛选
115 | def select_params(data, gamma_list, c_list, method='SVC'):
116 | best_score = 0
117 | best_parameters = {}
118 | for gamma in tqdm(gamma_list):
119 | for c in c_list:
120 |
121 | # 对于每种参数可能的组合，进行一次训练
122 | if method == 'SVC':
123 | svc = svm.SVC(gamma=gamma, C=c)
124 | elif method == 'LinearSVC':
125 | svc = svm.LinearSVC(gamma=gamma, C=c)
126 | else:
127 | svc = None
128 |
129 | # 5 折交叉验证
130 | scores = cross_val_score(svc, data[['X1', 'X2']], data[['y']].values.ravel(), cv=10)
131 | score = scores.mean()
132 | # print('gamma={}, C={}, score is {}' .format(gamma, c, score))
133 |
134 | # 找到表现最好的参数
135 | if score > best_score:
136 | best_parameters = {'gamma': gamma, "C": c}
137 | best_score = score
138 |
139 | print('THE BEST : gamma={}, C={}, score is {}' .format(best_parameters['gamma'], best_parameters['C'], best_score))
140 |
141 | return best_parameters
142 |
143 | best_parameters = select_params(data=data, gamma_list=range(50, 70, 5), c_list=[0.1, 1, 10, 50, 100])
144 |
145 | # 使用最佳参数，构建新的模型
146 | svc2 = svm.SVC(**best_parameters)
147 |
148 | # 使用训练集和验证集进行训练
149 | svc2.fit(data[['X1', 'X2']], data['y'])
150 |
151 | ax2 = plt.subplot2grid((2, 2), (0, 1), rowspan=1, colspan=1)
152 | plot_init_data(data, ax2, title='SVM With Gaussian_Kernel')
153 |
154 | # 开始训练模型
155 | # svc2 = svm.SVC(C=100, kernel='rbf', gamma=10, probability=True)
156 | # print(svc2)
157 |
158 | # svc2.fit(data[['X1', 'X2']], data['y'])
159 | print(svc2.score(data[['X1', 'X2']], data['y']))
160 |
161 | # 超平面边界可视化
162 | x1, x2 = find_decision_boundary(svc2, 0, 1.1, 0.35, 1.1, 7 * 10**-3)
163 | ax2.scatter(x1, x2, s=5, c='r', label='Boundary', alpha=0.4)
164 | ax2.legend()
165 |
166 |
167 |
168 | # 进行dataset3的数据分类
169 | raw_data = sio.loadmat('D:\Study\MachineLearning\Machine Learning WuEnda\homework\ex6\ex6data3.mat')
170 | # pprint(raw_data)
171 |
172 | data = pd.DataFrame(raw_data.get('X'), columns=['X1', 'X2'])
173 | data['y'] = raw_data.get('y')
174 | data_val = pd.DataFrame(raw_data.get('Xval'), columns=['X1', 'X2'])
175 | data_val['y'] = raw_data.get('yval')
176 | # print(data.head(), data_val.head())
177 |
178 | # 原始数据可视化
179 | ax3 = plt.subplot2grid((2, 2), (1, 0), rowspan=1, colspan=1)
180 | plot_init_data(data, ax3, title='plot3')
181 |
182 | # 开始筛选参数
183 | best_parameters = select_params(data=data, gamma_list=[0.1, 1, 10, 50, 100], c_list=[0.1, 1, 10, 50, 100], method='SVC')
184 |
185 | svc3 = svm.SVC(**best_parameters)
186 | svc3.fit(data[['X1', 'X2']], data['y'])
187 | print(svc3.score(data_val[['X1', 'X2']], data_val['y']))
188 |
189 | x1, x2 = find_decision_boundary(svc3, -0.6, 0.3, -0.7, 0.6, 2 * 10**-3)
190 | ax3.scatter(x1, x2, s=3, c='r', label='Boundary', alpha=0.4)
191 | ax3.legend()
192 |
193 | plt.show()
194 |
--------------------------------------------------------------------------------
/homework/ex6/ex6data1.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/ex6data1.mat
--------------------------------------------------------------------------------
/homework/ex6/ex6data2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/ex6data2.mat
--------------------------------------------------------------------------------
/homework/ex6/ex6data3.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/ex6data3.mat
--------------------------------------------------------------------------------
/homework/ex6/spamSample1.txt:
--------------------------------------------------------------------------------
1 | Do You Want To Make $1000 Or More Per Week?
2 |
3 |
4 |
5 | If you are a motivated and qualified individual - I
6 | will personally demonstrate to you a system that will
7 | make you $1,000 per week or more! This is NOT mlm.
8 |
9 |
10 |
11 | Call our 24 hour pre-recorded number to get the
12 | details.
13 |
14 |
15 |
16 | 000-456-789
17 |
18 |
19 |
20 | I need people who want to make serious money. Make
21 | the call and get the facts.
22 |
23 | Invest 2 minutes in yourself now!
24 |
25 |
26 |
27 | 000-456-789
28 |
29 |
30 |
31 | Looking forward to your call and I will introduce you
32 | to people like yourself who
33 | are currently making $10,000 plus per week!
34 |
35 |
36 |
37 | 000-456-789
38 |
39 |
40 |
41 | 3484lJGv6-241lEaN9080lRmS6-271WxHo7524qiyT5-438rjUv5615hQcf0-662eiDB9057dMtVl72
42 |
43 |
--------------------------------------------------------------------------------
/homework/ex6/spamSample2.txt:
--------------------------------------------------------------------------------
1 | Best Buy Viagra Generic Online
2 |
3 | Viagra 100mg x 60 Pills $125, Free Pills & Reorder Discount, Top Selling 100% Quality & Satisfaction guaranteed!
4 |
5 | We accept VISA, Master & E-Check Payments, 90000+ Satisfied Customers!
6 | http://medphysitcstech.ru
7 |
8 |
9 |
--------------------------------------------------------------------------------
/homework/ex6/spamTest.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/spamTest.mat
--------------------------------------------------------------------------------
/homework/ex6/spamTrain.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex6/spamTrain.mat
--------------------------------------------------------------------------------
/homework/ex6/test.py:
--------------------------------------------------------------------------------
1 | import numpy as np
2 | import matplotlib.pyplot as plt
3 | import pandas as pd
4 | from pprint import pprint
5 |
6 | import scipy.io as sio
7 | import scipy.optimize as opt
8 | from sklearn import svm
9 | from sklearn.model_selection import train_test_split
10 |
11 | data = pd.DataFrame({'x1': [i for i in range(10)], 'x2': [i**2 for i in range(10)],
12 | 'x3': [0, 0, 0, 0, 0, 1, 1, 1, 1, 1]})
13 | print(data)
14 |
15 | data_train, data_cv = train_test_split(data, test_size=0.4, random_state=1, stratify=data['x3'])
16 | data_left, data_test = train_test_split(data.drop(data_train), test_size=0.25, random_state=1, stratify=data['x3'])
17 | print(data_cv.reset_index(drop=True))
18 | print(data_train.reset_index(drop=True))
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
29 |
--------------------------------------------------------------------------------
/homework/ex6/vocab.txt:
--------------------------------------------------------------------------------
1 | 1 aa
2 | 2 ab
3 | 3 abil
4 | 4 abl
5 | 5 about
6 | 6 abov
7 | 7 absolut
8 | 8 abus
9 | 9 ac
10 | 10 accept
11 | 11 access
12 | 12 accord
13 | 13 account
14 | 14 achiev
15 | 15 acquir
16 | 16 across
17 | 17 act
18 | 18 action
19 | 19 activ
20 | 20 actual
21 | 21 ad
22 | 22 adam
23 | 23 add
24 | 24 addit
25 | 25 address
26 | 26 administr
27 | 27 adult
28 | 28 advanc
29 | 29 advantag
30 | 30 advertis
31 | 31 advic
32 | 32 advis
33 | 33 ae
34 | 34 af
35 | 35 affect
36 | 36 affili
37 | 37 afford
38 | 38 africa
39 | 39 after
40 | 40 ag
41 | 41 again
42 | 42 against
43 | 43 agenc
44 | 44 agent
45 | 45 ago
46 | 46 agre
47 | 47 agreement
48 | 48 aid
49 | 49 air
50 | 50 al
51 | 51 alb
52 | 52 align
53 | 53 all
54 | 54 allow
55 | 55 almost
56 | 56 alon
57 | 57 along
58 | 58 alreadi
59 | 59 alsa
60 | 60 also
61 | 61 altern
62 | 62 although
63 | 63 alwai
64 | 64 am
65 | 65 amaz
66 | 66 america
67 | 67 american
68 | 68 among
69 | 69 amount
70 | 70 amp
71 | 71 an
72 | 72 analysi
73 | 73 analyst
74 | 74 and
75 | 75 ani
76 | 76 anim
77 | 77 announc
78 | 78 annual
79 | 79 annuiti
80 | 80 anoth
81 | 81 answer
82 | 82 anti
83 | 83 anumb
84 | 84 anybodi
85 | 85 anymor
86 | 86 anyon
87 | 87 anyth
88 | 88 anywai
89 | 89 anywher
90 | 90 aol
91 | 91 ap
92 | 92 apolog
93 | 93 app
94 | 94 appar
95 | 95 appear
96 | 96 appl
97 | 97 appli
98 | 98 applic
99 | 99 appreci
100 | 100 approach
101 | 101 approv
102 | 102 apt
103 | 103 ar
104 | 104 archiv
105 | 105 area
106 | 106 aren
107 | 107 argument
108 | 108 arial
109 | 109 arm
110 | 110 around
111 | 111 arrai
112 | 112 arriv
113 | 113 art
114 | 114 articl
115 | 115 artist
116 | 116 as
117 | 117 ascii
118 | 118 ask
119 | 119 asset
120 | 120 assist
121 | 121 associ
122 | 122 assum
123 | 123 assur
124 | 124 at
125 | 125 atol
126 | 126 attach
127 | 127 attack
128 | 128 attempt
129 | 129 attent
130 | 130 attornei
131 | 131 attract
132 | 132 audio
133 | 133 aug
134 | 134 august
135 | 135 author
136 | 136 auto
137 | 137 autom
138 | 138 automat
139 | 139 avail
140 | 140 averag
141 | 141 avoid
142 | 142 awai
143 | 143 awar
144 | 144 award
145 | 145 ba
146 | 146 babi
147 | 147 back
148 | 148 background
149 | 149 backup
150 | 150 bad
151 | 151 balanc
152 | 152 ban
153 | 153 bank
154 | 154 bar
155 | 155 base
156 | 156 basenumb
157 | 157 basi
158 | 158 basic
159 | 159 bb
160 | 160 bc
161 | 161 bd
162 | 162 be
163 | 163 beat
164 | 164 beberg
165 | 165 becaus
166 | 166 becom
167 | 167 been
168 | 168 befor
169 | 169 begin
170 | 170 behalf
171 | 171 behavior
172 | 172 behind
173 | 173 believ
174 | 174 below
175 | 175 benefit
176 | 176 best
177 | 177 beta
178 | 178 better
179 | 179 between
180 | 180 bf
181 | 181 big
182 | 182 bill
183 | 183 billion
184 | 184 bin
185 | 185 binari
186 | 186 bit
187 | 187 black
188 | 188 blank
189 | 189 block
190 | 190 blog
191 | 191 blood
192 | 192 blue
193 | 193 bnumber
194 | 194 board
195 | 195 bodi
196 | 196 boi
197 | 197 bonu
198 | 198 book
199 | 199 boot
200 | 200 border
201 | 201 boss
202 | 202 boston
203 | 203 botan
204 | 204 both
205 | 205 bottl
206 | 206 bottom
207 | 207 boundari
208 | 208 box
209 | 209 brain
210 | 210 brand
211 | 211 break
212 | 212 brian
213 | 213 bring
214 | 214 broadcast
215 | 215 broker
216 | 216 browser
217 | 217 bug
218 | 218 bui
219 | 219 build
220 | 220 built
221 | 221 bulk
222 | 222 burn
223 | 223 bush
224 | 224 busi
225 | 225 but
226 | 226 button
227 | 227 by
228 | 228 byte
229 | 229 ca
230 | 230 cabl
231 | 231 cach
232 | 232 calcul
233 | 233 california
234 | 234 call
235 | 235 came
236 | 236 camera
237 | 237 campaign
238 | 238 can
239 | 239 canada
240 | 240 cannot
241 | 241 canon
242 | 242 capabl
243 | 243 capillari
244 | 244 capit
245 | 245 car
246 | 246 card
247 | 247 care
248 | 248 career
249 | 249 carri
250 | 250 cartridg
251 | 251 case
252 | 252 cash
253 | 253 cat
254 | 254 catch
255 | 255 categori
256 | 256 caus
257 | 257 cb
258 | 258 cc
259 | 259 cd
260 | 260 ce
261 | 261 cell
262 | 262 cent
263 | 263 center
264 | 264 central
265 | 265 centuri
266 | 266 ceo
267 | 267 certain
268 | 268 certainli
269 | 269 cf
270 | 270 challeng
271 | 271 chanc
272 | 272 chang
273 | 273 channel
274 | 274 char
275 | 275 charact
276 | 276 charg
277 | 277 charset
278 | 278 chat
279 | 279 cheap
280 | 280 check
281 | 281 cheer
282 | 282 chief
283 | 283 children
284 | 284 china
285 | 285 chip
286 | 286 choic
287 | 287 choos
288 | 288 chri
289 | 289 citi
290 | 290 citizen
291 | 291 civil
292 | 292 claim
293 | 293 class
294 | 294 classifi
295 | 295 clean
296 | 296 clear
297 | 297 clearli
298 | 298 click
299 | 299 client
300 | 300 close
301 | 301 clue
302 | 302 cnet
303 | 303 cnumber
304 | 304 co
305 | 305 code
306 | 306 collect
307 | 307 colleg
308 | 308 color
309 | 309 com
310 | 310 combin
311 | 311 come
312 | 312 comfort
313 | 313 command
314 | 314 comment
315 | 315 commentari
316 | 316 commerci
317 | 317 commiss
318 | 318 commit
319 | 319 common
320 | 320 commun
321 | 321 compani
322 | 322 compar
323 | 323 comparison
324 | 324 compat
325 | 325 compet
326 | 326 competit
327 | 327 compil
328 | 328 complet
329 | 329 comprehens
330 | 330 comput
331 | 331 concentr
332 | 332 concept
333 | 333 concern
334 | 334 condit
335 | 335 conf
336 | 336 confer
337 | 337 confid
338 | 338 confidenti
339 | 339 config
340 | 340 configur
341 | 341 confirm
342 | 342 conflict
343 | 343 confus
344 | 344 congress
345 | 345 connect
346 | 346 consid
347 | 347 consolid
348 | 348 constitut
349 | 349 construct
350 | 350 consult
351 | 351 consum
352 | 352 contact
353 | 353 contain
354 | 354 content
355 | 355 continu
356 | 356 contract
357 | 357 contribut
358 | 358 control
359 | 359 conveni
360 | 360 convers
361 | 361 convert
362 | 362 cool
363 | 363 cooper
364 | 364 copi
365 | 365 copyright
366 | 366 core
367 | 367 corpor
368 | 368 correct
369 | 369 correspond
370 | 370 cost
371 | 371 could
372 | 372 couldn
373 | 373 count
374 | 374 countri
375 | 375 coupl
376 | 376 cours
377 | 377 court
378 | 378 cover
379 | 379 coverag
380 | 380 crash
381 | 381 creat
382 | 382 creativ
383 | 383 credit
384 | 384 critic
385 | 385 cross
386 | 386 cultur
387 | 387 current
388 | 388 custom
389 | 389 cut
390 | 390 cv
391 | 391 da
392 | 392 dagga
393 | 393 dai
394 | 394 daili
395 | 395 dan
396 | 396 danger
397 | 397 dark
398 | 398 data
399 | 399 databas
400 | 400 datapow
401 | 401 date
402 | 402 dave
403 | 403 david
404 | 404 dc
405 | 405 de
406 | 406 dead
407 | 407 deal
408 | 408 dear
409 | 409 death
410 | 410 debt
411 | 411 decad
412 | 412 decid
413 | 413 decis
414 | 414 declar
415 | 415 declin
416 | 416 decor
417 | 417 default
418 | 418 defend
419 | 419 defens
420 | 420 defin
421 | 421 definit
422 | 422 degre
423 | 423 delai
424 | 424 delet
425 | 425 deliv
426 | 426 deliveri
427 | 427 dell
428 | 428 demand
429 | 429 democrat
430 | 430 depart
431 | 431 depend
432 | 432 deposit
433 | 433 describ
434 | 434 descript
435 | 435 deserv
436 | 436 design
437 | 437 desir
438 | 438 desktop
439 | 439 despit
440 | 440 detail
441 | 441 detect
442 | 442 determin
443 | 443 dev
444 | 444 devel
445 | 445 develop
446 | 446 devic
447 | 447 di
448 | 448 dial
449 | 449 did
450 | 450 didn
451 | 451 diet
452 | 452 differ
453 | 453 difficult
454 | 454 digit
455 | 455 direct
456 | 456 directli
457 | 457 director
458 | 458 directori
459 | 459 disabl
460 | 460 discount
461 | 461 discov
462 | 462 discoveri
463 | 463 discuss
464 | 464 disk
465 | 465 displai
466 | 466 disposit
467 | 467 distanc
468 | 468 distribut
469 | 469 dn
470 | 470 dnumber
471 | 471 do
472 | 472 doc
473 | 473 document
474 | 474 doe
475 | 475 doer
476 | 476 doesn
477 | 477 dollar
478 | 478 dollarac
479 | 479 dollarnumb
480 | 480 domain
481 | 481 don
482 | 482 done
483 | 483 dont
484 | 484 doubl
485 | 485 doubt
486 | 486 down
487 | 487 download
488 | 488 dr
489 | 489 draw
490 | 490 dream
491 | 491 drive
492 | 492 driver
493 | 493 drop
494 | 494 drug
495 | 495 due
496 | 496 dure
497 | 497 dvd
498 | 498 dw
499 | 499 dynam
500 | 500 ea
501 | 501 each
502 | 502 earli
503 | 503 earlier
504 | 504 earn
505 | 505 earth
506 | 506 easi
507 | 507 easier
508 | 508 easili
509 | 509 eat
510 | 510 eb
511 | 511 ebai
512 | 512 ec
513 | 513 echo
514 | 514 econom
515 | 515 economi
516 | 516 ed
517 | 517 edg
518 | 518 edit
519 | 519 editor
520 | 520 educ
521 | 521 eff
522 | 522 effect
523 | 523 effici
524 | 524 effort
525 | 525 either
526 | 526 el
527 | 527 electron
528 | 528 elimin
529 | 529 els
530 | 530 email
531 | 531 emailaddr
532 | 532 emerg
533 | 533 empir
534 | 534 employ
535 | 535 employe
536 | 536 en
537 | 537 enabl
538 | 538 encod
539 | 539 encourag
540 | 540 end
541 | 541 enemi
542 | 542 enenkio
543 | 543 energi
544 | 544 engin
545 | 545 english
546 | 546 enhanc
547 | 547 enjoi
548 | 548 enough
549 | 549 ensur
550 | 550 enter
551 | 551 enterpris
552 | 552 entertain
553 | 553 entir
554 | 554 entri
555 | 555 enumb
556 | 556 environ
557 | 557 equal
558 | 558 equip
559 | 559 equival
560 | 560 error
561 | 561 especi
562 | 562 essenti
563 | 563 establish
564 | 564 estat
565 | 565 estim
566 | 566 et
567 | 567 etc
568 | 568 euro
569 | 569 europ
570 | 570 european
571 | 571 even
572 | 572 event
573 | 573 eventu
574 | 574 ever
575 | 575 everi
576 | 576 everyon
577 | 577 everyth
578 | 578 evid
579 | 579 evil
580 | 580 exactli
581 | 581 exampl
582 | 582 excel
583 | 583 except
584 | 584 exchang
585 | 585 excit
586 | 586 exclus
587 | 587 execut
588 | 588 exercis
589 | 589 exist
590 | 590 exmh
591 | 591 expand
592 | 592 expect
593 | 593 expens
594 | 594 experi
595 | 595 expert
596 | 596 expir
597 | 597 explain
598 | 598 explor
599 | 599 express
600 | 600 extend
601 | 601 extens
602 | 602 extra
603 | 603 extract
604 | 604 extrem
605 | 605 ey
606 | 606 fa
607 | 607 face
608 | 608 fact
609 | 609 factor
610 | 610 fail
611 | 611 fair
612 | 612 fall
613 | 613 fals
614 | 614 famili
615 | 615 faq
616 | 616 far
617 | 617 fast
618 | 618 faster
619 | 619 fastest
620 | 620 fat
621 | 621 father
622 | 622 favorit
623 | 623 fax
624 | 624 fb
625 | 625 fd
626 | 626 featur
627 | 627 feder
628 | 628 fee
629 | 629 feed
630 | 630 feedback
631 | 631 feel
632 | 632 femal
633 | 633 few
634 | 634 ffffff
635 | 635 ffnumber
636 | 636 field
637 | 637 fight
638 | 638 figur
639 | 639 file
640 | 640 fill
641 | 641 film
642 | 642 filter
643 | 643 final
644 | 644 financ
645 | 645 financi
646 | 646 find
647 | 647 fine
648 | 648 finish
649 | 649 fire
650 | 650 firewal
651 | 651 firm
652 | 652 first
653 | 653 fit
654 | 654 five
655 | 655 fix
656 | 656 flag
657 | 657 flash
658 | 658 flow
659 | 659 fnumber
660 | 660 focu
661 | 661 folder
662 | 662 folk
663 | 663 follow
664 | 664 font
665 | 665 food
666 | 666 for
667 | 667 forc
668 | 668 foreign
669 | 669 forev
670 | 670 forget
671 | 671 fork
672 | 672 form
673 | 673 format
674 | 674 former
675 | 675 fortun
676 | 676 forward
677 | 677 found
678 | 678 foundat
679 | 679 four
680 | 680 franc
681 | 681 free
682 | 682 freedom
683 | 683 french
684 | 684 freshrpm
685 | 685 fri
686 | 686 fridai
687 | 687 friend
688 | 688 from
689 | 689 front
690 | 690 ftoc
691 | 691 ftp
692 | 692 full
693 | 693 fulli
694 | 694 fun
695 | 695 function
696 | 696 fund
697 | 697 further
698 | 698 futur
699 | 699 ga
700 | 700 gain
701 | 701 game
702 | 702 gari
703 | 703 garrigu
704 | 704 gave
705 | 705 gcc
706 | 706 geek
707 | 707 gener
708 | 708 get
709 | 709 gif
710 | 710 gift
711 | 711 girl
712 | 712 give
713 | 713 given
714 | 714 global
715 | 715 gnome
716 | 716 gnu
717 | 717 gnupg
718 | 718 go
719 | 719 goal
720 | 720 god
721 | 721 goe
722 | 722 gold
723 | 723 gone
724 | 724 good
725 | 725 googl
726 | 726 got
727 | 727 govern
728 | 728 gpl
729 | 729 grand
730 | 730 grant
731 | 731 graphic
732 | 732 great
733 | 733 greater
734 | 734 ground
735 | 735 group
736 | 736 grow
737 | 737 growth
738 | 738 gt
739 | 739 guarante
740 | 740 guess
741 | 741 gui
742 | 742 guid
743 | 743 ha
744 | 744 hack
745 | 745 had
746 | 746 half
747 | 747 ham
748 | 748 hand
749 | 749 handl
750 | 750 happen
751 | 751 happi
752 | 752 hard
753 | 753 hardwar
754 | 754 hat
755 | 755 hate
756 | 756 have
757 | 757 haven
758 | 758 he
759 | 759 head
760 | 760 header
761 | 761 headlin
762 | 762 health
763 | 763 hear
764 | 764 heard
765 | 765 heart
766 | 766 heaven
767 | 767 hei
768 | 768 height
769 | 769 held
770 | 770 hello
771 | 771 help
772 | 772 helvetica
773 | 773 her
774 | 774 herba
775 | 775 here
776 | 776 hermio
777 | 777 hettinga
778 | 778 hi
779 | 779 high
780 | 780 higher
781 | 781 highli
782 | 782 highlight
783 | 783 him
784 | 784 histori
785 | 785 hit
786 | 786 hold
787 | 787 home
788 | 788 honor
789 | 789 hope
790 | 790 host
791 | 791 hot
792 | 792 hour
793 | 793 hous
794 | 794 how
795 | 795 howev
796 | 796 hp
797 | 797 html
798 | 798 http
799 | 799 httpaddr
800 | 800 huge
801 | 801 human
802 | 802 hundr
803 | 803 ibm
804 | 804 id
805 | 805 idea
806 | 806 ident
807 | 807 identifi
808 | 808 idnumb
809 | 809 ie
810 | 810 if
811 | 811 ignor
812 | 812 ii
813 | 813 iii
814 | 814 iiiiiiihnumberjnumberhnumberjnumberhnumb
815 | 815 illeg
816 | 816 im
817 | 817 imag
818 | 818 imagin
819 | 819 immedi
820 | 820 impact
821 | 821 implement
822 | 822 import
823 | 823 impress
824 | 824 improv
825 | 825 in
826 | 826 inc
827 | 827 includ
828 | 828 incom
829 | 829 increas
830 | 830 incred
831 | 831 inde
832 | 832 independ
833 | 833 index
834 | 834 india
835 | 835 indian
836 | 836 indic
837 | 837 individu
838 | 838 industri
839 | 839 info
840 | 840 inform
841 | 841 initi
842 | 842 inlin
843 | 843 innov
844 | 844 input
845 | 845 insert
846 | 846 insid
847 | 847 instal
848 | 848 instanc
849 | 849 instant
850 | 850 instead
851 | 851 institut
852 | 852 instruct
853 | 853 insur
854 | 854 int
855 | 855 integr
856 | 856 intel
857 | 857 intellig
858 | 858 intend
859 | 859 interact
860 | 860 interest
861 | 861 interfac
862 | 862 intern
863 | 863 internet
864 | 864 interview
865 | 865 into
866 | 866 intro
867 | 867 introduc
868 | 868 inumb
869 | 869 invest
870 | 870 investig
871 | 871 investor
872 | 872 invok
873 | 873 involv
874 | 874 ip
875 | 875 ireland
876 | 876 irish
877 | 877 is
878 | 878 island
879 | 879 isn
880 | 880 iso
881 | 881 isp
882 | 882 issu
883 | 883 it
884 | 884 item
885 | 885 itself
886 | 886 jabber
887 | 887 jame
888 | 888 java
889 | 889 jim
890 | 890 jnumberiiiiiiihepihepihf
891 | 891 job
892 | 892 joe
893 | 893 john
894 | 894 join
895 | 895 journal
896 | 896 judg
897 | 897 judgment
898 | 898 jul
899 | 899 juli
900 | 900 jump
901 | 901 june
902 | 902 just
903 | 903 justin
904 | 904 keep
905 | 905 kei
906 | 906 kept
907 | 907 kernel
908 | 908 kevin
909 | 909 keyboard
910 | 910 kid
911 | 911 kill
912 | 912 kind
913 | 913 king
914 | 914 kingdom
915 | 915 knew
916 | 916 know
917 | 917 knowledg
918 | 918 known
919 | 919 la
920 | 920 lack
921 | 921 land
922 | 922 languag
923 | 923 laptop
924 | 924 larg
925 | 925 larger
926 | 926 largest
927 | 927 laser
928 | 928 last
929 | 929 late
930 | 930 later
931 | 931 latest
932 | 932 launch
933 | 933 law
934 | 934 lawrenc
935 | 935 le
936 | 936 lead
937 | 937 leader
938 | 938 learn
939 | 939 least
940 | 940 leav
941 | 941 left
942 | 942 legal
943 | 943 lender
944 | 944 length
945 | 945 less
946 | 946 lesson
947 | 947 let
948 | 948 letter
949 | 949 level
950 | 950 lib
951 | 951 librari
952 | 952 licens
953 | 953 life
954 | 954 lifetim
955 | 955 light
956 | 956 like
957 | 957 limit
958 | 958 line
959 | 959 link
960 | 960 linux
961 | 961 list
962 | 962 listen
963 | 963 littl
964 | 964 live
965 | 965 ll
966 | 966 lo
967 | 967 load
968 | 968 loan
969 | 969 local
970 | 970 locat
971 | 971 lock
972 | 972 lockergnom
973 | 973 log
974 | 974 long
975 | 975 longer
976 | 976 look
977 | 977 lose
978 | 978 loss
979 | 979 lost
980 | 980 lot
981 | 981 love
982 | 982 low
983 | 983 lower
984 | 984 lowest
985 | 985 lt
986 | 986 ma
987 | 987 mac
988 | 988 machin
989 | 989 made
990 | 990 magazin
991 | 991 mai
992 | 992 mail
993 | 993 mailer
994 | 994 main
995 | 995 maintain
996 | 996 major
997 | 997 make
998 | 998 maker
999 | 999 male
1000 | 1000 man
1001 | 1001 manag
1002 | 1002 mani
1003 | 1003 manual
1004 | 1004 manufactur
1005 | 1005 map
1006 | 1006 march
1007 | 1007 margin
1008 | 1008 mark
1009 | 1009 market
1010 | 1010 marshal
1011 | 1011 mass
1012 | 1012 master
1013 | 1013 match
1014 | 1014 materi
1015 | 1015 matter
1016 | 1016 matthia
1017 | 1017 mayb
1018 | 1018 me
1019 | 1019 mean
1020 | 1020 measur
1021 | 1021 mechan
1022 | 1022 media
1023 | 1023 medic
1024 | 1024 meet
1025 | 1025 member
1026 | 1026 membership
1027 | 1027 memori
1028 | 1028 men
1029 | 1029 mention
1030 | 1030 menu
1031 | 1031 merchant
1032 | 1032 messag
1033 | 1033 method
1034 | 1034 mh
1035 | 1035 michael
1036 | 1036 microsoft
1037 | 1037 middl
1038 | 1038 might
1039 | 1039 mike
1040 | 1040 mile
1041 | 1041 militari
1042 | 1042 million
1043 | 1043 mime
1044 | 1044 mind
1045 | 1045 mine
1046 | 1046 mini
1047 | 1047 minimum
1048 | 1048 minut
1049 | 1049 miss
1050 | 1050 mistak
1051 | 1051 mobil
1052 | 1052 mode
1053 | 1053 model
1054 | 1054 modem
1055 | 1055 modifi
1056 | 1056 modul
1057 | 1057 moment
1058 | 1058 mon
1059 | 1059 mondai
1060 | 1060 monei
1061 | 1061 monitor
1062 | 1062 month
1063 | 1063 monthli
1064 | 1064 more
1065 | 1065 morn
1066 | 1066 mortgag
1067 | 1067 most
1068 | 1068 mostli
1069 | 1069 mother
1070 | 1070 motiv
1071 | 1071 move
1072 | 1072 movi
1073 | 1073 mpnumber
1074 | 1074 mr
1075 | 1075 ms
1076 | 1076 msg
1077 | 1077 much
1078 | 1078 multi
1079 | 1079 multipart
1080 | 1080 multipl
1081 | 1081 murphi
1082 | 1082 music
1083 | 1083 must
1084 | 1084 my
1085 | 1085 myself
1086 | 1086 name
1087 | 1087 nation
1088 | 1088 natur
1089 | 1089 nbsp
1090 | 1090 near
1091 | 1091 nearli
1092 | 1092 necessari
1093 | 1093 need
1094 | 1094 neg
1095 | 1095 net
1096 | 1096 netscap
1097 | 1097 network
1098 | 1098 never
1099 | 1099 new
1100 | 1100 newslett
1101 | 1101 next
1102 | 1102 nextpart
1103 | 1103 nice
1104 | 1104 nigeria
1105 | 1105 night
1106 | 1106 no
1107 | 1107 nobodi
1108 | 1108 non
1109 | 1109 none
1110 | 1110 nor
1111 | 1111 normal
1112 | 1112 north
1113 | 1113 not
1114 | 1114 note
1115 | 1115 noth
1116 | 1116 notic
1117 | 1117 now
1118 | 1118 nt
1119 | 1119 null
1120 | 1120 number
1121 | 1121 numbera
1122 | 1122 numberam
1123 | 1123 numberanumb
1124 | 1124 numberb
1125 | 1125 numberbit
1126 | 1126 numberc
1127 | 1127 numbercb
1128 | 1128 numbercbr
1129 | 1129 numbercfont
1130 | 1130 numbercli
1131 | 1131 numbercnumb
1132 | 1132 numbercp
1133 | 1133 numberctd
1134 | 1134 numberd
1135 | 1135 numberdari
1136 | 1136 numberdnumb
1137 | 1137 numberenumb
1138 | 1138 numberf
1139 | 1139 numberfb
1140 | 1140 numberff
1141 | 1141 numberffont
1142 | 1142 numberfp
1143 | 1143 numberftd
1144 | 1144 numberk
1145 | 1145 numberm
1146 | 1146 numbermb
1147 | 1147 numberp
1148 | 1148 numberpd
1149 | 1149 numberpm
1150 | 1150 numberpx
1151 | 1151 numberst
1152 | 1152 numberth
1153 | 1153 numbertnumb
1154 | 1154 numberx
1155 | 1155 object
1156 | 1156 oblig
1157 | 1157 obtain
1158 | 1158 obvious
1159 | 1159 occur
1160 | 1160 oct
1161 | 1161 octob
1162 | 1162 of
1163 | 1163 off
1164 | 1164 offer
1165 | 1165 offic
1166 | 1166 offici
1167 | 1167 often
1168 | 1168 oh
1169 | 1169 ok
1170 | 1170 old
1171 | 1171 on
1172 | 1172 onc
1173 | 1173 onli
1174 | 1174 onlin
1175 | 1175 open
1176 | 1176 oper
1177 | 1177 opinion
1178 | 1178 opportun
1179 | 1179 opt
1180 | 1180 optim
1181 | 1181 option
1182 | 1182 or
1183 | 1183 order
1184 | 1184 org
1185 | 1185 organ
1186 | 1186 origin
1187 | 1187 os
1188 | 1188 osdn
1189 | 1189 other
1190 | 1190 otherwis
1191 | 1191 our
1192 | 1192 out
1193 | 1193 outlook
1194 | 1194 output
1195 | 1195 outsid
1196 | 1196 over
1197 | 1197 own
1198 | 1198 owner
1199 | 1199 oz
1200 | 1200 pacif
1201 | 1201 pack
1202 | 1202 packag
1203 | 1203 page
1204 | 1204 pai
1205 | 1205 paid
1206 | 1206 pain
1207 | 1207 palm
1208 | 1208 panel
1209 | 1209 paper
1210 | 1210 paragraph
1211 | 1211 parent
1212 | 1212 part
1213 | 1213 parti
1214 | 1214 particip
1215 | 1215 particular
1216 | 1216 particularli
1217 | 1217 partit
1218 | 1218 partner
1219 | 1219 pass
1220 | 1220 password
1221 | 1221 past
1222 | 1222 patch
1223 | 1223 patent
1224 | 1224 path
1225 | 1225 pattern
1226 | 1226 paul
1227 | 1227 payment
1228 | 1228 pc
1229 | 1229 peac
1230 | 1230 peopl
1231 | 1231 per
1232 | 1232 percent
1233 | 1233 percentag
1234 | 1234 perfect
1235 | 1235 perfectli
1236 | 1236 perform
1237 | 1237 perhap
1238 | 1238 period
1239 | 1239 perl
1240 | 1240 perman
1241 | 1241 permiss
1242 | 1242 person
1243 | 1243 pgp
1244 | 1244 phone
1245 | 1245 photo
1246 | 1246 php
1247 | 1247 phrase
1248 | 1248 physic
1249 | 1249 pick
1250 | 1250 pictur
1251 | 1251 piec
1252 | 1252 piiiiiiii
1253 | 1253 pipe
1254 | 1254 pjnumber
1255 | 1255 place
1256 | 1256 plai
1257 | 1257 plain
1258 | 1258 plan
1259 | 1259 planet
1260 | 1260 plant
1261 | 1261 planta
1262 | 1262 platform
1263 | 1263 player
1264 | 1264 pleas
1265 | 1265 plu
1266 | 1266 plug
1267 | 1267 pm
1268 | 1268 pocket
1269 | 1269 point
1270 | 1270 polic
1271 | 1271 polici
1272 | 1272 polit
1273 | 1273 poor
1274 | 1274 pop
1275 | 1275 popul
1276 | 1276 popular
1277 | 1277 port
1278 | 1278 posit
1279 | 1279 possibl
1280 | 1280 post
1281 | 1281 potenti
1282 | 1282 pound
1283 | 1283 powel
1284 | 1284 power
1285 | 1285 powershot
1286 | 1286 practic
1287 | 1287 pre
1288 | 1288 predict
1289 | 1289 prefer
1290 | 1290 premium
1291 | 1291 prepar
1292 | 1292 present
1293 | 1293 presid
1294 | 1294 press
1295 | 1295 pretti
1296 | 1296 prevent
1297 | 1297 previou
1298 | 1298 previous
1299 | 1299 price
1300 | 1300 principl
1301 | 1301 print
1302 | 1302 printabl
1303 | 1303 printer
1304 | 1304 privaci
1305 | 1305 privat
1306 | 1306 prize
1307 | 1307 pro
1308 | 1308 probabl
1309 | 1309 problem
1310 | 1310 procedur
1311 | 1311 process
1312 | 1312 processor
1313 | 1313 procmail
1314 | 1314 produc
1315 | 1315 product
1316 | 1316 profession
1317 | 1317 profil
1318 | 1318 profit
1319 | 1319 program
1320 | 1320 programm
1321 | 1321 progress
1322 | 1322 project
1323 | 1323 promis
1324 | 1324 promot
1325 | 1325 prompt
1326 | 1326 properti
1327 | 1327 propos
1328 | 1328 proprietari
1329 | 1329 prospect
1330 | 1330 protect
1331 | 1331 protocol
1332 | 1332 prove
1333 | 1333 proven
1334 | 1334 provid
1335 | 1335 proxi
1336 | 1336 pub
1337 | 1337 public
1338 | 1338 publish
1339 | 1339 pudg
1340 | 1340 pull
1341 | 1341 purchas
1342 | 1342 purpos
1343 | 1343 put
1344 | 1344 python
1345 | 1345 qnumber
1346 | 1346 qualifi
1347 | 1347 qualiti
1348 | 1348 quarter
1349 | 1349 question
1350 | 1350 quick
1351 | 1351 quickli
1352 | 1352 quit
1353 | 1353 quot
1354 | 1354 radio
1355 | 1355 ragga
1356 | 1356 rais
1357 | 1357 random
1358 | 1358 rang
1359 | 1359 rate
1360 | 1360 rather
1361 | 1361 ratio
1362 | 1362 razor
1363 | 1363 razornumb
1364 | 1364 re
1365 | 1365 reach
1366 | 1366 read
1367 | 1367 reader
1368 | 1368 readi
1369 | 1369 real
1370 | 1370 realiz
1371 | 1371 realli
1372 | 1372 reason
1373 | 1373 receiv
1374 | 1374 recent
1375 | 1375 recipi
1376 | 1376 recommend
1377 | 1377 record
1378 | 1378 red
1379 | 1379 redhat
1380 | 1380 reduc
1381 | 1381 refer
1382 | 1382 refin
1383 | 1383 reg
1384 | 1384 regard
1385 | 1385 region
1386 | 1386 regist
1387 | 1387 regul
1388 | 1388 regular
1389 | 1389 rel
1390 | 1390 relat
1391 | 1391 relationship
1392 | 1392 releas
1393 | 1393 relev
1394 | 1394 reliabl
1395 | 1395 remain
1396 | 1396 rememb
1397 | 1397 remot
1398 | 1398 remov
1399 | 1399 replac
1400 | 1400 repli
1401 | 1401 report
1402 | 1402 repositori
1403 | 1403 repres
1404 | 1404 republ
1405 | 1405 request
1406 | 1406 requir
1407 | 1407 research
1408 | 1408 reserv
1409 | 1409 resid
1410 | 1410 resourc
1411 | 1411 respect
1412 | 1412 respond
1413 | 1413 respons
1414 | 1414 rest
1415 | 1415 result
1416 | 1416 retail
1417 | 1417 return
1418 | 1418 reveal
1419 | 1419 revenu
1420 | 1420 revers
1421 | 1421 review
1422 | 1422 revok
1423 | 1423 rh
1424 | 1424 rich
1425 | 1425 right
1426 | 1426 risk
1427 | 1427 road
1428 | 1428 robert
1429 | 1429 rock
1430 | 1430 role
1431 | 1431 roll
1432 | 1432 rom
1433 | 1433 roman
1434 | 1434 room
1435 | 1435 root
1436 | 1436 round
1437 | 1437 rpm
1438 | 1438 rss
1439 | 1439 rule
1440 | 1440 run
1441 | 1441 sa
1442 | 1442 safe
1443 | 1443 sai
1444 | 1444 said
1445 | 1445 sale
1446 | 1446 same
1447 | 1447 sampl
1448 | 1448 san
1449 | 1449 saou
1450 | 1450 sat
1451 | 1451 satellit
1452 | 1452 save
1453 | 1453 saw
1454 | 1454 scan
1455 | 1455 schedul
1456 | 1456 school
1457 | 1457 scienc
1458 | 1458 score
1459 | 1459 screen
1460 | 1460 script
1461 | 1461 se
1462 | 1462 search
1463 | 1463 season
1464 | 1464 second
1465 | 1465 secret
1466 | 1466 section
1467 | 1467 secur
1468 | 1468 see
1469 | 1469 seed
1470 | 1470 seek
1471 | 1471 seem
1472 | 1472 seen
1473 | 1473 select
1474 | 1474 self
1475 | 1475 sell
1476 | 1476 seminar
1477 | 1477 send
1478 | 1478 sender
1479 | 1479 sendmail
1480 | 1480 senior
1481 | 1481 sens
1482 | 1482 sensit
1483 | 1483 sent
1484 | 1484 sep
1485 | 1485 separ
1486 | 1486 septemb
1487 | 1487 sequenc
1488 | 1488 seri
1489 | 1489 serif
1490 | 1490 seriou
1491 | 1491 serv
1492 | 1492 server
1493 | 1493 servic
1494 | 1494 set
1495 | 1495 setup
1496 | 1496 seven
1497 | 1497 seventh
1498 | 1498 sever
1499 | 1499 sex
1500 | 1500 sexual
1501 | 1501 sf
1502 | 1502 shape
1503 | 1503 share
1504 | 1504 she
1505 | 1505 shell
1506 | 1506 ship
1507 | 1507 shop
1508 | 1508 short
1509 | 1509 shot
1510 | 1510 should
1511 | 1511 show
1512 | 1512 side
1513 | 1513 sign
1514 | 1514 signatur
1515 | 1515 signific
1516 | 1516 similar
1517 | 1517 simpl
1518 | 1518 simpli
1519 | 1519 sinc
1520 | 1520 sincer
1521 | 1521 singl
1522 | 1522 sit
1523 | 1523 site
1524 | 1524 situat
1525 | 1525 six
1526 | 1526 size
1527 | 1527 skeptic
1528 | 1528 skill
1529 | 1529 skin
1530 | 1530 skip
1531 | 1531 sleep
1532 | 1532 slow
1533 | 1533 small
1534 | 1534 smart
1535 | 1535 smoke
1536 | 1536 smtp
1537 | 1537 snumber
1538 | 1538 so
1539 | 1539 social
1540 | 1540 societi
1541 | 1541 softwar
1542 | 1542 sold
1543 | 1543 solut
1544 | 1544 solv
1545 | 1545 some
1546 | 1546 someon
1547 | 1547 someth
1548 | 1548 sometim
1549 | 1549 son
1550 | 1550 song
1551 | 1551 soni
1552 | 1552 soon
1553 | 1553 sorri
1554 | 1554 sort
1555 | 1555 sound
1556 | 1556 sourc
1557 | 1557 south
1558 | 1558 space
1559 | 1559 spain
1560 | 1560 spam
1561 | 1561 spamassassin
1562 | 1562 spamd
1563 | 1563 spammer
1564 | 1564 speak
1565 | 1565 spec
1566 | 1566 special
1567 | 1567 specif
1568 | 1568 specifi
1569 | 1569 speech
1570 | 1570 speed
1571 | 1571 spend
1572 | 1572 sponsor
1573 | 1573 sport
1574 | 1574 spot
1575 | 1575 src
1576 | 1576 ssh
1577 | 1577 st
1578 | 1578 stabl
1579 | 1579 staff
1580 | 1580 stai
1581 | 1581 stand
1582 | 1582 standard
1583 | 1583 star
1584 | 1584 start
1585 | 1585 state
1586 | 1586 statement
1587 | 1587 statu
1588 | 1588 step
1589 | 1589 steve
1590 | 1590 still
1591 | 1591 stock
1592 | 1592 stop
1593 | 1593 storag
1594 | 1594 store
1595 | 1595 stori
1596 | 1596 strategi
1597 | 1597 stream
1598 | 1598 street
1599 | 1599 string
1600 | 1600 strip
1601 | 1601 strong
1602 | 1602 structur
1603 | 1603 studi
1604 | 1604 stuff
1605 | 1605 stupid
1606 | 1606 style
1607 | 1607 subject
1608 | 1608 submit
1609 | 1609 subscrib
1610 | 1610 subscript
1611 | 1611 substanti
1612 | 1612 success
1613 | 1613 such
1614 | 1614 suffer
1615 | 1615 suggest
1616 | 1616 suit
1617 | 1617 sum
1618 | 1618 summari
1619 | 1619 summer
1620 | 1620 sun
1621 | 1621 super
1622 | 1622 suppli
1623 | 1623 support
1624 | 1624 suppos
1625 | 1625 sure
1626 | 1626 surpris
1627 | 1627 suse
1628 | 1628 suspect
1629 | 1629 sweet
1630 | 1630 switch
1631 | 1631 system
1632 | 1632 tab
1633 | 1633 tabl
1634 | 1634 tablet
1635 | 1635 tag
1636 | 1636 take
1637 | 1637 taken
1638 | 1638 talk
1639 | 1639 tape
1640 | 1640 target
1641 | 1641 task
1642 | 1642 tax
1643 | 1643 teach
1644 | 1644 team
1645 | 1645 tech
1646 | 1646 technic
1647 | 1647 techniqu
1648 | 1648 technolog
1649 | 1649 tel
1650 | 1650 telecom
1651 | 1651 telephon
1652 | 1652 tell
1653 | 1653 temperatur
1654 | 1654 templ
1655 | 1655 ten
1656 | 1656 term
1657 | 1657 termin
1658 | 1658 terror
1659 | 1659 terrorist
1660 | 1660 test
1661 | 1661 texa
1662 | 1662 text
1663 | 1663 than
1664 | 1664 thank
1665 | 1665 that
1666 | 1666 the
1667 | 1667 thei
1668 | 1668 their
1669 | 1669 them
1670 | 1670 themselv
1671 | 1671 then
1672 | 1672 theori
1673 | 1673 there
1674 | 1674 therefor
1675 | 1675 these
1676 | 1676 thi
1677 | 1677 thing
1678 | 1678 think
1679 | 1679 thinkgeek
1680 | 1680 third
1681 | 1681 those
1682 | 1682 though
1683 | 1683 thought
1684 | 1684 thousand
1685 | 1685 thread
1686 | 1686 threat
1687 | 1687 three
1688 | 1688 through
1689 | 1689 thu
1690 | 1690 thursdai
1691 | 1691 ti
1692 | 1692 ticket
1693 | 1693 tim
1694 | 1694 time
1695 | 1695 tip
1696 | 1696 tire
1697 | 1697 titl
1698 | 1698 tm
1699 | 1699 to
1700 | 1700 todai
1701 | 1701 togeth
1702 | 1702 token
1703 | 1703 told
1704 | 1704 toll
1705 | 1705 tom
1706 | 1706 toner
1707 | 1707 toni
1708 | 1708 too
1709 | 1709 took
1710 | 1710 tool
1711 | 1711 top
1712 | 1712 topic
1713 | 1713 total
1714 | 1714 touch
1715 | 1715 toward
1716 | 1716 track
1717 | 1717 trade
1718 | 1718 tradit
1719 | 1719 traffic
1720 | 1720 train
1721 | 1721 transact
1722 | 1722 transfer
1723 | 1723 travel
1724 | 1724 treat
1725 | 1725 tree
1726 | 1726 tri
1727 | 1727 trial
1728 | 1728 trick
1729 | 1729 trip
1730 | 1730 troubl
1731 | 1731 true
1732 | 1732 truli
1733 | 1733 trust
1734 | 1734 truth
1735 | 1735 try
1736 | 1736 tue
1737 | 1737 tuesdai
1738 | 1738 turn
1739 | 1739 tv
1740 | 1740 two
1741 | 1741 type
1742 | 1742 uk
1743 | 1743 ultim
1744 | 1744 un
1745 | 1745 under
1746 | 1746 understand
1747 | 1747 unfortun
1748 | 1748 uniqu
1749 | 1749 unison
1750 | 1750 unit
1751 | 1751 univers
1752 | 1752 unix
1753 | 1753 unless
1754 | 1754 unlik
1755 | 1755 unlimit
1756 | 1756 unseen
1757 | 1757 unsolicit
1758 | 1758 unsubscrib
1759 | 1759 until
1760 | 1760 up
1761 | 1761 updat
1762 | 1762 upgrad
1763 | 1763 upon
1764 | 1764 urgent
1765 | 1765 url
1766 | 1766 us
1767 | 1767 usa
1768 | 1768 usag
1769 | 1769 usb
1770 | 1770 usd
1771 | 1771 usdollarnumb
1772 | 1772 useless
1773 | 1773 user
1774 | 1774 usr
1775 | 1775 usual
1776 | 1776 util
1777 | 1777 vacat
1778 | 1778 valid
1779 | 1779 valu
1780 | 1780 valuabl
1781 | 1781 var
1782 | 1782 variabl
1783 | 1783 varieti
1784 | 1784 variou
1785 | 1785 ve
1786 | 1786 vendor
1787 | 1787 ventur
1788 | 1788 veri
1789 | 1789 verifi
1790 | 1790 version
1791 | 1791 via
1792 | 1792 video
1793 | 1793 view
1794 | 1794 virtual
1795 | 1795 visa
1796 | 1796 visit
1797 | 1797 visual
1798 | 1798 vnumber
1799 | 1799 voic
1800 | 1800 vote
1801 | 1801 vs
1802 | 1802 vulner
1803 | 1803 wa
1804 | 1804 wai
1805 | 1805 wait
1806 | 1806 wake
1807 | 1807 walk
1808 | 1808 wall
1809 | 1809 want
1810 | 1810 war
1811 | 1811 warm
1812 | 1812 warn
1813 | 1813 warranti
1814 | 1814 washington
1815 | 1815 wasn
1816 | 1816 wast
1817 | 1817 watch
1818 | 1818 water
1819 | 1819 we
1820 | 1820 wealth
1821 | 1821 weapon
1822 | 1822 web
1823 | 1823 weblog
1824 | 1824 websit
1825 | 1825 wed
1826 | 1826 wednesdai
1827 | 1827 week
1828 | 1828 weekli
1829 | 1829 weight
1830 | 1830 welcom
1831 | 1831 well
1832 | 1832 went
1833 | 1833 were
1834 | 1834 west
1835 | 1835 what
1836 | 1836 whatev
1837 | 1837 when
1838 | 1838 where
1839 | 1839 whether
1840 | 1840 which
1841 | 1841 while
1842 | 1842 white
1843 | 1843 whitelist
1844 | 1844 who
1845 | 1845 whole
1846 | 1846 whose
1847 | 1847 why
1848 | 1848 wi
1849 | 1849 wide
1850 | 1850 width
1851 | 1851 wife
1852 | 1852 will
1853 | 1853 william
1854 | 1854 win
1855 | 1855 window
1856 | 1856 wing
1857 | 1857 winner
1858 | 1858 wireless
1859 | 1859 wish
1860 | 1860 with
1861 | 1861 within
1862 | 1862 without
1863 | 1863 wnumberp
1864 | 1864 woman
1865 | 1865 women
1866 | 1866 won
1867 | 1867 wonder
1868 | 1868 word
1869 | 1869 work
1870 | 1870 worker
1871 | 1871 world
1872 | 1872 worldwid
1873 | 1873 worri
1874 | 1874 worst
1875 | 1875 worth
1876 | 1876 would
1877 | 1877 wouldn
1878 | 1878 write
1879 | 1879 written
1880 | 1880 wrong
1881 | 1881 wrote
1882 | 1882 www
1883 | 1883 ximian
1884 | 1884 xml
1885 | 1885 xp
1886 | 1886 yahoo
1887 | 1887 ye
1888 | 1888 yeah
1889 | 1889 year
1890 | 1890 yesterdai
1891 | 1891 yet
1892 | 1892 york
1893 | 1893 you
1894 | 1894 young
1895 | 1895 your
1896 | 1896 yourself
1897 | 1897 zdnet
1898 | 1898 zero
1899 | 1899 zip
1900 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/.gitignore:
--------------------------------------------------------------------------------
1 | # Default ignored files
2 | /workspace.xml
3 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/ex7.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/inspectionProfiles/profiles_settings.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/misc.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/homework/ex7/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/homework/ex7/Kmeans.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/Kmeans.py
--------------------------------------------------------------------------------
/homework/ex7/bird_small.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/bird_small.mat
--------------------------------------------------------------------------------
/homework/ex7/bird_small.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/bird_small.png
--------------------------------------------------------------------------------
/homework/ex7/ex7.pdf:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/ex7.pdf
--------------------------------------------------------------------------------
/homework/ex7/ex7data1.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/ex7data1.mat
--------------------------------------------------------------------------------
/homework/ex7/ex7data2.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/ex7data2.mat
--------------------------------------------------------------------------------
/homework/ex7/ex7faces.mat:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Currycomb/Machine-Learning-WuEnda/e89bd4d9c650aa01ad61cade90fc1b662b0f3456/homework/ex7/ex7faces.mat
--------------------------------------------------------------------------------
/算法.txt:
--------------------------------------------------------------------------------
1 | 1.梯度下降算法
2 | 2.正规方程算法
--------------------------------------------------------------------------------