├── test.seq ├── test.hmm ├── README.md ├── test_hmm.py └── hmm.py /test.seq: -------------------------------------------------------------------------------- 1 | T= 10 2 | 1, 1, 1, 1, 2, 1, 2, 2, 2, 2 3 | -------------------------------------------------------------------------------- /test.hmm: -------------------------------------------------------------------------------- 1 | M= 2 2 | N= 3 3 | A: 4 | 0.333, 0.333, 0.333 5 | 0.333, 0.333, 0.333 6 | 0.333, 0.333, 0.333 7 | B: 8 | 0.5, 0.5 9 | 0.75, 0.25 10 | 0.25, 0.75 11 | pi: 12 | 0.333, 0.333, 0.333 13 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | #简介# 2 | 3 | 这是我第一个比较完整的python程序，把``UMDHMM``的大部分功能用python重写了，没有经过完整的测试。 4 | 数据输入格式与UMDHMM基本相同，都是在后缀为``.hmm``的文件中包含初始向量``pi``、转移矩阵``A``、混淆矩阵 5 | ``B``，在后缀为``.seq``的文件中包含观察序列及其个数``T``。稍微不同的是，每个数据的后面比UMDHMM的输入数据 6 | 多一个逗号，你可以直接运行test_hmm.py观察结果。 7 | 8 | [UMDHMM][1]是一款轻量级的HMM（Hidden Markov Model）C语言实现，更详细的说明可以参考 9 | [ 《HMM学习最佳范例五：前向算法4》 ][2]。 10 | 11 | 本程序需要[numpy][3]库的支持。 12 | 13 | 14 | ##待做## 15 | 16 | 1. 添加注释及docstring 17 | 2. 测试viterbi算法 18 | 3. 代码优化 19 | 20 | [1]: http://www.kanungo.com/software/software.html 21 | [2]: http://www.52nlp.cn/hmm-learn-best-practices-five-forward-algorithm-4 22 | [3]: http://www.numpy.org/ 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /test_hmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from hmm import HMM, read_hmm, read_sequence 4 | 5 | 6 | hmmfile = "test.hmm" 7 | seqfile = "test.seq" 8 | 9 | M, N, pi, A, B = read_hmm(hmmfile) 10 | T, obs = read_sequence(seqfile) 11 | 12 | hmm_object = HMM(pi, A, B) 13 | 14 | #test forward algorithm 15 | prob, alpha = hmm_object.forward(obs) 16 | print "forward probability is %f" % np.log(prob) 17 | prob, alpha, scale = hmm_object.forward_with_scale(obs) 18 | print "forward probability with scale is %f" % prob 19 | 20 | # test backward algorithm 21 | prob, beta = hmm_object.backward(obs) 22 | print "backward probability is %f" % prob 23 | beta = hmm_object.backward_with_scale(obs, scale) 24 | 25 | # test baum-welch algorithm 26 | logprobinit, logprobfinal = hmm_object.baum_welch(obs) 27 | print "------------------------------------------------" 28 | print "estimated parameters are: " 29 | print "pi is:" 30 | print hmm_object.pi 31 | print "A is:" 32 | print hmm_object.A 33 | print "B is:" 34 | print hmm_object.B 35 | print "------------------------------------------------" 36 | print "initial log probability is:" 37 | print logprobinit 38 | print "final log probability is:" 39 | print logprobfinal -------------------------------------------------------------------------------- /hmm.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | DELTA = 0.001 4 | 5 | class HMM: 6 | 7 | 8 | def __init__(self, pi, A, B): 9 | self.pi = pi 10 | self.A = A 11 | self.B = B 12 | self.M = B.shape[1] 13 | self.N = A.shape[0] 14 | 15 | def forward(self,obs): 16 | T = len(obs) 17 | N = self.N 18 | 19 | alpha = np.zeros([N,T]) 20 | alpha[:,0] = self.pi[:] * self.B[:,obs[0]-1] 21 | 22 | for t in xrange(1,T): 23 | for n in xrange(0,N): 24 | alpha[n,t] = np.sum(alpha[:,t-1] * self.A[:,n]) * self.B[n,obs[t]-1] 25 | 26 | prob = np.sum(alpha[:,T-1]) 27 | return prob, alpha 28 | 29 | def forward_with_scale(self, obs): 30 | """see scaling chapter in "A tutorial on hidden Markov models and 31 | selected applications in speech recognition." 32 | """ 33 | T = len(obs) 34 | N = self.N 35 | alpha = np.zeros([N,T]) 36 | scale = np.zeros(T) 37 | 38 | alpha[:,0] = self.pi[:] * self.B[:,obs[0]-1] 39 | scale[0] = np.sum(alpha[:,0]) 40 | alpha[:,0] /= scale[0] 41 | 42 | for t in xrange(1,T): 43 | for n in xrange(0,N): 44 | alpha[n,t] = np.sum(alpha[:,t-1] * self.A[:,n]) * self.B[n,obs[t]-1] 45 | scale[t] = np.sum(alpha[:,t]) 46 | alpha[:,t] /= scale[t] 47 | 48 | logprob = np.sum(np.log(scale[:])) 49 | return logprob, alpha, scale 50 | 51 | def backward(self, obs): 52 | T = len(obs) 53 | N = self.N 54 | beta = np.zeros([N,T]) 55 | 56 | beta[:,T-1] = 1 57 | for t in reversed(xrange(0,T-1)): 58 | for n in xrange(0,N): 59 | beta[n,t] = np.sum(self.B[:,obs[t+1]-1] * self.A[n,:] * beta[:,t+1]) 60 | 61 | prob = np.sum(beta[:,0]) 62 | return prob, beta 63 | 64 | def backward_with_scale(self, obs, scale): 65 | T = len(obs) 66 | N = self.N 67 | beta = np.zeros([N,T]) 68 | 69 | beta[:,T-1] = 1 / scale[T-1] 70 | for t in reversed(xrange(0,T-1)): 71 | for n in xrange(0,N): 72 | beta[n,t] = np.sum(self.B[:,obs[t+1]-1] * self.A[n,:] * beta[:,t+1]) 73 | beta[n,t] /= scale[t] 74 | 75 | return beta 76 | 77 | def viterbi(self, obs): 78 | T = len(obs) 79 | N = self.N 80 | psi = np.zeros([N,T]) # reverse pointer 81 | delta = np.zeros([N,T]) 82 | q = np.zeros(T) 83 | temp = np.zeros(N) 84 | 85 | delta[:,0] = self.pi[:] * self.B[:,obs[0]-1] 86 | 87 | for t in xrange(1,T): 88 | for n in xrange(0,N): 89 | temp = delta[:,t-1] * self.A[:,n] 90 | max_ind = argmax(temp) 91 | psi[n,t] = max_ind 92 | delta[n,t] = self.B[n,obs[t]-1] * temp[max_ind] 93 | 94 | max_ind = argmax(delta[:,T-1]) 95 | q[T-1] = max_ind 96 | prob = delta[:,T-1][max_ind] 97 | 98 | for t in reversed(xrange(1,T-1)): 99 | q[t] = psi[q[t+1],t+1] 100 | 101 | return prob, q, delta 102 | 103 | def viterbi_log(self, obs): 104 | 105 | T = len(obs) 106 | N = self.N 107 | psi = np.zeros([N,T]) 108 | delta = np.zeros([N,T]) 109 | pi = np.zeros(self.pi.shape) 110 | A = np.zeros(self.A.shape) 111 | biot = np.zeros([N,T]) 112 | 113 | pi = np.log(self.pi) 114 | A = np.log(self.A) 115 | biot = np.log(self.B[:,obs[:]-1]) 116 | 117 | delta[:,0] = pi[:] + biot[:,0] 118 | 119 | for t in xrange(1,T): 120 | for n in xrange(0,N): 121 | temp = delta[:,t-1] + self.A[:,n] 122 | max_ind = argmax(temp) 123 | psi[n,t] = max_ind 124 | delta[n,t] = temp[max_ind] + biot[n,t] 125 | 126 | max_ind = argmax(delta[:,T-1]) 127 | q[T-1] = max_ind 128 | logprob = delta[max_ind,T-1] 129 | 130 | for t in reversed(xrange(1,T-1)): 131 | q[t] = psi[q[t+1],t+1] 132 | 133 | return logprob, q, delta 134 | 135 | def baum_welch(self, obs): 136 | T = len(obs) 137 | M = self.M 138 | N = self.N 139 | alpha = np.zeros([N,T]) 140 | beta = np.zeros([N,T]) 141 | scale = np.zeros(T) 142 | gamma = np.zeros([N,T]) 143 | xi = np.zeros([N,N,T-1]) 144 | 145 | # caculate initial parameters 146 | logprobprev, alpha, scale = self.forward_with_scale(obs) 147 | beta = self.backward_with_scale(obs, scale) 148 | gamma = self.compute_gamma(alpha, beta) 149 | xi = self.compute_xi(obs, alpha, beta) 150 | logprobinit = logprobprev 151 | 152 | # start interative 153 | while True: 154 | # E-step 155 | self.pi = 0.001 + 0.999*gamma[:,0] 156 | for i in xrange(N): 157 | denominator = np.sum(gamma[i,0:T-1]) 158 | for j in xrange(N): 159 | numerator = np.sum(xi[i,j,0:T-1]) 160 | self.A[i,j] = numerator / denominator 161 | 162 | self.A = 0.001 + 0.999*self.A 163 | for j in xrange(0,N): 164 | denominator = np.sum(gamma[j,:]) 165 | for k in xrange(0,M): 166 | numerator = 0.0 167 | for t in xrange(0,T): 168 | if obs[t]-1 == k: 169 | numerator += gamma[j,t] 170 | self.B[j,k] = numerator / denominator 171 | self.B = 0.001 + 0.999*self.B 172 | 173 | # M-step 174 | logprobcur, alpha, scale = self.forward_with_scale(obs) 175 | beta = self.backward_with_scale(obs, scale) 176 | gamma = self.compute_gamma(alpha, beta) 177 | xi = self.compute_xi(obs, alpha, beta) 178 | 179 | delta = logprobcur - logprobprev 180 | logprobprev = logprobcur 181 | # print "delta is ", delta 182 | if delta <= DELTA: 183 | break 184 | 185 | logprobfinal = logprobcur 186 | return logprobinit, logprobfinal 187 | 188 | def compute_gamma(self, alpha, beta): 189 | gamma = np.zeros(alpha.shape) 190 | gamma = alpha[:,:] * beta[:,:] 191 | gamma = gamma / np.sum(gamma,0) 192 | return gamma 193 | 194 | def compute_xi(self, obs, alpha, beta): 195 | T = len(obs) 196 | N = self.N 197 | xi = np.zeros((N, N, T-1)) 198 | 199 | for t in xrange(0,T-1): 200 | for i in xrange(0,N): 201 | for j in xrange(0,N): 202 | xi[i,j,t] = alpha[i,t] * self.A[i,j] * \ 203 | self.B[j,obs[t+1]-1] * beta[j,t+1] 204 | xi[:,:,t] /= np.sum(np.sum(xi[:,:,t],1),0) 205 | return xi 206 | 207 | def read_hmm(hmmfile): 208 | fhmm = open(hmmfile,'r') 209 | 210 | M = int(fhmm.readline().split(' ')[1]) 211 | N = int(fhmm.readline().split(' ')[1]) 212 | 213 | A = np.array([]) 214 | fhmm.readline() 215 | for i in xrange(N): 216 | line = fhmm.readline() 217 | if i == 0: 218 | A = np.array(map(float,line.split(','))) 219 | else: 220 | A = np.vstack((A,map(float,line.split(',')))) 221 | 222 | B = np.array([]) 223 | fhmm.readline() 224 | for i in xrange(N): 225 | line = fhmm.readline() 226 | if i == 0: 227 | B = np.array(map(float,line.split(','))) 228 | else: 229 | B = np.vstack((B,map(float,line.split(',')))) 230 | 231 | fhmm.readline() 232 | line = fhmm.readline() 233 | pi = np.array(map(float,line.split(','))) 234 | 235 | fhmm.close() 236 | return M, N, pi, A, B 237 | 238 | def read_sequence(seqfile): 239 | fseq = open(seqfile,'r') 240 | 241 | T = int(fseq.readline().split(' ')[1]) 242 | line = fseq.readline() 243 | obs = np.array(map(int,line.split(','))) 244 | 245 | fseq.close() 246 | return T, obs 247 | 248 | --------------------------------------------------------------------------------