├── CDSSM_dense ├── Normalizer.lua ├── th_utils.lua ├── DSSM_Predict.lua ├── Calculate_Alpha.lua ├── ComputelogPD.lua ├── Batch.lua ├── PairInputStream.lua ├── SequenceInputStream.lua ├── utils.lua ├── train.lua ├── PreProcess.lua ├── BatchSample.lua ├── DSSM_MMI_Criterion.lua ├── WordHash.lua ├── DSSM_Train.lua └── data │ └── l3g.txt ├── CDSSM_sparse ├── nn │ ├── test.lua │ ├── SparseLinearBatch.lua │ ├── CDSSMSparseConvolution.lua │ ├── SparseLinearBatch.c │ └── CDSSMSparseConvolution.c ├── th_utils.lua ├── Calculate_Alpha.lua ├── ComputelogPD.lua ├── Batch.lua ├── utils.lua ├── PreProcess.lua ├── sparse_train.lua ├── DSSM_MMI_Criterion.lua ├── WordHash.lua └── data │ └── l3g.txt └── readme.md /CDSSM_dense/Normalizer.lua: -------------------------------------------------------------------------------- 1 | local Normalizer = {} 2 | Normalizer.__index = Sample 3 | function Normalizer.init(featureDim) 4 | self = {} 5 | setmetatable(self, Normalizer) 6 | 7 | self.featureDim = 0 8 | 9 | return self 10 | 11 | end 12 | 13 | function Normalizer.CreateFeatureNormalize(type, featureSize) 14 | local norm = nil 15 | if type == 'None' then 16 | norm = Normalizer.init(featureSize) 17 | 18 | elseif type == 'min_max' then 19 | 20 | end 21 | 22 | return norm 23 | end 24 | 25 | return Normalizer -------------------------------------------------------------------------------- /CDSSM_sparse/nn/test.lua: -------------------------------------------------------------------------------- 1 | 2 | require 'nn' 3 | 4 | 5 | a = nn.SparseLinearBatch(2, 3) 6 | 7 | data = torch.Tensor({{1,1,1}, {1,2,1}, {2,1,1}}) 8 | out = a:forward(data) 9 | grad = torch.Tensor(2,3):fill(1) 10 | --a.weight:fill(1) 11 | print(a:backward(data, grad)) 12 | print(a.weight) 13 | print(a.bias) 14 | 15 | data = torch.Tensor({{1,1},{1,0}}) 16 | b = nn.Linear(2,3) 17 | b.weight:copy(a.weight) 18 | b.bias:copy(a.bias) 19 | out = b:forward(data) 20 | grad = torch.Tensor(2,3):fill(1) 21 | 22 | --print(b.weight) 23 | print(b:backward(data, grad)) 24 | --print(b.gradWeight) 25 | print(b.bias) 26 | print(b.weight) 27 | 28 | 29 | c = nn.CDSSMSparseConvolution(2,3,10) 30 | --c = nn.SparseLinearBatch(2, 3) 31 | data = torch.Tensor({{1,1,1,1}, {1,1,2,1},{2,1,1,1}}) 32 | 33 | --c.weight:fill(1) 34 | --c.bias:fill(0) 35 | c.weight:copy(a.weight) 36 | c.bias:copy(a.bias) 37 | 38 | print(c:forward(data)) 39 | 40 | grad = torch.Tensor(2,10,3):fill(1) 41 | 42 | print(c:backward(data, grad)) 43 | print(c.weight) 44 | print(c.bias) 45 | --print(data) -------------------------------------------------------------------------------- /CDSSM_dense/th_utils.lua: -------------------------------------------------------------------------------- 1 | -- useful functions enhance the lua or torch 2 | local th_utils = {} 3 | th_utils.__index = th_utils 4 | 5 | function th_utils.split(str, pat) 6 | 7 | local t = {} -- NOTE: use {n = 0} in Lua-5.0 8 | local fpat = "(.-)" .. pat 9 | local last_end = 1 10 | local s, e, cap = str:find(fpat, 1) 11 | while s do 12 | if s ~= 1 or cap ~= "" then 13 | table.insert(t,cap) 14 | end 15 | last_end = e+1 16 | s, e, cap = str:find(fpat, last_end) 17 | end 18 | if last_end <= #str then 19 | cap = str:sub(last_end) 20 | table.insert(t, cap) 21 | end 22 | return t 23 | end 24 | 25 | function th_utils.trim(s) 26 | return (s:gsub("^%s*(.-)%s*$", "%1")) 27 | end 28 | 29 | function th_utils.spairs(t, order) 30 | local keys = {} 31 | for k in pairs(t) do keys[#keys+1] = k end 32 | 33 | if order then 34 | table.sort(keys, function(a,b) return order(t,a,b) end) 35 | else 36 | table.sort(keys) 37 | end 38 | local sorted = {} 39 | for i = 1,#keys do 40 | sorted[i] = keys[i] 41 | end 42 | return sorted 43 | end 44 | 45 | function th_utils.IsFile(path) 46 | if io.open(path,'r')~=nil then 47 | return true 48 | else 49 | return false 50 | end 51 | end 52 | 53 | return th_utils -------------------------------------------------------------------------------- /CDSSM_sparse/th_utils.lua: -------------------------------------------------------------------------------- 1 | -- useful functions enhance the lua or torch 2 | local th_utils = {} 3 | th_utils.__index = th_utils 4 | 5 | function th_utils.split(str, pat) 6 | 7 | local t = {} -- NOTE: use {n = 0} in Lua-5.0 8 | local fpat = "(.-)" .. pat 9 | local last_end = 1 10 | local s, e, cap = str:find(fpat, 1) 11 | while s do 12 | if s ~= 1 or cap ~= "" then 13 | table.insert(t,cap) 14 | end 15 | last_end = e+1 16 | s, e, cap = str:find(fpat, last_end) 17 | end 18 | if last_end <= #str then 19 | cap = str:sub(last_end) 20 | table.insert(t, cap) 21 | end 22 | return t 23 | end 24 | 25 | function th_utils.trim(s) 26 | return (s:gsub("^%s*(.-)%s*$", "%1")) 27 | end 28 | 29 | function th_utils.spairs(t, order) 30 | local keys = {} 31 | for k in pairs(t) do keys[#keys+1] = k end 32 | 33 | if order then 34 | table.sort(keys, function(a,b) return order(t,a,b) end) 35 | else 36 | table.sort(keys) 37 | end 38 | local sorted = {} 39 | for i = 1,#keys do 40 | sorted[i] = keys[i] 41 | end 42 | return sorted 43 | end 44 | 45 | function th_utils.IsFile(path) 46 | if io.open(path,'r')~=nil then 47 | return true 48 | else 49 | return false 50 | end 51 | end 52 | 53 | return th_utils -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | ## Note: Currently, we only have torch c version batch input sparse linear and sparse covolution. 2 | 3 | ##implementation of (c)DSSM in torch 4 | 5 | * For more detail of the paper, see (http://research.microsoft.com/en-us/projects/dssm/) 6 | 7 | * contributer: Jiasen Lu 8 | 9 | ##dependencies: 10 | * torch 11 | * tds (DSSM dense) 12 | 13 | ##Data Preprocessing 14 | ###Related Functions: 15 | * Batch.lua 16 | * WordHash.lua 17 | * ComputelogPD.lua 18 | * Preprocess.lua 19 | 20 | 21 | ##Tranining 22 | 1: generate data from dataset. The data format follows the C# implementation. Each query and document in the same line, and the seperator is 'Tab'. 23 | 2: generate vocabulary for question and answers. Using WordHash.Pair2Voc(). 24 | you should get the result like this: 25 | ''' 26 | Creating Voc file form ... 27 | srcVoc contains vocabulary: 5584 28 | tgtVoc contains vocabulary: 10876 29 | ''' 30 | 3: Create Pair2Seq Feature and save to txt. Using WordHash.Pair2SeqFea() 31 | 32 | 4: Convert the seq Feature to Binay file, we give the batchsize here. (this can't be change after you train the model. for orginial data, the batch size is 1024. Using WordHash.SeqFea2Bin(), See more info under the function. 33 | 34 | ###Related functions 35 | * (Data Provider): BatchSample.lua, SequenceInputStream.lua, PairInputStream.lua 36 | * (Model): DSSM_Train, DSSM_MMI_Criterion.lua 37 | * (Training): th train.lua 38 | 39 | ##Predicting 40 | 1: generate feature file, refer PreProcess.lua for details 41 | 42 | * Preprocessing: 43 | 44 | * (Predict): th predict.lua 45 | 46 | ##To-do List 47 | 48 | * testing the cu implementation of sparse Linear. 49 | * implement the cu implementation of sparse convolution. 50 | 51 | 52 | -------------------------------------------------------------------------------- /CDSSM_dense/DSSM_Predict.lua: -------------------------------------------------------------------------------- 1 | local DSSM_Predict = {} 2 | DSSM_Predict.__index = DSSM_Predict 3 | 4 | function DSSM_Predict:updateOutput(input) 5 | 6 | local Q_input, D_input = input[1], input[2] 7 | 8 | local Q_len = Q_input:size(1) 9 | local D_len = D_input:size(1) 10 | 11 | local prob_matrix = torch.Tensor(Q_len, D_len):zero() 12 | 13 | self.input2 = D_input 14 | for i = 1, Q_len do 15 | self.input1 = Q_input:sub(i,i):expandAs(self.input2) 16 | if not self.buffer then 17 | self.buffer = self.input1.new() 18 | self.w1 = self.input1.new() 19 | self.w22 = self.input1.new() 20 | self.w = self.input1.new() 21 | self.w32 = self.input1.new() 22 | self.ones = self.input1.new() 23 | end 24 | 25 | self.buffer:cmul(self.input1, self.input2) 26 | self.w1:sum(self.buffer,2) 27 | 28 | local epsilon = 1e-12 29 | self.buffer:cmul(self.input1,self.input1) 30 | self.w22:sum(self.buffer,2):add(epsilon) 31 | self.ones:resizeAs(self.w22):fill(1) 32 | self.w22:cdiv(self.ones, self.w22) 33 | self.w:resizeAs(self.w22):copy(self.w22) 34 | 35 | self.buffer:cmul(self.input2,self.input2) 36 | self.w32:sum(self.buffer,2):add(epsilon) 37 | self.w32:cdiv(self.ones, self.w32) 38 | self.w:cmul(self.w32) 39 | self.w:sqrt() 40 | 41 | self.dist = torch.cmul(self.w1,self.w) 42 | self.dist = self.dist:select(2,1) 43 | prob_matrix:sub(i,i):copy(self.dist) 44 | end 45 | 46 | -- find the max value 47 | local _, predict = torch.max(prob_matrix,2) 48 | local correct = 0 49 | for i = 1, D_len do 50 | local predict_label = label[predict[i][1]] 51 | if predict_label == label[i] then 52 | correct = correct + 1 53 | end 54 | end 55 | print('accuracy are: ' .. correct / D_len) 56 | --print(math.sum() / D_len) 57 | return prob_matrix 58 | end 59 | 60 | 61 | return DSSM_Predict -------------------------------------------------------------------------------- /CDSSM_dense/Calculate_Alpha.lua: -------------------------------------------------------------------------------- 1 | 2 | local Calculate_Alpha = {} 3 | Calculate_Alpha.__index = Calculate_Alpha 4 | 5 | 6 | function Calculate_Alpha.cal_alpha(alpha, nTrial, batchsize, gamma) 7 | -- alpha[negtive] = exp(-gamma * (alpha[positive] - alpha[negative]) 8 | local pos_alpha = alpha:sub(1,batchsize) 9 | local neg_alpha = alpha:sub(batchsize+1, -1) 10 | 11 | local pos_alpha_replicate = pos_alpha:repeatTensor(nTrial) 12 | 13 | neg_alpha = torch.add(pos_alpha_replicate, -neg_alpha) 14 | neg_alpha = torch.exp(torch.mul(neg_alpha, -gamma)) 15 | 16 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 17 | alpha:sub(batchsize+1, -1):copy(neg_alpha) 18 | return alpha 19 | end 20 | 21 | function Calculate_Alpha.cal_alpha_sum(alpha, nTrial, batchsize, gamma, init) 22 | -- alpha[postive] = init + alpha[postive + negative] (sum of all the (pos - negtive)) 23 | local pos_alpha = alpha:sub(1,batchsize):zero() 24 | local neg_alpha = alpha:sub(batchsize+1, -1) 25 | local neg_alpha_split = neg_alpha:split(batchsize, 1) 26 | for i = 1, nTrial do 27 | pos_alpha:add(neg_alpha_split[i]) 28 | end 29 | pos_alpha:add(init) 30 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 31 | alpha:sub(1,batchsize):copy(pos_alpha) 32 | return alpha 33 | end 34 | 35 | function Calculate_Alpha.cal_alpha_norm(alpha, nTrial, batchsize, gamma) 36 | -- alpha[negative] = gamma * alpha[negative]./ alpha[positive] (normalization) 37 | local pos_alpha = alpha:sub(1,batchsize) 38 | local neg_alpha = alpha:sub(batchsize+1, -1) 39 | local positive_array = torch.range(1,batchsize):type('torch.IntTensor') 40 | positive_array = positive_array:repeatTensor(nTrial) -- replicate the nTrail times 41 | positive_array = positive_array:type('torch.LongTensor') 42 | local pos_alpha_replicate = pos_alpha:index(1,positive_array) 43 | 44 | neg_alpha = torch.cdiv(neg_alpha, pos_alpha_replicate) 45 | neg_alpha = torch.mul(neg_alpha, gamma) 46 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 47 | alpha:sub(batchsize+1, -1):copy(neg_alpha) 48 | 49 | return alpha 50 | end 51 | 52 | return Calculate_Alpha -------------------------------------------------------------------------------- /CDSSM_sparse/Calculate_Alpha.lua: -------------------------------------------------------------------------------- 1 | 2 | local Calculate_Alpha = {} 3 | Calculate_Alpha.__index = Calculate_Alpha 4 | 5 | 6 | function Calculate_Alpha.cal_alpha(alpha, nTrial, batchsize, gamma) 7 | -- alpha[negtive] = exp(-gamma * (alpha[positive] - alpha[negative]) 8 | local pos_alpha = alpha:sub(1,batchsize) 9 | local neg_alpha = alpha:sub(batchsize+1, -1) 10 | 11 | local pos_alpha_replicate = pos_alpha:repeatTensor(nTrial) 12 | 13 | neg_alpha = torch.add(pos_alpha_replicate, -neg_alpha) 14 | neg_alpha = torch.exp(torch.mul(neg_alpha, -gamma)) 15 | 16 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 17 | alpha:sub(batchsize+1, -1):copy(neg_alpha) 18 | return alpha 19 | end 20 | 21 | function Calculate_Alpha.cal_alpha_sum(alpha, nTrial, batchsize, gamma, init) 22 | -- alpha[postive] = init + alpha[postive + negative] (sum of all the (pos - negtive)) 23 | local pos_alpha = alpha:sub(1,batchsize):zero() 24 | local neg_alpha = alpha:sub(batchsize+1, -1) 25 | local neg_alpha_split = neg_alpha:split(batchsize, 1) 26 | for i = 1, nTrial do 27 | pos_alpha:add(neg_alpha_split[i]) 28 | end 29 | pos_alpha:add(init) 30 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 31 | alpha:sub(1,batchsize):copy(pos_alpha) 32 | return alpha 33 | end 34 | 35 | function Calculate_Alpha.cal_alpha_norm(alpha, nTrial, batchsize, gamma) 36 | -- alpha[negative] = gamma * alpha[negative]./ alpha[positive] (normalization) 37 | local pos_alpha = alpha:sub(1,batchsize) 38 | local neg_alpha = alpha:sub(batchsize+1, -1) 39 | local positive_array = torch.range(1,batchsize):type('torch.IntTensor') 40 | positive_array = positive_array:repeatTensor(nTrial) -- replicate the nTrail times 41 | positive_array = positive_array:type('torch.LongTensor') 42 | local pos_alpha_replicate = pos_alpha:index(1,positive_array) 43 | 44 | neg_alpha = torch.cdiv(neg_alpha, pos_alpha_replicate) 45 | neg_alpha = torch.mul(neg_alpha, gamma) 46 | --local new_alpha = torch.cat(pos_alpha, neg_alpha) 47 | alpha:sub(batchsize+1, -1):copy(neg_alpha) 48 | 49 | return alpha 50 | end 51 | 52 | return Calculate_Alpha -------------------------------------------------------------------------------- /CDSSM_sparse/nn/SparseLinearBatch.lua: -------------------------------------------------------------------------------- 1 | local SparseLinearBatch, parent = torch.class('nn.SparseLinearBatch', 'nn.Module') 2 | 3 | function SparseLinearBatch:__init(inputSize, outputSize) 4 | parent.__init(self) 5 | 6 | self.weightDecay = 0 7 | self.weight = torch.Tensor(outputSize, inputSize):zero() 8 | self.bias = torch.Tensor(outputSize):zero() 9 | self.gradWeight = torch.Tensor(outputSize, inputSize):zero() 10 | self.gradBias = torch.Tensor(outputSize):zero() 11 | self.lastInput = nil 12 | 13 | self.gradInput:resize(inputSize) 14 | 15 | self:reset() 16 | end 17 | 18 | function SparseLinearBatch:reset(stdv) 19 | if stdv then 20 | stdv = stdv * math.sqrt(3) 21 | else 22 | stdv = 1./math.sqrt(self.weight:size(2)) 23 | end 24 | if nn.oldSeed then 25 | for i=1,self.weight:size(1) do 26 | self.weight:select(1, i):apply(function() 27 | return torch.uniform(-stdv, stdv) 28 | end) 29 | self.bias[i] = torch.uniform(-stdv, stdv) * 0.000001 30 | end 31 | else 32 | self.weight:uniform(-stdv, stdv) 33 | self.bias:uniform(-stdv, stdv):mul(0.000001) 34 | end 35 | end 36 | 37 | function SparseLinearBatch:updateOutput(input) 38 | self.batchSize = torch.max(input:select(2,1),1)[1] 39 | local outputSize = self.bias:size(1) 40 | if torch.getnumthreads() > 1 and outputSize >= 128 then 41 | self.shardBuffer = torch.Tensor(self.batchSize, outputSize, torch.getnumthreads()) 42 | end 43 | self.output:resize(self.batchSize, outputSize):zero() 44 | 45 | return input.nn.SparseLinearBatch_updateOutput(self, input) 46 | end 47 | 48 | function SparseLinearBatch:accGradParameters(input, gradOutput, scale) 49 | if not self.lastInput then 50 | self.lastInput = input:clone() 51 | else 52 | self.lastInput:resizeAs(input):copy(input) 53 | end 54 | 55 | return input.nn.SparseLinearBatch_accGradParameters(self, input, gradOutput, scale) 56 | end 57 | 58 | function SparseLinearBatch:updateGradInput(input, gradOutput) 59 | if self.gradInput then 60 | input.nn.SparseLinearBatch_updateGradInput(self, input, gradOutput) 61 | return self.gradInput 62 | end 63 | end 64 | 65 | function SparseLinearBatch:updateParameters(learningRate) 66 | self.lastInput.nn.SparseLinearBatch_updateParameters(self, learningRate) 67 | end 68 | -------------------------------------------------------------------------------- /CDSSM_sparse/nn/CDSSMSparseConvolution.lua: -------------------------------------------------------------------------------- 1 | local CDSSMSparseConvolution, parent = torch.class('nn.CDSSMSparseConvolution', 'nn.Module') 2 | 3 | function CDSSMSparseConvolution:__init(inputSize, outputSize, winSize) 4 | -- CDSSMSparseConvlution Layer input is n*4 tensor. where 5 | -- [batchIdx, winIdx, Idx, Val] 6 | 7 | 8 | parent.__init(self) 9 | 10 | self.weightDecay = 0 11 | self.weight = torch.Tensor(outputSize, inputSize):zero() 12 | self.bias = torch.Tensor(outputSize):zero() 13 | self.gradWeight = torch.Tensor(outputSize, inputSize):zero() 14 | self.gradBias = torch.Tensor(outputSize):zero() 15 | self.lastInput = nil 16 | self.winSize = winSize 17 | self.gradInput:resize(inputSize) 18 | 19 | self:reset() 20 | end 21 | 22 | function CDSSMSparseConvolution:reset(stdv) 23 | if stdv then 24 | stdv = stdv * math.sqrt(3) 25 | else 26 | stdv = 1./math.sqrt(self.weight:size(2)) 27 | end 28 | if nn.oldSeed then 29 | for i=1,self.weight:size(1) do 30 | self.weight:select(1, i):apply(function() 31 | return torch.uniform(-stdv, stdv) 32 | end) 33 | self.bias[i] = torch.uniform(-stdv, stdv) * 0.000001 34 | end 35 | else 36 | self.weight:uniform(-stdv, stdv) 37 | self.bias:uniform(-stdv, stdv):mul(0.000001) 38 | end 39 | end 40 | 41 | function CDSSMSparseConvolution:updateOutput(input) 42 | self.batchSize = torch.max(input:select(2,1),1)[1] 43 | local outputSize = self.bias:size(1) 44 | --if torch.getnumthreads() > 1 and outputSize >= 128 then 45 | self.shardBuffer = torch.Tensor(self.batchSize, self.winSize, outputSize, torch.getnumthreads()) 46 | --end 47 | self.output:resize(self.batchSize, self.winSize, outputSize):zero() 48 | 49 | return input.nn.CDSSMSparseConvolution_updateOutput(self, input) 50 | end 51 | 52 | function CDSSMSparseConvolution:accGradParameters(input, gradOutput, scale) 53 | if not self.lastInput then 54 | self.lastInput = input:clone() 55 | else 56 | self.lastInput:resizeAs(input):copy(input) 57 | end 58 | 59 | return input.nn.CDSSMSparseConvolution_accGradParameters(self, input, gradOutput, scale) 60 | end 61 | 62 | function CDSSMSparseConvolution:updateGradInput(input, gradOutput) 63 | if self.gradInput then 64 | input.nn.CDSSMSparseConvolution_updateGradInput(self, input, gradOutput) 65 | return self.gradInput 66 | end 67 | end 68 | 69 | function CDSSMSparseConvolution:updateParameters(learningRate) 70 | self.lastInput.nn.CDSSMSparseConvolution_updateParameters(self, learningRate) 71 | end 72 | -------------------------------------------------------------------------------- /CDSSM_dense/ComputelogPD.lua: -------------------------------------------------------------------------------- 1 | local ComputelogPD = {} 2 | ComputelogPD.__index = ComputelogPD 3 | 4 | local th_utils = require 'th_utils' 5 | function ComputelogPD.LargeScaleComputeLogPD(inTsv, DColumn, scale, binNum, outTsv) 6 | -- binNum = 1 7 | 8 | local colidx = DColumn 9 | local ht_list = {} 10 | for i = 1, binNum do 11 | -- add sub dictionary of ht_list. 12 | ht_list[i] = {} 13 | end 14 | 15 | local icnt = 0 16 | local ucnt = 0 17 | 18 | local f = assert(io.open(inTsv, "r")) 19 | for line in f:lines() do 20 | icnt = icnt + 1 21 | 22 | -- split the src and tgt by '\t' 23 | local cols = {} 24 | for value in line:gmatch("[^\t]+") do 25 | table.insert(cols, value) 26 | end 27 | 28 | local key = th_utils.trim(cols[colidx]) 29 | 30 | local keyIdx = 1 -- MD5Hash key % binNum here we set 0 31 | 32 | if not ht_list[keyIdx][key] then 33 | 34 | ht_list[keyIdx][key] = 1 35 | ucnt = ucnt + 1 36 | else 37 | ht_list[keyIdx][key] = ht_list[keyIdx][key] + 1 38 | end 39 | end 40 | f:close() 41 | print('total ' .. icnt .. ' lines, unique ' .. ucnt .. ' lines') 42 | 43 | local denom = icnt 44 | if scale > -0.0001 then 45 | print('re-scale by ' .. scale) 46 | denom = 0 47 | for idx, ht in pairs(ht_list) do 48 | for key, value in pairs(ht) do 49 | local x = math.pow(value, scale) 50 | denom = denom + x 51 | ht_list[idx][key] = x 52 | end 53 | end 54 | end 55 | 56 | local logdenom = math.log(denom) 57 | 58 | local Ent = 0 59 | for idx, ht in pairs(ht_list) do 60 | for key, value in pairs(ht) do 61 | local logp = math.log(value) - logdenom 62 | Ent = Ent + math.exp(logp) * logp 63 | end 64 | end 65 | 66 | print('Entropy '.. Ent) 67 | 68 | local f = assert(io.open(inTsv, "r")) 69 | local dis = assert(io.open(outTsv, "w+")) 70 | 71 | local icnt = 0 72 | for line in f:lines() do 73 | icnt = icnt + 1 74 | 75 | -- split the src and tgt by '\t' 76 | local cols = {} 77 | for value in line:gmatch("[^\t]+") do 78 | table.insert(cols, value) 79 | end 80 | local key = th_utils.trim(cols[colidx]) 81 | local keyIdx = 1 -- MD5Hash key % binNum here we set 0 82 | 83 | local logp = math.log(ht_list[keyIdx][key]) - logdenom 84 | 85 | dis:write(string.format('%6f', logp), '\n') 86 | end 87 | 88 | f:close() 89 | dis:close() 90 | end 91 | 92 | 93 | function ComputelogPD.CalculateMD5Hash(input) 94 | -- to be implement 95 | end 96 | 97 | 98 | return ComputelogPD -------------------------------------------------------------------------------- /CDSSM_sparse/ComputelogPD.lua: -------------------------------------------------------------------------------- 1 | local ComputelogPD = {} 2 | ComputelogPD.__index = ComputelogPD 3 | 4 | local th_utils = require 'th_utils' 5 | function ComputelogPD.LargeScaleComputeLogPD(inTsv, DColumn, scale, binNum, outTsv) 6 | -- binNum = 1 7 | 8 | local colidx = DColumn 9 | local ht_list = {} 10 | for i = 1, binNum do 11 | -- add sub dictionary of ht_list. 12 | ht_list[i] = {} 13 | end 14 | 15 | local icnt = 0 16 | local ucnt = 0 17 | 18 | local f = assert(io.open(inTsv, "r")) 19 | for line in f:lines() do 20 | icnt = icnt + 1 21 | 22 | -- split the src and tgt by '\t' 23 | local cols = {} 24 | for value in line:gmatch("[^\t]+") do 25 | table.insert(cols, value) 26 | end 27 | 28 | local key = th_utils.trim(cols[colidx]) 29 | 30 | local keyIdx = 1 -- MD5Hash key % binNum here we set 0 31 | 32 | if not ht_list[keyIdx][key] then 33 | 34 | ht_list[keyIdx][key] = 1 35 | ucnt = ucnt + 1 36 | else 37 | ht_list[keyIdx][key] = ht_list[keyIdx][key] + 1 38 | end 39 | end 40 | f:close() 41 | print('total ' .. icnt .. ' lines, unique ' .. ucnt .. ' lines') 42 | 43 | local denom = icnt 44 | if scale > -0.0001 then 45 | print('re-scale by ' .. scale) 46 | denom = 0 47 | for idx, ht in pairs(ht_list) do 48 | for key, value in pairs(ht) do 49 | local x = math.pow(value, scale) 50 | denom = denom + x 51 | ht_list[idx][key] = x 52 | end 53 | end 54 | end 55 | 56 | local logdenom = math.log(denom) 57 | 58 | local Ent = 0 59 | for idx, ht in pairs(ht_list) do 60 | for key, value in pairs(ht) do 61 | local logp = math.log(value) - logdenom 62 | Ent = Ent + math.exp(logp) * logp 63 | end 64 | end 65 | 66 | print('Entropy '.. Ent) 67 | 68 | local f = assert(io.open(inTsv, "r")) 69 | local dis = assert(io.open(outTsv, "w+")) 70 | 71 | local icnt = 0 72 | for line in f:lines() do 73 | icnt = icnt + 1 74 | 75 | -- split the src and tgt by '\t' 76 | local cols = {} 77 | for value in line:gmatch("[^\t]+") do 78 | table.insert(cols, value) 79 | end 80 | local key = th_utils.trim(cols[colidx]) 81 | local keyIdx = 1 -- MD5Hash key % binNum here we set 0 82 | 83 | local logp = math.log(ht_list[keyIdx][key]) - logdenom 84 | 85 | dis:write(string.format('%6f', logp), '\n') 86 | end 87 | 88 | f:close() 89 | dis:close() 90 | end 91 | 92 | 93 | function ComputelogPD.CalculateMD5Hash(input) 94 | -- to be implement 95 | end 96 | 97 | 98 | return ComputelogPD -------------------------------------------------------------------------------- /CDSSM_dense/Batch.lua: -------------------------------------------------------------------------------- 1 | local Batch = {} 2 | Batch.__index = Batch 3 | 4 | function Batch.init() 5 | local self = {} 6 | setmetatable(self, Batch) 7 | 8 | self.BatchSize = 0 9 | self.ElementSize = 0 10 | self.SegSize = 0 11 | self.m_nFeatureDim = 0 12 | self.m_rgFeaIdx = {} 13 | self.m_rgFeaVal = {} 14 | self.m_rgSampleIdx = {} 15 | self.m_rgSegIdx = {} 16 | 17 | return self 18 | end 19 | 20 | function Batch:Clear() 21 | self.BatchSize = 0 22 | self.ElementSize = 0 23 | self.SegSize = 0 24 | self.m_nFeatureDim = 0 25 | self.m_rgFeaIdx = {} 26 | self.m_rgFeaVal = {} 27 | self.m_rgSampleIdx = {} 28 | self.m_rgSegIdx = {} 29 | end 30 | 31 | function Batch:LoadSeqSample(rgDict) 32 | 33 | local nMaxFeatureDimension = 0; 34 | 35 | local sid = (self.BatchSize == 0) and 0 or self.m_rgSampleIdx[self.BatchSize] 36 | table.insert(self.m_rgSampleIdx, sid + #rgDict) 37 | self.BatchSize = self.BatchSize + 1 38 | 39 | for _, seg in pairs(rgDict) do 40 | -- get the number of seg 41 | local count = 0 42 | for _,_ in pairs(seg) do 43 | count = count + 1 44 | end 45 | local wid = (self.SegSize == 0) and 0 or self.m_rgSegIdx[self.SegSize] 46 | table.insert(self.m_rgSegIdx, wid + count) 47 | self.SegSize = self.SegSize + 1 48 | 49 | for key, value in pairs(seg) do 50 | key = tonumber(key) 51 | value = tonumber(value) 52 | table.insert(self.m_rgFeaIdx, key) 53 | table.insert(self.m_rgFeaVal, value) 54 | self.ElementSize = self.ElementSize + 1 55 | if key >= nMaxFeatureDimension then 56 | nMaxFeatureDimension = key 57 | end 58 | end 59 | end 60 | 61 | return nMaxFeatureDimension 62 | end 63 | 64 | function Batch:WriteSeqSample() 65 | -- return the Torch.IntTensor which store one batch 66 | local dim = 1 + 1 + self.BatchSize + self.SegSize + self.ElementSize * 2 67 | local tensor = torch.IntTensor(dim) 68 | 69 | local idx = 1 70 | 71 | tensor[idx] = self.SegSize 72 | idx = idx + 1 73 | 74 | tensor[idx] = self.ElementSize 75 | idx = idx + 1 76 | 77 | for i = 1,self.BatchSize do 78 | tensor[idx] = self.m_rgSampleIdx[i] 79 | idx = idx + 1 80 | end 81 | 82 | for i = 1, self.SegSize do 83 | tensor[idx] = self.m_rgSegIdx[i] 84 | idx = idx + 1 85 | end 86 | 87 | for i = 1, self.ElementSize do 88 | tensor[idx] = self.m_rgFeaIdx[i] 89 | idx = idx + 1 90 | end 91 | 92 | for i = 1, self.ElementSize do 93 | tensor[idx] = self.m_rgFeaVal[i] 94 | idx = idx + 1 95 | end 96 | 97 | return tensor 98 | 99 | end 100 | 101 | return Batch -------------------------------------------------------------------------------- /CDSSM_sparse/Batch.lua: -------------------------------------------------------------------------------- 1 | local Batch = {} 2 | Batch.__index = Batch 3 | 4 | function Batch.init() 5 | local self = {} 6 | setmetatable(self, Batch) 7 | 8 | self.BatchSize = 0 9 | self.ElementSize = 0 10 | self.SegSize = 0 11 | self.m_nFeatureDim = 0 12 | self.m_rgFeaIdx = {} 13 | self.m_rgFeaVal = {} 14 | self.m_rgSampleIdx = {} 15 | self.m_rgSegIdx = {} 16 | 17 | return self 18 | end 19 | 20 | function Batch:Clear() 21 | self.BatchSize = 0 22 | self.ElementSize = 0 23 | self.SegSize = 0 24 | self.m_nFeatureDim = 0 25 | self.m_rgFeaIdx = {} 26 | self.m_rgFeaVal = {} 27 | self.m_rgSampleIdx = {} 28 | self.m_rgSegIdx = {} 29 | end 30 | 31 | function Batch:LoadSeqSample(rgDict) 32 | 33 | local nMaxFeatureDimension = 0; 34 | 35 | local sid = (self.BatchSize == 0) and 0 or self.m_rgSampleIdx[self.BatchSize] 36 | table.insert(self.m_rgSampleIdx, sid + #rgDict) 37 | self.BatchSize = self.BatchSize + 1 38 | 39 | for _, seg in pairs(rgDict) do 40 | -- get the number of seg 41 | local count = 0 42 | for _,_ in pairs(seg) do 43 | count = count + 1 44 | end 45 | local wid = (self.SegSize == 0) and 0 or self.m_rgSegIdx[self.SegSize] 46 | table.insert(self.m_rgSegIdx, wid + count) 47 | self.SegSize = self.SegSize + 1 48 | 49 | for key, value in pairs(seg) do 50 | key = tonumber(key) 51 | value = tonumber(value) 52 | table.insert(self.m_rgFeaIdx, key) 53 | table.insert(self.m_rgFeaVal, value) 54 | self.ElementSize = self.ElementSize + 1 55 | if key >= nMaxFeatureDimension then 56 | nMaxFeatureDimension = key 57 | end 58 | end 59 | end 60 | 61 | return nMaxFeatureDimension 62 | end 63 | 64 | function Batch:WriteSeqSample() 65 | -- return the Torch.IntTensor which store one batch 66 | local dim = 1 + 1 + self.BatchSize + self.SegSize + self.ElementSize * 2 67 | local tensor = torch.IntTensor(dim) 68 | 69 | local idx = 1 70 | 71 | tensor[idx] = self.SegSize 72 | idx = idx + 1 73 | 74 | tensor[idx] = self.ElementSize 75 | idx = idx + 1 76 | 77 | for i = 1,self.BatchSize do 78 | tensor[idx] = self.m_rgSampleIdx[i] 79 | idx = idx + 1 80 | end 81 | 82 | for i = 1, self.SegSize do 83 | tensor[idx] = self.m_rgSegIdx[i] 84 | idx = idx + 1 85 | end 86 | 87 | for i = 1, self.ElementSize do 88 | tensor[idx] = self.m_rgFeaIdx[i] 89 | idx = idx + 1 90 | end 91 | 92 | for i = 1, self.ElementSize do 93 | tensor[idx] = self.m_rgFeaVal[i] 94 | idx = idx + 1 95 | end 96 | 97 | return tensor 98 | 99 | end 100 | 101 | return Batch -------------------------------------------------------------------------------- /CDSSM_dense/PairInputStream.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | Data preparation before intilizaed the model. Including: 3 | 4 | Construct ShuffleTrainFiles if multiple input ?? need more info here. 5 | 6 | Get the dimension of document and query, get the normalizer, NCE etc. 7 | 8 | ]]-- 9 | local SequenceInputStream = require 'SequenceInputStream' 10 | 11 | local PairInputStream = {} 12 | PairInputStream.__index = PairInputStream 13 | 14 | function PairInputStream.init() 15 | local self = {} 16 | setmetatable(self, PairInputStream) 17 | 18 | self.Query_MaxSegment_batch = 40000 19 | self.Doc_MaxSegment_batch = 40000 20 | self.maxSegment_batch = 40000 21 | self.srNCEProbDist = nil 22 | self.qStream = nil 23 | self.dStream = nil 24 | 25 | return self 26 | end 27 | 28 | 29 | function PairInputStream:Load_Train_PairData(data_dir, qFileName, dFileName, nceProbDisFile, opt) 30 | 31 | local qData, dData = self:Load_PairData(data_dir, qFileName, dFileName, nceProbDisFile, opt) 32 | 33 | if opt.feature_dimension_query <= 0 or opt.feature_dimension_doc <= 0 then 34 | opt.feature_dimension_query = self.qStream.feature_size 35 | opt.feature_dimension_doc = self.dStream.feature_size 36 | end 37 | 38 | if opt.mirror_init == 1 then 39 | -- need to implement that. 40 | end 41 | 42 | return qData, dData 43 | end 44 | 45 | 46 | function PairInputStream:Load_PairData(data_dir, qFileName, dFileName, nceProbDisFile, opt) 47 | self.qStream = SequenceInputStream.init() 48 | local qData = self.qStream:get_dimension(data_dir, qFileName, opt.feature_dimension_query, opt.batch_size) 49 | 50 | self.dStream = SequenceInputStream.init() 51 | local dData = self.dStream:get_dimension(data_dir, dFileName, opt.feature_dimension_doc, opt.batch_size) 52 | 53 | if nceProbDisFile ~= nil then 54 | 55 | end 56 | self.Query_MaxSegment_batch = math.max(self.Query_MaxSegment_batch, self.qStream.maxSequence_perBatch) 57 | self.Doc_MaxSegment_batch = math.max(self.Doc_MaxSegment_batch, self.dStream.maxSequence_perBatch) 58 | self.maxSegment_batch = math.max(self.Query_MaxSegment_batch, self.Doc_MaxSegment_batch) 59 | 60 | return qData, dData 61 | end 62 | 63 | function PairInputStream:InitFeatureNorm(srcNormalizer, tgtNormalizer) 64 | -- only handle min_max normalization here. 65 | 66 | end 67 | 68 | function PairInputStream:Init_Batch() 69 | -- set the batch index to be 0 70 | self.qStream:init_batch() 71 | self.dStream:init_batch() 72 | end 73 | 74 | function PairInputStream:Next_Batch(qData, dData, srcNorm, tgtNorm, opt) 75 | -- get the next batch data. 76 | local qFlag, qData = self.qStream:Fill(qData, opt.feature_dimension_query, opt) 77 | local dFlag, dData = self.dStream:Fill(dData, opt.feature_dimension_doc, opt) 78 | if (not qFlag) or (not dFlag) then 79 | return false, qData, dData 80 | end 81 | 82 | if srcNorm ~= nil then 83 | --srcNorm.ProcessBatch() 84 | end 85 | 86 | if tgtNorm ~= nil then 87 | --tgtNorm.ProcessBatch() 88 | end 89 | 90 | return true, qData, dData 91 | end 92 | 93 | return PairInputStream -------------------------------------------------------------------------------- /CDSSM_dense/SequenceInputStream.lua: -------------------------------------------------------------------------------- 1 | 2 | local BatchSample = require 'BatchSample' 3 | 4 | local SequenceInputStream = {} 5 | SequenceInputStream.__index = SequenceInputStream 6 | 7 | function SequenceInputStream.init() 8 | local self = {} 9 | setmetatable(self, SequenceInputStream) 10 | 11 | self.total_batch_size = 0 12 | self.feature_size = 0 13 | self.maxElement_perBatch = 0 14 | self.maxSequence_perBatch = 0 15 | self.batch_num = 0 16 | self.batch_index = 0 17 | self.batch_size = 0 18 | self.last_incomplete_batch_size = 0 19 | self.mstream = nil 20 | self.dataFun = nil -- on the function, not inlcude the data 21 | 22 | self.sparse_flag = 0 -- initalized the sparse data or dense data. 23 | 24 | return self 25 | end 26 | 27 | function SequenceInputStream:get_dimension(data_dir, fileName, feature_dimension, batchSize) 28 | 29 | self.batch_size = batchSize 30 | self.mstream = torch.load(path.join(data_dir, fileName)) 31 | self.feature_size = self.mstream[-5] 32 | self.total_batch_size = self.mstream[-4] 33 | self.maxSequence_perBatch = self.mstream[-3] 34 | self.maxElement_perBatch = self.mstream[-2] 35 | 36 | local batch_size = self.mstream[-1] 37 | 38 | if feature_dimension > self.feature_size then 39 | self.feature_size = feature_dimension 40 | end 41 | print(batch_size, batchSize) 42 | assert(batch_size == batchSize, "batch_size does not match bettwen configuration and input data!") 43 | 44 | self.dataFun = BatchSample.init() 45 | local Data = BatchSample:init_Data(batch_size, self.maxSequence_perBatch, self.maxElement_perBatch) 46 | self.batch_num = math.ceil(self.total_batch_size/batch_size) 47 | self.last_incomplete_batch_size = self.total_batch_size % batch_size 48 | self.batch_index = 1 49 | 50 | return Data 51 | end 52 | 53 | function SequenceInputStream:init_batch() 54 | self.batch_index = 1 -- set the first batch 55 | end 56 | 57 | function SequenceInputStream:Fill(Data, allowedFeatureDimension, opt) 58 | 59 | if self.batch_index > self.batch_num then 60 | return false, Data 61 | end 62 | 63 | Data = self:LoadDataBatch(Data, allowedFeatureDimension, opt) 64 | self.batch_index = self.batch_index+1 65 | return true, Data 66 | end 67 | 68 | 69 | function SequenceInputStream:LoadDataBatch(Data, allowedFeatureDimension, opt) 70 | local expectedBatchSize = self.batch_size 71 | if self.batch_index == self.batch_num and self.last_incomplete_batch_size ~= 0 then 72 | expectedBatchSize = self.last_incomplete_batch_size 73 | end 74 | if self.feature_size <= allowedFeatureDimension then 75 | 76 | Data = self.dataFun:Load(Data, self.mstream, expectedBatchSize, self.batch_index) 77 | end 78 | if opt.data_format == 0 then 79 | -- if the input is dense. 80 | if opt.convo == 1 then 81 | Data = self.dataFun:Sparse_to_Dense(Data, expectedBatchSize, self.feature_size, opt) 82 | else 83 | Data = self.dataFun:Sparse_to_Dense_Linear(Data, expectedBatchSize, self.feature_size, opt) 84 | end 85 | end 86 | return Data 87 | end 88 | 89 | return SequenceInputStream -------------------------------------------------------------------------------- /CDSSM_dense/utils.lua: -------------------------------------------------------------------------------- 1 | local utils = {} 2 | utils.__index = utils 3 | 4 | local th_utils = require 'th_utils' 5 | 6 | function utils.String2FeatStrSeq(s, N, nMaxLength, featType) 7 | -- convert input string to letter-n-gram sequence, each word is a letter-n-gram vector. 8 | 9 | -- currently only the l3g featType 10 | local rgw = {} 11 | for token in s:gmatch("[^%s]+") do 12 | table.insert(rgw, token) 13 | end 14 | local rgWfs = {} 15 | for i = 1, math.min(#rgw, nMaxLength-1) do 16 | if featType == 'l3g' then 17 | table.insert(rgWfs, utils.String2L3g(rgw[i], N)) 18 | elseif featType == 'word' then 19 | -- to be implement 20 | end 21 | end 22 | 23 | local dict = {} 24 | 25 | for i = nMaxLength, #rgw do 26 | local tmp_dict = {} 27 | if featType == 'l3g' then 28 | tmp_dict = utils.String2L3g(rgw[i], N) 29 | elseif featType == 'word' then 30 | -- to be implement 31 | end 32 | 33 | for k, v in pairs(tmp_dict) do 34 | if not dict[k] then 35 | dict[k] = v 36 | else 37 | dict[k] = dict[k] + v 38 | end 39 | end 40 | end 41 | 42 | --print(table.getn(dict)) 43 | if next(dict) ~= nil then 44 | table.insert(rgWfs, dict) 45 | end 46 | return rgWfs 47 | 48 | end 49 | 50 | 51 | function utils.String2L3g(s, N) 52 | -- convert input string to letter-n-gram vector 53 | -- s : input string 54 | -- N : ngram 55 | N = N - 1 -- Lua start from 1 56 | local wfs = {} 57 | for w in s:gmatch("[^%s]+") do 58 | local src = '#' .. w .. '#' 59 | for i = 1,#src-N do 60 | local l3g = src:sub(i, i+N) 61 | if not wfs[l3g] then 62 | wfs[l3g] = 1 63 | else 64 | wfs[l3g] = wfs[l3g] + 1 65 | end 66 | end 67 | end 68 | 69 | return wfs 70 | end 71 | 72 | function utils.StrFreq2IdFreq(strFeqList, Vocab) 73 | ans = {} 74 | for _, inpDict in pairs(strFeqList) do 75 | dict = {} 76 | for k, v in pairs(inpDict) do 77 | if Vocab[k] then 78 | dict[Vocab[k]] = v 79 | end 80 | end 81 | table.insert(ans, dict) 82 | end 83 | return ans 84 | end 85 | 86 | function utils.Vector2String(vec) 87 | local str = '' 88 | local bFirst = true 89 | for k,v in pairs(vec) do 90 | if bFirst then 91 | str = k .. ':' .. v 92 | bFirst = false 93 | else 94 | str = str .. ' ' .. k .. ':' .. v 95 | end 96 | end 97 | return str 98 | end 99 | 100 | function utils.String2Vector(s) 101 | local vec = {} 102 | local rgs = th_utils.split(s, ' ') 103 | 104 | for _, v in pairs(rgs) do 105 | rgw = th_utils.split(v, ':') 106 | if not vec[rgw[1]] then 107 | vec[rgw[1]] = rgw[2] 108 | else 109 | vec[rgw[1]] = vec[rgw[1]] + rgw[2] 110 | end 111 | end 112 | return vec 113 | end 114 | 115 | function utils.Matrix2String(mt) 116 | local bFirst = true 117 | local s = '' 118 | for _,vec in pairs(mt) do 119 | if bFirst then 120 | s = utils.Vector2String(vec) 121 | bFirst = false 122 | else 123 | s = s .. '#' .. utils.Vector2String(vec) 124 | end 125 | end 126 | return s 127 | end 128 | 129 | function utils.String2Matrix(s) 130 | local mt = {} 131 | local rgv = th_utils.split(s, '#') 132 | for _,vec in pairs(rgv) do 133 | vec = utils.String2Vector(vec) 134 | table.insert(mt, vec) 135 | end 136 | return mt 137 | end 138 | 139 | return utils -------------------------------------------------------------------------------- /CDSSM_sparse/utils.lua: -------------------------------------------------------------------------------- 1 | local utils = {} 2 | utils.__index = utils 3 | 4 | local th_utils = require 'th_utils' 5 | 6 | function utils.String2FeatStrSeq(s, N, nMaxLength, featType) 7 | -- convert input string to letter-n-gram sequence, each word is a letter-n-gram vector. 8 | 9 | -- currently only the l3g featType 10 | local rgw = {} 11 | for token in s:gmatch("[^%s]+") do 12 | table.insert(rgw, token) 13 | end 14 | local rgWfs = {} 15 | for i = 1, math.min(#rgw, nMaxLength-1) do 16 | if featType == 'l3g' then 17 | table.insert(rgWfs, utils.String2L3g(rgw[i], N)) 18 | elseif featType == 'word' then 19 | -- to be implement 20 | end 21 | end 22 | 23 | local dict = {} 24 | 25 | for i = nMaxLength, #rgw do 26 | local tmp_dict = {} 27 | if featType == 'l3g' then 28 | tmp_dict = utils.String2L3g(rgw[i], N) 29 | elseif featType == 'word' then 30 | -- to be implement 31 | end 32 | 33 | for k, v in pairs(tmp_dict) do 34 | if not dict[k] then 35 | dict[k] = v 36 | else 37 | dict[k] = dict[k] + v 38 | end 39 | end 40 | end 41 | 42 | --print(table.getn(dict)) 43 | if next(dict) ~= nil then 44 | table.insert(rgWfs, dict) 45 | end 46 | return rgWfs 47 | 48 | end 49 | 50 | 51 | function utils.String2L3g(s, N) 52 | -- convert input string to letter-n-gram vector 53 | -- s : input string 54 | -- N : ngram 55 | N = N - 1 -- Lua start from 1 56 | local wfs = {} 57 | for w in s:gmatch("[^%s]+") do 58 | local src = '#' .. w .. '#' 59 | for i = 1,#src-N do 60 | local l3g = src:sub(i, i+N) 61 | if not wfs[l3g] then 62 | wfs[l3g] = 1 63 | else 64 | wfs[l3g] = wfs[l3g] + 1 65 | end 66 | end 67 | end 68 | 69 | return wfs 70 | end 71 | 72 | function utils.StrFreq2IdFreq(strFeqList, Vocab) 73 | ans = {} 74 | for _, inpDict in pairs(strFeqList) do 75 | dict = {} 76 | for k, v in pairs(inpDict) do 77 | if Vocab[k] then 78 | dict[Vocab[k]] = v 79 | end 80 | end 81 | table.insert(ans, dict) 82 | end 83 | return ans 84 | end 85 | 86 | function utils.Vector2String(vec) 87 | local str = '' 88 | local bFirst = true 89 | for k,v in pairs(vec) do 90 | if bFirst then 91 | str = k .. ':' .. v 92 | bFirst = false 93 | else 94 | str = str .. ' ' .. k .. ':' .. v 95 | end 96 | end 97 | return str 98 | end 99 | 100 | function utils.String2Vector(s) 101 | local vec = {} 102 | local rgs = th_utils.split(s, ' ') 103 | 104 | for _, v in pairs(rgs) do 105 | rgw = th_utils.split(v, ':') 106 | if not vec[rgw[1]] then 107 | vec[rgw[1]] = rgw[2] 108 | else 109 | vec[rgw[1]] = vec[rgw[1]] + rgw[2] 110 | end 111 | end 112 | return vec 113 | end 114 | 115 | function utils.Matrix2String(mt) 116 | local bFirst = true 117 | local s = '' 118 | for _,vec in pairs(mt) do 119 | if bFirst then 120 | s = utils.Vector2String(vec) 121 | bFirst = false 122 | else 123 | s = s .. '#' .. utils.Vector2String(vec) 124 | end 125 | end 126 | return s 127 | end 128 | 129 | function utils.String2Matrix(s) 130 | local mt = {} 131 | local rgv = th_utils.split(s, '#') 132 | for _,vec in pairs(rgv) do 133 | vec = utils.String2Vector(vec) 134 | table.insert(mt, vec) 135 | end 136 | return mt 137 | end 138 | 139 | return utils -------------------------------------------------------------------------------- /CDSSM_sparse/PreProcess.lua: -------------------------------------------------------------------------------- 1 | -- Preprocess the code to generate the Sparse input format. 2 | 3 | 4 | require 'torch' 5 | 6 | local batch_size = 1024 7 | local WordHash = require 'WordHash' 8 | local ComputelogPD = require 'ComputelogPD' 9 | local th_utils = require 'th_utils' 10 | 11 | -- generate the vocabulary, the answer should not do any processing. 12 | local inFile = 'data/train.pair.tok.tsv' 13 | local outFile = 'data/train' 14 | 15 | if not th_utils.IsFile(outFile..'.src.l3g.txt') then 16 | print('Creating Voc file for '.. inFile) 17 | WordHash.Pair2Voc(inFile, 3, 1, 2, outFile, 0) 18 | end 19 | 20 | -- creating pair to sequence feature. 21 | local inFile = 'data/train.pair.tok.tsv' 22 | local srcVocFile = 'data/train.src.l3g.txt' 23 | local tgtVocFile = 'data/train.tgt.l3g.txt' 24 | local nMaxLength = 20 25 | local outFile = 'data/train' 26 | 27 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 28 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 29 | print('Creating Pair2Seq Feature') 30 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 31 | end 32 | 33 | 34 | local inFile = 'data/train.src.seq.fea' 35 | local BatchSize = batch_size 36 | local outFile = 'data/train.src.seq.sparse.t7' 37 | local WinSize = 3 38 | 39 | --if not th_utils.IsFile(outFile) then 40 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 41 | WordHash.SeqFea2SparseMatrix(inFile, BatchSize, WinSize, 2118, outFile) 42 | --end 43 | 44 | local inFile = 'data/train.tgt.seq.fea' 45 | local BatchSize = batch_size 46 | local outFile = 'data/train.tgt.seq.sparse.t7' 47 | local WinSize = 3 48 | if not th_utils.IsFile(outFile) then 49 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 50 | WordHash.SeqFea2SparseMatrix(inFile, BatchSize, WinSize, 2255, outFile) 51 | end 52 | --[[ 53 | -- generate validation validation file 54 | local inFile = 'data/vqa/VQA_pair_val.txt' 55 | local srcVocFile = 'data/vqa/train.src.l3g.txt' 56 | local tgtVocFile = 'data/vqa/train.tgt.l3g.txt' 57 | local nMaxLength = 1 58 | local outFile = 'data/vqa/val' 59 | 60 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 61 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 62 | print('Creating Pair2Seq Feature for '..inFile) 63 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 64 | end 65 | 66 | local inFile = 'data/vqa/val.src.seq.fea' 67 | local BatchSize = batch_size 68 | local outFile = 'data/vqa/val.src.seq.fea.t7' 69 | 70 | if not th_utils.IsFile(outFile) then 71 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 72 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 73 | end 74 | 75 | local inFile = 'data/vqa/val.tgt.seq.fea' 76 | local BatchSize = batch_size 77 | local outFile = 'data/vqa/val.tgt.seq.fea.t7' 78 | 79 | if not th_utils.IsFile(outFile) then 80 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 81 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 82 | end 83 | 84 | -- generate test file (the test file don't have GT, and is for prediction) 85 | 86 | local inFile = 'data/vqa/VQA_pair_test.txt' 87 | local srcVocFile = 'data/vqa/train.src.l3g.txt' 88 | local tgtVocFile = 'data/vqa/train.tgt.l3g.txt' 89 | local nMaxLength = 1 90 | local outFile = 'data/vqa/test' 91 | 92 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 93 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 94 | print('Creating Pair2Seq Feature for '..inFile) 95 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 96 | end 97 | 98 | local inFile = 'data/vqa/test.src.seq.fea' 99 | local BatchSize = batch_size 100 | local outFile = 'data/vqa/test.src.seq.fea.t7' 101 | 102 | if not th_utils.IsFile(outFile) then 103 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 104 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 105 | end 106 | 107 | local inFile = 'data/vqa/test.tgt.seq.fea' 108 | local BatchSize = batch_size 109 | local outFile = 'data/vqa/test.tgt.seq.fea.t7' 110 | 111 | if not th_utils.IsFile(outFile) then 112 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 113 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 114 | end 115 | ]]-- -------------------------------------------------------------------------------- /CDSSM_dense/train.lua: -------------------------------------------------------------------------------- 1 | -- Parameter setting 2 | -- meet the 2GB out of memory again, it seems that the nested class structure 3 | -- didn't fit luaJit well. Split the data from the function. 4 | 5 | require 'torch' 6 | require 'nn' 7 | local tds = require 'tds' 8 | local DSSM_Train = require 'DSSM_Train' 9 | 10 | cmd = torch.CmdLine() 11 | cmd:text() 12 | cmd:text('Train CDSSM model') 13 | 14 | cmd:text() 15 | cmd:option('-objective', 'MMI', 'Currently only working NCE and MMI') 16 | cmd:option('-loss_report', 1, '') 17 | cmd:option('-model_path', 'cdssm/model/cdssm', '') 18 | cmd:option('-log_file', 'cdssm/model/log.txt', '') 19 | cmd:option('-qfile', 'cdssm/train.src.seq.fea.bin', '') 20 | cmd:option('-dfile', 'cdssm/train.tgt.seq.fea.bin', '') 21 | cmd:option('-nce_prob_file', 'cdssm/train.logpd.s75', '') 22 | cmd:option('-batch_size', 512, '') 23 | cmd:option('-ntrial', 30, '') 24 | cmd:option('-max_iter', 60, '') 25 | cmd:option('-gamma', 25, '') 26 | cmd:option('-learning_rate', 0.05, '') 27 | cmd:option('-weight_decay',0,'') 28 | cmd:option('-momentum',0.9,'') 29 | cmd:option('-lr', 0.005, '') 30 | cmd:option('-lr_batch', 20, '') 31 | cmd:option('-optim', 'sgd') 32 | 33 | cmd:option('-loadmodel', '', '') 34 | cmd:option('-feature_dimension_query', 0, '') 35 | cmd:option('-feature_dimension_doc', 0, '') 36 | cmd:option('-data_format', 0, '0=dense, 1=sparse') 37 | cmd:option('-word_len', 20, '') 38 | cmd:option('-mode', 'gpu', '1:gpu, 0:cpu') 39 | cmd:option('-gpu_device', 2, '') 40 | 41 | cmd:option('-convo', 0) 42 | cmd:text() 43 | 44 | opt = cmd:parse(arg) 45 | local pridiction_only = 0 46 | 47 | if opt.objective == 'NCE' then 48 | 49 | end 50 | 51 | -- the Data is the root Cdata place for storing the data. 52 | if opt.mode == 'gpu' then 53 | print('==> switching to CUDA') 54 | require 'cunn' 55 | torch.setdefaulttensortype('torch.FloatTensor') 56 | cutorch.setDevice(opt.gpu_device) 57 | end 58 | 59 | local data_dir = 'data' 60 | if pridiction_only == 0 then 61 | local qFileNameTrain = 'vqa/train.src.seq.fea.t7' 62 | local dFileNameTrain = 'vqa/train.tgt.seq.fea.t7' 63 | 64 | local dssm_train = DSSM_Train.init() 65 | local qDataTrain, dDataTrain = dssm_train:LoadTrainData(data_dir, qFileNameTrain, dFileNameTrain, nceProbDisFile, opt) 66 | 67 | local qFileNameVal = 'vqa/val.src.seq.fea.t7' 68 | local dFileNameVal = 'vqa/val.tgt.seq.fea.t7' 69 | local dssm_val = DSSM_Train.init() 70 | local qDataVal, dDataVal = dssm_val:LoadTrainData(data_dir, qFileNameVal, dFileNameVal, nceProbDisFile, opt) 71 | 72 | dssm_train:ModelInit_FromConfig(opt) 73 | local Min_err = 100000 74 | local err = nil 75 | for i = 1,opt.max_iter do 76 | print('iter ' .. i .. '...') 77 | model = dssm_train:Training(qDataTrain, dDataTrain, opt, i) 78 | dssm_train:reset_pointer() 79 | err = dssm_val:evaluate(qDataVal, dDataVal, opt, model) 80 | dssm_val:reset_pointer() 81 | 82 | if i > 20 and err < Min_err then 83 | Min_err = err 84 | torch.save('model.net', model) 85 | end 86 | end 87 | end 88 | -- doing Prediction 89 | local qFileNameTest = 'vqa/test.src.seq.fea.t7' 90 | local dFileNameTest = 'vqa/test.tgt.seq.fea.t7' 91 | local qFileNameTestId = 'data/vqa/VQA_pair_id_test.txt' 92 | local qFileNameTestMC = 'data/vqa/VQA_pair_test.txt' 93 | local dssm_test = DSSM_Train.init() 94 | local qDataTest, dDataTest = dssm_test:LoadTrainData(data_dir, qFileNameTest, dFileNameTest, nceProbDisFile, opt) 95 | print('Doing Prediction ...') 96 | model = torch.load('model.net') 97 | local question_id = {} 98 | local f = assert(io.open(qFileNameTestId, "r")) 99 | 100 | for line in f:lines() do 101 | local sent = {} 102 | for value in line:gmatch("[^\t]+") do 103 | table.insert(sent, value) 104 | end 105 | table.insert(question_id, tonumber(sent[1])) 106 | end 107 | f:close() 108 | 109 | local MC_ans = {} 110 | local f = assert(io.open(qFileNameTestMC, "r")) 111 | 112 | for line in f:lines() do 113 | local sent = {} 114 | for value in line:gmatch("[^\t]+") do 115 | table.insert(sent, value) 116 | end 117 | 118 | local tmp = {} 119 | for i = 1, 18 do 120 | table.insert(tmp, sent[i+1]) 121 | end 122 | table.insert(MC_ans, tmp) 123 | end 124 | f:close() 125 | 126 | result_table = dssm_test:predict(qDataTest, dDataTest, opt, model, question_id, MC_ans) 127 | local cjson = require 'cjson' 128 | local encode_result = cjson.encode(result_table) 129 | 130 | local file = torch.DiskFile('result/result.json','w') 131 | file:writeString(encode_result) 132 | file:close() 133 | -------------------------------------------------------------------------------- /CDSSM_dense/PreProcess.lua: -------------------------------------------------------------------------------- 1 | -- Test the function. 2 | 3 | require 'torch' 4 | 5 | local batch_size = 512 6 | local WordHash = require 'WordHash' 7 | local ComputelogPD = require 'ComputelogPD' 8 | local SequenceInputStream = require 'SequenceInputStream' 9 | local th_utils = require 'th_utils' 10 | 11 | -- generate the vocabulary, the answer should not do any processing. 12 | local inFile = 'data/vqa/VQA_pair_train.txt' 13 | local outFile = 'data/vqa/train' 14 | 15 | if not th_utils.IsFile(outFile..'.src.l3g.txt') then 16 | print('Creating Voc file for '.. inFile) 17 | WordHash.Pair2Voc(inFile, 3, 1, 2, outFile, 0) 18 | end 19 | 20 | -- during testing, for each batch, we first generate all unique label in training, and 21 | -- for each batch, we rank the answers by all the unqiue label. (for open-ended) 22 | 23 | -- for each questions in testing, using the multiple choices, and rank to get the predict 24 | -- labels. (for multiple choice.) 25 | 26 | local inFile = 'data/vqa/VQA_pair_train.txt' 27 | local srcVocFile = 'data/vqa/train.src.l3g.txt' 28 | local tgtVocFile = 'data/vqa/train.tgt.l3g.txt' 29 | local nMaxLength = 1 30 | local outFile = 'data/vqa/train' 31 | 32 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 33 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 34 | print('Creating Pair2Seq Feature') 35 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 36 | end 37 | 38 | local inFile = 'data/vqa/train.src.seq.fea' 39 | local BatchSize = batch_size 40 | local outFile = 'data/vqa/train.src.seq.fea.t7' 41 | 42 | if not th_utils.IsFile(outFile) then 43 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 44 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 45 | end 46 | 47 | local inFile = 'data/vqa/train.tgt.seq.fea' 48 | local BatchSize = batch_size 49 | local outFile = 'data/vqa/train.tgt.seq.fea.t7' 50 | 51 | if not th_utils.IsFile(outFile) then 52 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 53 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 54 | end 55 | 56 | -- generate validation validation file 57 | local inFile = 'data/vqa/VQA_pair_val.txt' 58 | local srcVocFile = 'data/vqa/train.src.l3g.txt' 59 | local tgtVocFile = 'data/vqa/train.tgt.l3g.txt' 60 | local nMaxLength = 1 61 | local outFile = 'data/vqa/val' 62 | 63 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 64 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 65 | print('Creating Pair2Seq Feature for '..inFile) 66 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 67 | end 68 | 69 | local inFile = 'data/vqa/val.src.seq.fea' 70 | local BatchSize = batch_size 71 | local outFile = 'data/vqa/val.src.seq.fea.t7' 72 | 73 | if not th_utils.IsFile(outFile) then 74 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 75 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 76 | end 77 | 78 | local inFile = 'data/vqa/val.tgt.seq.fea' 79 | local BatchSize = batch_size 80 | local outFile = 'data/vqa/val.tgt.seq.fea.t7' 81 | 82 | if not th_utils.IsFile(outFile) then 83 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 84 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 85 | end 86 | 87 | -- generate test file (the test file don't have GT, and is for prediction) 88 | 89 | local inFile = 'data/vqa/VQA_pair_test.txt' 90 | local srcVocFile = 'data/vqa/train.src.l3g.txt' 91 | local tgtVocFile = 'data/vqa/train.tgt.l3g.txt' 92 | local nMaxLength = 1 93 | local outFile = 'data/vqa/test' 94 | 95 | if not (th_utils.IsFile(outFile..'.src.seq.fea') and th_utils.IsFile(outFile..'.tgt.seq.fea') 96 | and th_utils.IsFile(outFile..'.NegTgt.seq.fea')) then 97 | print('Creating Pair2Seq Feature for '..inFile) 98 | WordHash.Pair2SeqFea(inFile, srcVocFile, tgtVocFile, nMaxLength, 1, 2, outFile, 'l3g') 99 | end 100 | 101 | local inFile = 'data/vqa/test.src.seq.fea' 102 | local BatchSize = batch_size 103 | local outFile = 'data/vqa/test.src.seq.fea.t7' 104 | 105 | if not th_utils.IsFile(outFile) then 106 | print('Converting Seq feature to bin: '.. inFile .. ' BatchSize: ' .. BatchSize) 107 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 108 | end 109 | 110 | local inFile = 'data/vqa/test.tgt.seq.fea' 111 | local BatchSize = batch_size 112 | local outFile = 'data/vqa/test.tgt.seq.fea.t7' 113 | 114 | if not th_utils.IsFile(outFile) then 115 | print('Converting Seq feature to bin: '.. inFile.. ' BatchSize: ' .. BatchSize) 116 | WordHash.SeqFea2Bin(inFile, BatchSize, outFile) 117 | end -------------------------------------------------------------------------------- /CDSSM_dense/BatchSample.lua: -------------------------------------------------------------------------------- 1 | local BatchSample = {} 2 | BatchSample.__index = BatchSample 3 | local tds = require 'tds' 4 | 5 | function BatchSample.init() 6 | -- allocate the space for each sample 7 | local self = {} 8 | setmetatable(self, BatchSample) 9 | self.batch_size = nil 10 | self.pointer = 1 11 | 12 | self.add_pointer = function() self.pointer = self.pointer + 1 end 13 | return self, Data 14 | end 15 | 16 | function BatchSample:init_Data(max_batch_size, maxSequence_perBatch, maxElements_perBatch) 17 | local Data = tds.hash() 18 | 19 | Data.sample_Idx_Mem = torch.IntTensor(max_batch_size) 20 | Data.seg_Idx_Mem = torch.IntTensor(maxSequence_perBatch) 21 | Data.seg_Margin_Mem = torch.IntTensor(maxSequence_perBatch) 22 | Data.seg_Len_Mem = torch.IntTensor(maxSequence_perBatch) 23 | Data.fea_Idx_Mem = torch.IntTensor(maxElements_perBatch) 24 | Data.fea_Value_Mem = torch.FloatTensor(maxElements_perBatch) 25 | Data.data_matrix = nil 26 | Data.batch_size = nil 27 | return Data 28 | end 29 | 30 | function BatchSample:Load(Data, mstream, expectedBatchSize, index) 31 | 32 | Data.batch_size = expectedBatchSize 33 | 34 | -- load into 1D sequence vector. 35 | self.batch_size = expectedBatchSize 36 | local segsize = mstream[self.pointer] 37 | self.add_pointer() 38 | 39 | local elementsize = mstream[self.pointer] 40 | self.add_pointer() 41 | 42 | --print(batch_size, segsize, elementsize) 43 | 44 | 45 | for i = 1, self.batch_size do 46 | Data.sample_Idx_Mem[i] = mstream[self.pointer] 47 | self.add_pointer() 48 | end 49 | 50 | --print(self.sample_Idx_Mem) --test right 51 | 52 | local smp_index = 1 53 | for i = 1, segsize do 54 | Data.seg_Idx_Mem[i] = mstream[self.pointer] 55 | self.add_pointer() 56 | while Data.sample_Idx_Mem[smp_index] < i do 57 | smp_index = smp_index + 1 58 | end 59 | Data.seg_Margin_Mem[i] = smp_index 60 | Data.seg_Len_Mem[i] = 0 61 | end 62 | 63 | for i = 1, elementsize do 64 | Data.fea_Idx_Mem[i] = mstream[self.pointer] 65 | self.add_pointer() 66 | end 67 | --print(self.fea_Idx_Mem)-- the word level hash order maybe different-- mater? 68 | 69 | local sum = 0 70 | local seg_index = 1 71 | 72 | for i = 1, elementsize do 73 | Data.fea_Value_Mem[i] = mstream[self.pointer] 74 | self.add_pointer() 75 | while (Data.seg_Idx_Mem[seg_index] < i) do 76 | Data.seg_Len_Mem[seg_index] = sum 77 | seg_index = seg_index + 1 78 | sum = 0 79 | end 80 | 81 | sum = sum + Data.fea_Value_Mem[i] 82 | end 83 | Data.seg_Len_Mem[seg_index] = sum 84 | --print(self.seg_Len_Mem) --checked right 85 | return Data 86 | end 87 | 88 | function BatchSample:Sparse_to_Dense(Data, expectedBatchSize, feature_size, opt) 89 | -- transfer the sparse vector to dense matrix batch. 90 | local win_size = 3 91 | local seg_len = opt.word_len -- we need to fix the seg_len here, the last dim is bow encodding. 92 | -- initialize the tensor. 93 | local data_matrix = torch.IntTensor(expectedBatchSize, seg_len, feature_size):zero() 94 | local zero_matrix = torch.IntTensor(1):zero() 95 | Data.sample_Idx_Mem = torch.cat(zero_matrix, Data.sample_Idx_Mem) 96 | Data.seg_Idx_Mem = torch.cat(zero_matrix, Data.seg_Idx_Mem) 97 | 98 | for i = 1, expectedBatchSize do 99 | -- for each sentence 100 | local sample_Idx_start = Data.sample_Idx_Mem[i] 101 | local sample_Idx_end = Data.sample_Idx_Mem[i+1] 102 | for j = sample_Idx_start+1, sample_Idx_end do 103 | -- for each words 104 | local seg_Idx_start = Data.seg_Idx_Mem[j] 105 | local seg_Idx_end = Data.seg_Idx_Mem[j+1] 106 | for k = seg_Idx_start+1, seg_Idx_end do 107 | local idx = Data.fea_Idx_Mem[k] 108 | local value = Data.fea_Value_Mem[k] 109 | data_matrix[i][j-sample_Idx_start][idx] = value 110 | end 111 | end 112 | end 113 | -- this is only when window size == 1, 114 | Data.data_matrix = data_matrix:sub(1,expectedBatchSize, 1, seg_len - win_size +1) 115 | 116 | for i = 2,win_size do 117 | Data.data_matrix = torch.cat(Data.data_matrix, data_matrix:sub(1, expectedBatchSize, i, seg_len - win_size + i)) 118 | end 119 | return Data 120 | end 121 | 122 | 123 | function BatchSample:Sparse_to_Dense_Linear(Data, expectedBatchSize, feature_size, opt) 124 | -- transfer the sparse vector to dense matrix batch. 125 | local win_size = 1 126 | local seg_len = opt.word_len -- we need to fix the seg_len here, the last dim is bow encodding. 127 | -- initialize the tensor. 128 | local data_matrix = torch.IntTensor(expectedBatchSize, feature_size):zero() 129 | local zero_matrix = torch.IntTensor(1):zero() 130 | 131 | Data.sample_Idx_Mem = torch.cat(zero_matrix, Data.sample_Idx_Mem) 132 | Data.seg_Idx_Mem = torch.cat(zero_matrix, Data.seg_Idx_Mem) 133 | 134 | for i = 1, expectedBatchSize do 135 | -- for each sentence 136 | local sample_Idx_start = Data.sample_Idx_Mem[i] 137 | local sample_Idx_end = Data.sample_Idx_Mem[i+1] 138 | for j = sample_Idx_start+1, sample_Idx_end do 139 | -- for each words 140 | local seg_Idx_start = Data.seg_Idx_Mem[j] 141 | local seg_Idx_end = Data.seg_Idx_Mem[j+1] 142 | for k = seg_Idx_start+1, seg_Idx_end do 143 | local idx = Data.fea_Idx_Mem[k] 144 | local value = Data.fea_Value_Mem[k] 145 | data_matrix[i][idx] = value 146 | end 147 | end 148 | end 149 | 150 | Data.data_matrix = data_matrix 151 | -- this is only when window size == 1, 152 | --Data.data_matrix = data_matrix:sub(1,expectedBatchSize, 1, seg_len - win_size +1) 153 | 154 | return Data 155 | end 156 | 157 | return BatchSample -------------------------------------------------------------------------------- /CDSSM_sparse/sparse_train.lua: -------------------------------------------------------------------------------- 1 | -- Sparse Format CDSSM, Parameter setting 2 | 3 | require 'torch' 4 | require 'nn' 5 | local tds = require 'tds' 6 | 7 | require 'nngraph' 8 | require 'DSSM_MMI_Criterion' 9 | require 'optim' 10 | local th_utils = require 'th_utils' 11 | 12 | cmd = torch.CmdLine() 13 | cmd:text() 14 | cmd:text('Train CDSSM model') 15 | 16 | cmd:text() 17 | cmd:option('-objective', 'MMI', 'Currently only working NCE and MMI') 18 | cmd:option('-batch_size', 1024, '') 19 | cmd:option('-ntrial', 50, '') 20 | cmd:option('-max_iter', 60, '') 21 | cmd:option('-gamma', 25, '') 22 | cmd:option('-learning_rate', 0.01, '') 23 | cmd:option('-weight_decay',0,'') 24 | cmd:option('-momentum',0.9,'') 25 | cmd:option('-lr', 0.001, '') 26 | cmd:option('-lr_batch', 20, '') 27 | cmd:option('-optim', 'sgd') 28 | 29 | cmd:option('-loadmodel', '', '') 30 | cmd:option('-feature_dimension_query', 2118, '') 31 | cmd:option('-feature_dimension_doc', 2255, '') 32 | cmd:option('-mode', 'cpu', '1:gpu, 0:cpu') 33 | cmd:option('-gpu_device', 2, '') 34 | 35 | cmd:option('-convo', 1) 36 | cmd:text() 37 | 38 | opt = cmd:parse(arg) 39 | 40 | if opt.mode == 'gpu' then 41 | print('==> switching to CUDA') 42 | require 'cunn' 43 | torch.setdefaulttensortype('torch.FloatTensor') 44 | cutorch.setDevice(opt.gpu_device) 45 | end 46 | 47 | local data_dir = 'data' 48 | local qFileNameTrain = 'data/train.src.seq.sparse.t7' 49 | local dFileNameTrain = 'data/train.tgt.seq.sparse.t7' 50 | 51 | qDataTrain = torch.load(qFileNameTrain) 52 | dDataTrain = torch.load(dFileNameTrain) 53 | 54 | 55 | -- define the model 56 | local wind_size = 3 57 | 58 | local Q_feature_size = opt.feature_dimension_query 59 | local D_feature_size = opt.feature_dimension_doc 60 | if opt.convo == 0 then 61 | local Q_layer = nn.Sequential() 62 | Q_layer:add(nn.SparseLinearBatch(Q_feature_size, 1000)) 63 | Q_layer:add(nn.Tanh()) 64 | Q_layer:add(nn.Dropout(0.5)) 65 | Q_layer:add(nn.Linear(1000, 300)) 66 | Q_layer:add(nn.Tanh()) 67 | 68 | 69 | local D_layer = nn.Sequential() 70 | D_layer:add(nn.SparseLinearBatch(D_feature_size, 1000)) 71 | D_layer:add(nn.Tanh()) 72 | D_layer:add(nn.Dropout(0.5)) 73 | D_layer:add(nn.Linear(1000, 300)) 74 | D_layer:add(nn.Tanh()) 75 | 76 | model = nn.ParallelTable() 77 | model:add(Q_layer) 78 | model:add(D_layer) 79 | else 80 | local Q_layer = nn.Sequential() 81 | Q_layer:add(nn.CDSSMSparseConvolution(Q_feature_size * wind_size, 1000, 20)) 82 | Q_layer:add(nn.Tanh()) 83 | Q_layer:add(nn.TemporalMaxPooling(20)) 84 | Q_layer:add(nn.Reshape(1000, true)) 85 | Q_layer:add(nn.Linear(1000, 300)) 86 | 87 | local D_layer = nn.Sequential() 88 | D_layer:add(nn.CDSSMSparseConvolution(D_feature_size * wind_size, 1000, 20)) 89 | D_layer:add(nn.Tanh()) 90 | D_layer:add(nn.TemporalMaxPooling(20)) 91 | D_layer:add(nn.Reshape(1000, true)) 92 | D_layer:add(nn.Linear(1000, 300)) 93 | 94 | model = nn.ParallelTable() 95 | model:add(Q_layer) 96 | model:add(D_layer) 97 | end 98 | 99 | if opt.mode == 'gpu' then 100 | model:cuda() 101 | end 102 | 103 | parameters,gradParameters = model:getParameters() 104 | 105 | if opt.optim == 'sgd' then 106 | optimState = { 107 | learningRate = opt.learning_rate, 108 | weightDecay = opt.weight_decay, 109 | momentum = opt.momentum, 110 | learningRateDecay = 1e-7 111 | } 112 | optimMethod = optim.sgd 113 | end 114 | if opt.optim == 'rmsprop' then 115 | optimState = { 116 | learningRate = nil, 117 | alpha = nil, 118 | epsilon = nil 119 | } 120 | optimMethod = optim.rmsprop 121 | end 122 | 123 | print(model) 124 | -- training: 125 | 126 | function train(qData, dData, opt, epoch) 127 | if epoch > opt.lr_batch then 128 | self.optimState.learningRate = opt.lr 129 | end 130 | --print('learning rate: '..optimState.learningRate) 131 | if opt.loadmodel ~= '' then 132 | model = torch.load(opt.loadmodel) 133 | end 134 | model:training() 135 | local trainingLoss = 0 136 | local total_err = 0 137 | 138 | for i = 1, #qData do 139 | local qTensor, dTensor 140 | if opt.mode == 'gpu' then 141 | qTensor = qData[i]:cuda() 142 | dTensor = dData[i]:cuda() 143 | else 144 | qTensor = qData[i]:double() 145 | dTensor = dData[i]:double() 146 | end 147 | batch_size = torch.max(qTensor:select(2,1)) 148 | local feval = function(x) 149 | if x ~= parameters then 150 | parameters:copy(x) 151 | end 152 | gradParameters:zero() 153 | -- get the cosine similarity for all positive and negtive. 154 | local output = model:forward({qTensor, dTensor}) 155 | 156 | if opt.objective == 'NCE' then 157 | -- load the doc distance for NCE Training. 158 | end 159 | if opt.objective == 'MMI' then 160 | criterion = nn.DSSM_MMI_Criterion(batch_size, opt.ntrial, opt.gamma, opt.batch_size) 161 | end 162 | if opt.mode == 'gpu' then 163 | criterion:cuda() 164 | end 165 | local err = criterion:updateOutput(output) 166 | local do_df = criterion:updateGradInput() 167 | model:backward({qTensor, dTensor}, do_df) 168 | gradParameters:div(batch_size) 169 | total_err = total_err + err 170 | err = err / batch_size 171 | return err, gradParameters 172 | end 173 | optimMethod(feval, parameters, optimState) 174 | 175 | end 176 | collectgarbage() 177 | return total_err 178 | end 179 | 180 | for i = 1,opt.max_iter do 181 | local timer = torch.Timer(); 182 | total_err = train(qDataTrain, dDataTrain, opt, 1) 183 | print('training error is: ' .. total_err) 184 | --print(string.format('load x:%f',timer:time().real)); 185 | end 186 | -------------------------------------------------------------------------------- /CDSSM_dense/DSSM_MMI_Criterion.lua: -------------------------------------------------------------------------------- 1 | local DSSM_MMI_Criterion, parent = torch.class('nn.DSSM_MMI_Criterion', 'nn.Criterion') 2 | 3 | local Calculate_Alpha = require 'Calculate_Alpha' 4 | 5 | function DSSM_MMI_Criterion:__init(batch_size, nTrail, gamma, BATCH_SIZE) 6 | parent.__init(self) 7 | self.batch_size = batch_size 8 | self.nTrail = nTrail 9 | self.gamma = gamma 10 | -- do negative sampling 11 | self.BATCH_SIZE = BATCH_SIZE 12 | self.D_negtive_array = torch.IntTensor(batch_size * nTrail) 13 | 14 | for i = 1, nTrail do 15 | local randpos = torch.random(0.8*batch_size) + math.floor(0.1 * batch_size) 16 | for k = 1, batch_size do 17 | local bs = (randpos + k) % batch_size + 1 18 | self.D_negtive_array[(i-1)*batch_size + k] = bs 19 | end 20 | end 21 | 22 | --self.D_negtive_array = torch.load('negSampline') 23 | 24 | 25 | -- inverse negtive array 26 | self.D_inver_negtive_index_array = torch.IntTensor(batch_size * nTrail):zero() 27 | self.D_inver_negtive_value_array = torch.IntTensor(batch_size * nTrail):zero() 28 | 29 | for i = 1, nTrail do 30 | local mlist = torch.IntTensor(batch_size):zero() 31 | 32 | for k = 1, batch_size do 33 | local bs = self.D_negtive_array[(i-1) * batch_size + k] 34 | mlist[bs] = k 35 | end 36 | 37 | local ptotal = 0 38 | local pindex = 1 39 | for k = 1, batch_size do 40 | self.D_inver_negtive_value_array[(i-1)*batch_size+pindex] = mlist[k] 41 | pindex = pindex + 1 42 | self.D_inver_negtive_index_array[(i-1)*batch_size+k] = ptotal + 1 43 | ptotal = self.D_inver_negtive_index_array[(i-1)*batch_size+k] 44 | 45 | end 46 | end 47 | 48 | -- concatinate with the postive. 49 | self.gradInput = {torch.Tensor(), torch.Tensor()} 50 | end 51 | 52 | function DSSM_MMI_Criterion:updateOutput(input) 53 | 54 | local Q_input, D_input = input[1],input[2] 55 | --Q_input:tanh() 56 | --D_input:tanh() 57 | 58 | self.dimension = Q_input:size()[2] 59 | self.input1 = Q_input:repeatTensor(self.nTrail+1,1) 60 | self.input2 = torch.Tensor((self.nTrail+1)*self.batch_size, self.dimension):zero():typeAs(D_input) 61 | self.input2:sub(1, self.batch_size):copy(D_input) -- copy the positive 62 | 63 | for i = 1, self.nTrail * self.batch_size do 64 | self.input2:sub(i+self.batch_size, i+self.batch_size):copy(D_input:sub(self.D_negtive_array[i],self.D_negtive_array[i])) 65 | end 66 | 67 | 68 | -- 1: calculate the cosine distance for positive and negtive array 69 | 70 | if not self.buffer then 71 | self.buffer = self.input1.new() 72 | self.w1 = self.input1.new() 73 | self.w22 = self.input1.new() 74 | self.w = self.input1.new() 75 | self.w32 = self.input1.new() 76 | self.ones = self.input1.new() 77 | end 78 | 79 | self.buffer:cmul(self.input1, self.input2) 80 | self.w1:sum(self.buffer,2) 81 | 82 | local epsilon = 1e-12 83 | self.buffer:cmul(self.input1,self.input1) 84 | self.w22:sum(self.buffer,2):add(epsilon) 85 | self.ones:resizeAs(self.w22):fill(1) 86 | self.w22:cdiv(self.ones, self.w22) 87 | self.w:resizeAs(self.w22):copy(self.w22) 88 | 89 | self.buffer:cmul(self.input2,self.input2) 90 | self.w32:sum(self.buffer,2):add(epsilon) 91 | self.w32:cdiv(self.ones, self.w32) 92 | self.w:cmul(self.w32) 93 | self.w:sqrt() 94 | -- self.w = b*c 95 | -- self.w32 = c^2 96 | -- self.w22 = b^2 97 | -- self.w1 = a 98 | self.alpha_buffer= torch.cmul(self.w1,self.w) 99 | self.alpha_buffer = self.alpha_buffer:select(2,1) 100 | 101 | -- 2: calculate the alpha 102 | self.alpha_buffer = Calculate_Alpha.cal_alpha(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma) -- check right. 103 | self.alpha_buffer = Calculate_Alpha.cal_alpha_sum(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma,1) 104 | self.alpha_buffer = Calculate_Alpha.cal_alpha_norm(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma) 105 | self.alpha_buffer = Calculate_Alpha.cal_alpha_sum(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma,0) 106 | 107 | -- 3: calculate the loss 108 | local err = 0 109 | local eps = 1.4e-45 110 | for i = 1, self.batch_size do 111 | err = err + math.log(math.max(eps, (1+self.alpha_buffer[i] / math.max(self.gamma - self.alpha_buffer[i], eps)))) 112 | --err = err + math.log(math.max(self.alpha_buffer[i]) 113 | end 114 | return err 115 | end 116 | 117 | function DSSM_MMI_Criterion:updateGradInput() 118 | 119 | local gw1 = torch.Tensor():typeAs(self.input1) 120 | local gw2 = torch.Tensor():typeAs(self.input2) 121 | gw1:resizeAs(self.input1):zero() 122 | gw2:resizeAs(self.input2):zero() 123 | 124 | self.w = self.w:expandAs(self.input1) 125 | self.w22 = self.w22:expandAs(self.input1) 126 | self.w32 = self.w32:expandAs(self.input1) 127 | self.w1 = self.w1:expandAs(self.input1) 128 | 129 | gw1:cmul(self.input2, self.w) -- bc * Yd 130 | 131 | -- q * a * (b*b*b*c) 132 | self.buffer:cmul(self.w, self.w22) -- (b*b*b*c) 133 | self.buffer:cmul(self.w1, self.buffer) -- a * ... 134 | self.buffer:cmul(self.input1, self.buffer) -- Yq * ... 135 | 136 | gw1:add(-self.buffer) -- bc * Yd - a * (b*b*b*c) * Yq 137 | 138 | -- for deriv_d 139 | -- q / (b * c) 140 | 141 | gw2:cmul(self.input1, self.w) 142 | -- d * a / (b*c*c*c) 143 | self.buffer:cmul(self.w, self.w32) 144 | self.buffer:cmul(self.w1, self.buffer) 145 | self.buffer:cmul(self.input2, self.buffer) 146 | 147 | -- q / (b*c) - d*a / (bbbc) 148 | gw2:add(-self.buffer) 149 | 150 | -- matrix_weightAdd 151 | -- for i = 1, batchsize*dimension 152 | local ngw1 = torch.Tensor(self.batch_size, self.dimension):zero():typeAs(gw1) 153 | local ngw2 = torch.Tensor(self.batch_size, self.dimension):zero():typeAs(gw2) 154 | 155 | ngw1:cmul(gw1:sub(1, self.batch_size), self.alpha_buffer:sub(1, self.batch_size):view(-1,1):expandAs(ngw1)) 156 | 157 | for i = 1, self.nTrail do 158 | ngw1:addcmul(-1, gw1:sub(i*self.batch_size+1, (i+1)*self.batch_size), 159 | self.alpha_buffer:sub(i*self.batch_size+1, (i+1)*self.batch_size):view(-1,1):expandAs(ngw1)) 160 | end 161 | 162 | 163 | ngw2:cmul(gw2:sub(1, self.batch_size), self.alpha_buffer:sub(1, self.batch_size):view(-1,1):expandAs(ngw2)) 164 | for i = 1, self.batch_size do 165 | for j = 1, self.nTrail do 166 | local col = self.D_inver_negtive_index_array[(j-1)*self.batch_size + i] 167 | local row = self.D_inver_negtive_value_array[(j-1)*self.batch_size + col] 168 | ngw2:sub(i,i):addcmul(-1, gw2:sub(j*self.batch_size + row, j*self.batch_size + row), 169 | self.alpha_buffer:sub(j*self.batch_size + row, j*self.batch_size + row):view(-1,1):expandAs(ngw2:sub(i,i))) 170 | end 171 | end 172 | 173 | self.gradInput = {-ngw1, -ngw2} 174 | 175 | return self.gradInput 176 | 177 | end 178 | -------------------------------------------------------------------------------- /CDSSM_sparse/DSSM_MMI_Criterion.lua: -------------------------------------------------------------------------------- 1 | local DSSM_MMI_Criterion, parent = torch.class('nn.DSSM_MMI_Criterion', 'nn.Criterion') 2 | 3 | local Calculate_Alpha = require 'Calculate_Alpha' 4 | 5 | function DSSM_MMI_Criterion:__init(batch_size, nTrail, gamma, BATCH_SIZE) 6 | parent.__init(self) 7 | self.batch_size = batch_size 8 | self.nTrail = nTrail 9 | self.gamma = gamma 10 | -- do negative sampling 11 | self.BATCH_SIZE = BATCH_SIZE 12 | self.D_negtive_array = torch.IntTensor(batch_size * nTrail) 13 | 14 | for i = 1, nTrail do 15 | local randpos = torch.random(0.8*batch_size) + math.floor(0.1 * batch_size) 16 | for k = 1, batch_size do 17 | local bs = (randpos + k) % batch_size + 1 18 | self.D_negtive_array[(i-1)*batch_size + k] = bs 19 | end 20 | end 21 | 22 | --self.D_negtive_array = torch.load('negSampline') 23 | 24 | 25 | -- inverse negtive array 26 | self.D_inver_negtive_index_array = torch.IntTensor(batch_size * nTrail):zero() 27 | self.D_inver_negtive_value_array = torch.IntTensor(batch_size * nTrail):zero() 28 | 29 | for i = 1, nTrail do 30 | local mlist = torch.IntTensor(batch_size):zero() 31 | 32 | for k = 1, batch_size do 33 | local bs = self.D_negtive_array[(i-1) * batch_size + k] 34 | mlist[bs] = k 35 | end 36 | 37 | local ptotal = 0 38 | local pindex = 1 39 | for k = 1, batch_size do 40 | self.D_inver_negtive_value_array[(i-1)*batch_size+pindex] = mlist[k] 41 | pindex = pindex + 1 42 | self.D_inver_negtive_index_array[(i-1)*batch_size+k] = ptotal + 1 43 | ptotal = self.D_inver_negtive_index_array[(i-1)*batch_size+k] 44 | 45 | end 46 | end 47 | 48 | -- concatinate with the postive. 49 | self.gradInput = {torch.Tensor(), torch.Tensor()} 50 | end 51 | 52 | function DSSM_MMI_Criterion:updateOutput(input) 53 | 54 | local Q_input, D_input = input[1],input[2] 55 | --Q_input:tanh() 56 | --D_input:tanh() 57 | 58 | self.dimension = Q_input:size()[2] 59 | self.input1 = Q_input:repeatTensor(self.nTrail+1,1) 60 | self.input2 = torch.Tensor((self.nTrail+1)*self.batch_size, self.dimension):zero():typeAs(D_input) 61 | self.input2:sub(1, self.batch_size):copy(D_input) -- copy the positive 62 | 63 | for i = 1, self.nTrail * self.batch_size do 64 | self.input2:sub(i+self.batch_size, i+self.batch_size):copy(D_input:sub(self.D_negtive_array[i],self.D_negtive_array[i])) 65 | end 66 | 67 | 68 | -- 1: calculate the cosine distance for positive and negtive array 69 | 70 | if not self.buffer then 71 | self.buffer = self.input1.new() 72 | self.w1 = self.input1.new() 73 | self.w22 = self.input1.new() 74 | self.w = self.input1.new() 75 | self.w32 = self.input1.new() 76 | self.ones = self.input1.new() 77 | end 78 | 79 | self.buffer:cmul(self.input1, self.input2) 80 | self.w1:sum(self.buffer,2) 81 | 82 | local epsilon = 1e-12 83 | self.buffer:cmul(self.input1,self.input1) 84 | self.w22:sum(self.buffer,2):add(epsilon) 85 | self.ones:resizeAs(self.w22):fill(1) 86 | self.w22:cdiv(self.ones, self.w22) 87 | self.w:resizeAs(self.w22):copy(self.w22) 88 | 89 | self.buffer:cmul(self.input2,self.input2) 90 | self.w32:sum(self.buffer,2):add(epsilon) 91 | self.w32:cdiv(self.ones, self.w32) 92 | self.w:cmul(self.w32) 93 | self.w:sqrt() 94 | -- self.w = b*c 95 | -- self.w32 = c^2 96 | -- self.w22 = b^2 97 | -- self.w1 = a 98 | self.alpha_buffer= torch.cmul(self.w1,self.w) 99 | self.alpha_buffer = self.alpha_buffer:select(2,1) 100 | 101 | -- 2: calculate the alpha 102 | self.alpha_buffer = Calculate_Alpha.cal_alpha(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma) -- check right. 103 | self.alpha_buffer = Calculate_Alpha.cal_alpha_sum(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma,1) 104 | self.alpha_buffer = Calculate_Alpha.cal_alpha_norm(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma) 105 | self.alpha_buffer = Calculate_Alpha.cal_alpha_sum(self.alpha_buffer, self.nTrail, self.batch_size, self.gamma,0) 106 | 107 | -- 3: calculate the loss 108 | local err = 0 109 | local eps = 1.4e-45 110 | for i = 1, self.batch_size do 111 | err = err + math.log(math.max(eps, (1+self.alpha_buffer[i] / math.max(self.gamma - self.alpha_buffer[i], eps)))) 112 | --err = err + math.log(math.max(self.alpha_buffer[i]) 113 | end 114 | return err 115 | end 116 | 117 | function DSSM_MMI_Criterion:updateGradInput() 118 | 119 | local gw1 = torch.Tensor():typeAs(self.input1) 120 | local gw2 = torch.Tensor():typeAs(self.input2) 121 | gw1:resizeAs(self.input1):zero() 122 | gw2:resizeAs(self.input2):zero() 123 | 124 | self.w = self.w:expandAs(self.input1) 125 | self.w22 = self.w22:expandAs(self.input1) 126 | self.w32 = self.w32:expandAs(self.input1) 127 | self.w1 = self.w1:expandAs(self.input1) 128 | 129 | gw1:cmul(self.input2, self.w) -- bc * Yd 130 | 131 | -- q * a * (b*b*b*c) 132 | self.buffer:cmul(self.w, self.w22) -- (b*b*b*c) 133 | self.buffer:cmul(self.w1, self.buffer) -- a * ... 134 | self.buffer:cmul(self.input1, self.buffer) -- Yq * ... 135 | 136 | gw1:add(-self.buffer) -- bc * Yd - a * (b*b*b*c) * Yq 137 | 138 | -- for deriv_d 139 | -- q / (b * c) 140 | 141 | gw2:cmul(self.input1, self.w) 142 | -- d * a / (b*c*c*c) 143 | self.buffer:cmul(self.w, self.w32) 144 | self.buffer:cmul(self.w1, self.buffer) 145 | self.buffer:cmul(self.input2, self.buffer) 146 | 147 | -- q / (b*c) - d*a / (bbbc) 148 | gw2:add(-self.buffer) 149 | 150 | -- matrix_weightAdd 151 | -- for i = 1, batchsize*dimension 152 | local ngw1 = torch.Tensor(self.batch_size, self.dimension):zero():typeAs(gw1) 153 | local ngw2 = torch.Tensor(self.batch_size, self.dimension):zero():typeAs(gw2) 154 | 155 | ngw1:cmul(gw1:sub(1, self.batch_size), self.alpha_buffer:sub(1, self.batch_size):view(-1,1):expandAs(ngw1)) 156 | 157 | for i = 1, self.nTrail do 158 | ngw1:addcmul(-1, gw1:sub(i*self.batch_size+1, (i+1)*self.batch_size), 159 | self.alpha_buffer:sub(i*self.batch_size+1, (i+1)*self.batch_size):view(-1,1):expandAs(ngw1)) 160 | end 161 | 162 | 163 | ngw2:cmul(gw2:sub(1, self.batch_size), self.alpha_buffer:sub(1, self.batch_size):view(-1,1):expandAs(ngw2)) 164 | for i = 1, self.batch_size do 165 | for j = 1, self.nTrail do 166 | local col = self.D_inver_negtive_index_array[(j-1)*self.batch_size + i] 167 | local row = self.D_inver_negtive_value_array[(j-1)*self.batch_size + col] 168 | ngw2:sub(i,i):addcmul(-1, gw2:sub(j*self.batch_size + row, j*self.batch_size + row), 169 | self.alpha_buffer:sub(j*self.batch_size + row, j*self.batch_size + row):view(-1,1):expandAs(ngw2:sub(i,i))) 170 | end 171 | end 172 | 173 | self.gradInput = {-ngw1, -ngw2} 174 | 175 | return self.gradInput 176 | 177 | end 178 | -------------------------------------------------------------------------------- /CDSSM_sparse/WordHash.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | Implementation of CDSSM Model in Lua. The function name are the same 3 | of the orginial C# implementation. 4 | 5 | ]]-- 6 | --local iconv = require "iconv" 7 | local utils = require 'utils' 8 | local Batch = require 'Batch' 9 | local th_utils = require 'th_utils' 10 | local WordHash = {} 11 | WordHash.__index = WordHash 12 | function WordHash.GenLable(inFile, uniqueFile, outFile) 13 | 14 | local label = {} 15 | local unique = {} 16 | local f = assert(io.open(uniqueFile, "r")) 17 | 18 | for line in f:lines() do 19 | table.insert(unique, line) 20 | end 21 | count = 0 22 | local f = assert(io.open(inFile, "r")) 23 | for line in f:lines() do 24 | count = count + 1 25 | -- split the src and tgt by '\t' 26 | local sent = {} 27 | for value in line:gmatch("[^\t]+") do 28 | table.insert(sent, value) 29 | end 30 | 31 | for i = 1, #unique do 32 | if sent[2] == unique[i] then 33 | label[count] = i 34 | end 35 | end 36 | end 37 | 38 | torch.save('label.t7', label) 39 | 40 | end 41 | 42 | function WordHash.Pair2Voc(inFile, N, srcIdx, tgtIdx, outFile, STsame) 43 | -- get the LNg vocabulary for src and tgt. if STsame == 1, then src and tgt will get same vocabulary, 44 | -- otherwise, different vocab file 45 | 46 | local srcVoc, tgtVoc 47 | srcVoc = {} 48 | tgtVoc = {} 49 | 50 | --local decoder = iconv.new('utf8','utf8') 51 | local f = assert(io.open(inFile, "r")) 52 | 53 | for line in f:lines() do 54 | -- split the src and tgt by '\t' 55 | --line = decoder:iconv(line) 56 | local sent = {} 57 | for value in line:gmatch("[^\t]+") do 58 | table.insert(sent, value) 59 | end 60 | -- get the seq l3g for each sentence. 61 | local featStrFeq = utils.String2FeatStrSeq(sent[srcIdx], N, 1, 'l3g') 62 | -- get the seq l3g for each sentence. 63 | local featTgtFeq = utils.String2FeatStrSeq(sent[tgtIdx], N, 1, 'l3g') 64 | if STsame == 1 then 65 | for key, value in pairs(featStrFeq[1]) do 66 | if not srcVoc[key] then 67 | srcVoc[key] = 1 68 | else 69 | srcVoc[key] = srcVoc[key] + value 70 | end 71 | end 72 | for key, value in pairs(featTgtFeq[1]) do 73 | if not srcVoc[key] then 74 | srcVoc[key] = 1 75 | 76 | else 77 | srcVoc[key] = srcVoc[key] + value 78 | end 79 | end 80 | else 81 | for key, value in pairs(featStrFeq[1]) do 82 | if not srcVoc[key] then 83 | srcVoc[key] = 1 84 | else 85 | srcVoc[key] = srcVoc[key] + value 86 | end 87 | end 88 | 89 | for key, value in pairs(featTgtFeq[1]) do 90 | if not tgtVoc[key] then 91 | tgtVoc[key] = 1 92 | else 93 | tgtVoc[key] = tgtVoc[key] + value 94 | end 95 | end 96 | end 97 | end 98 | f:close() 99 | 100 | if STsame == 1 then 101 | tgtVoc = srcVoc 102 | end 103 | -- ranked by frequency 104 | srcVoc = th_utils.spairs(srcVoc, function(t,a,b) return t[b] 0 and i + win < #rgWfs+1 then 245 | for idx = i-win, i+win do 246 | for key, val in pairs(rgWfs[idx]) do 247 | table.insert(S, {nLine, count, key + vocSize*(idx-i+1), val}) 248 | end 249 | end 250 | count = count+1 251 | end 252 | end 253 | nLine = nLine + 1 254 | end 255 | -- insert the last one. 256 | table.insert(sparseMatrix, torch.IntTensor(S)) 257 | print('saving ' .. outFile) 258 | torch.save(outFile, sparseMatrix) 259 | 260 | end 261 | 262 | end 263 | 264 | return WordHash 265 | -------------------------------------------------------------------------------- /CDSSM_dense/WordHash.lua: -------------------------------------------------------------------------------- 1 | --[[ 2 | Implementation of CDSSM Model in Lua. The function name are the same 3 | of the orginial C# implementation. 4 | 5 | ]]-- 6 | local iconv = require "iconv" 7 | local utils = require 'utils' 8 | local Batch = require 'Batch' 9 | local th_utils = require 'th_utils' 10 | local WordHash = {} 11 | WordHash.__index = WordHash 12 | function WordHash.GenLable(inFile, uniqueFile, outFile) 13 | 14 | local label = {} 15 | local unique = {} 16 | local f = assert(io.open(uniqueFile, "r")) 17 | 18 | for line in f:lines() do 19 | table.insert(unique, line) 20 | end 21 | count = 0 22 | local f = assert(io.open(inFile, "r")) 23 | for line in f:lines() do 24 | count = count + 1 25 | -- split the src and tgt by '\t' 26 | local sent = {} 27 | for value in line:gmatch("[^\t]+") do 28 | table.insert(sent, value) 29 | end 30 | 31 | for i = 1, #unique do 32 | if sent[2] == unique[i] then 33 | label[count] = i 34 | end 35 | end 36 | end 37 | 38 | torch.save('label.t7', label) 39 | 40 | end 41 | 42 | function WordHash.Pair2Voc(inFile, N, srcIdx, tgtIdx, outFile, STsame) 43 | -- get the LNg vocabulary for src and tgt. if STsame == 1, then src and tgt will get same vocabulary, 44 | -- otherwise, different vocab file 45 | 46 | local srcVoc, tgtVoc 47 | srcVoc = {} 48 | tgtVoc = {} 49 | 50 | local decoder = iconv.new('utf8','utf8') 51 | local f = assert(io.open(inFile, "r")) 52 | 53 | for line in f:lines() do 54 | -- split the src and tgt by '\t' 55 | line = decoder:iconv(line) 56 | local sent = {} 57 | for value in line:gmatch("[^\t]+") do 58 | table.insert(sent, value) 59 | end 60 | -- get the seq l3g for each sentence. 61 | local featStrFeq = utils.String2FeatStrSeq(sent[srcIdx], N, 1, 'l3g') 62 | -- get the seq l3g for each sentence. 63 | local featTgtFeq = utils.String2FeatStrSeq(sent[tgtIdx], N, 1, 'l3g') 64 | if STsame == 1 then 65 | for key, value in pairs(featStrFeq[1]) do 66 | if not srcVoc[key] then 67 | srcVoc[key] = 1 68 | else 69 | srcVoc[key] = srcVoc[key] + value 70 | end 71 | end 72 | for key, value in pairs(featTgtFeq[1]) do 73 | if not srcVoc[key] then 74 | srcVoc[key] = 1 75 | 76 | else 77 | srcVoc[key] = srcVoc[key] + value 78 | end 79 | end 80 | else 81 | for key, value in pairs(featStrFeq[1]) do 82 | if not srcVoc[key] then 83 | srcVoc[key] = 1 84 | else 85 | srcVoc[key] = srcVoc[key] + value 86 | end 87 | end 88 | 89 | for key, value in pairs(featTgtFeq[1]) do 90 | if not tgtVoc[key] then 91 | tgtVoc[key] = 1 92 | else 93 | tgtVoc[key] = tgtVoc[key] + value 94 | end 95 | end 96 | end 97 | end 98 | f:close() 99 | 100 | if STsame == 1 then 101 | tgtVoc = srcVoc 102 | end 103 | -- ranked by frequency 104 | srcVoc = th_utils.spairs(srcVoc, function(t,a,b) return t[b] nMaxFeatureNumPerBatch then 233 | nMaxFeatureNumPerBatch = batch.ElementSize 234 | end 235 | 236 | if not tensor then 237 | tensor = batch:WriteSeqSample() 238 | else 239 | tensor = torch.cat(tensor, batch:WriteSeqSample()) 240 | 241 | end 242 | batch:Clear() 243 | end 244 | 245 | local featureDimension = batch:LoadSeqSample(rgWfs) 246 | if featureDimension > nMaxFeatureDimension then 247 | nMaxFeatureDimension = featureDimension 248 | end 249 | if batch.SegSize > nMaxSegmentSize then 250 | nMaxSegmentSize = batch.SegSize 251 | end 252 | end 253 | 254 | if batch.BatchSize > 0 then 255 | if batch.ElementSize > nMaxFeatureNumPerBatch then 256 | nMaxFeatureNumPerBatch = batch.ElementSize 257 | end 258 | if not tensor then 259 | tensor = batch:WriteSeqSample() 260 | else 261 | tensor = torch.cat(tensor, batch:WriteSeqSample()) 262 | end 263 | batch:Clear() 264 | end 265 | 266 | local last_tensor = torch.IntTensor(5) 267 | 268 | last_tensor[1] = nMaxFeatureDimension 269 | last_tensor[2] = nLine 270 | last_tensor[3] = nMaxSegmentSize 271 | last_tensor[4] = nMaxFeatureNumPerBatch 272 | last_tensor[5] = BatchSize 273 | 274 | tensor = torch.cat(tensor, last_tensor) 275 | print(tensor[1], tensor[2], tensor[3], tensor[4], tensor[5]) 276 | -- save to torch.t7 277 | print('saving ' .. outFile) 278 | torch.save(outFile, tensor) 279 | end 280 | 281 | return WordHash 282 | -------------------------------------------------------------------------------- /CDSSM_dense/DSSM_Train.lua: -------------------------------------------------------------------------------- 1 | 2 | local PairInputStream = require 'PairInputStream' 3 | local Normalizer = require 'Normalizer' 4 | local tds = require 'tds' 5 | require 'nngraph' 6 | require 'nn' 7 | require 'DSSM_MMI_Criterion' 8 | require 'optim' 9 | local th_utils = require 'th_utils' 10 | local DSSM_Predict = require 'DSSM_Predict' 11 | local DSSM_Train = {} 12 | DSSM_Train.__index = DSSM_Train 13 | 14 | function DSSM_Train.init() 15 | self = {} 16 | setmetatable(self, DSSM_Train) 17 | 18 | self.PairStream = nil 19 | self.PairValidStream = nil 20 | self.SrcNorm = nil 21 | self.TgtNorm = nil 22 | self.pairTrainFileIdx = 0 23 | 24 | self.model = nil 25 | return self 26 | end 27 | 28 | function DSSM_Train:reset_pointer() 29 | self.PairStream.qStream.dataFun.pointer = 1 30 | self.PairStream.dStream.dataFun.pointer = 1 31 | end 32 | function DSSM_Train:LoadTrainData(data_dir, qFileName, dFileName, nceProbDisFile, opt) 33 | -- doing shuffle and ... 34 | 35 | local qData, dData = self:LoadPairDataAtIdx(data_dir, qFileName, dFileName, nceProbDisFile, opt) 36 | 37 | print('Loading training doc query stream done') 38 | return qData, dData 39 | end 40 | function DSSM_Train:LoadPairDataAtIdx(data_dir, qFileName, dFileName, nceProbDisFile, opt) 41 | 42 | self.PairStream = PairInputStream.init() 43 | 44 | local qData, dData = self.PairStream:Load_Train_PairData(data_dir, qFileName, dFileName, nceProbDisFile, opt) 45 | 46 | self.ScrNorm = Normalizer.CreateFeatureNormalize(opt.Q_FeaNorm, self.PairStream.qStream.feature_size) 47 | self.TgtNorm = Normalizer.CreateFeatureNormalize(opt.D_FeaNorm, self.PairStream.dStream.feature_size) 48 | 49 | -- PairStream:initFeatureNorm() 50 | self.pairTrainFileIdx = 0 51 | 52 | return qData, dData 53 | end 54 | 55 | function DSSM_Train:ModelInit_FromConfig(opt) 56 | 57 | -- we first try to build fix structure model 58 | print('Initializing the Model...') 59 | local Q_feature_size = self.PairStream.qStream.feature_size 60 | local D_feature_size = self.PairStream.dStream.feature_size 61 | print(Q_feature_size) 62 | print(D_feature_size) 63 | 64 | local wind_size = 3 65 | 66 | local model 67 | if opt.convo == 0 then 68 | local Q_layer = nn.Sequential() 69 | Q_layer:add(nn.Linear(Q_feature_size, 1000)) 70 | Q_layer:add(nn.Tanh()) 71 | Q_layer:add(nn.Dropout(0.5)) 72 | Q_layer:add(nn.Linear(1000, 300)) 73 | Q_layer:add(nn.Tanh()) 74 | 75 | 76 | local D_layer = nn.Sequential() 77 | D_layer:add(nn.Linear(D_feature_size, 1000)) 78 | D_layer:add(nn.Tanh()) 79 | D_layer:add(nn.Dropout(0.5)) 80 | D_layer:add(nn.Linear(1000, 300)) 81 | D_layer:add(nn.Tanh()) 82 | 83 | model = nn.ParallelTable() 84 | model:add(Q_layer) 85 | model:add(D_layer) 86 | else 87 | 88 | local Q_layer = nn.Sequential() 89 | Q_layer:add(nn.TemporalConvolution(Q_feature_size * wind_size, 1000, 1)) 90 | Q_layer:add(nn.Tanh()) 91 | Q_layer:add(nn.TemporalMaxPooling(18)) 92 | Q_layer:add(nn.Reshape(1000, true)) 93 | Q_layer:add(nn.Linear(1000, 300)) 94 | 95 | local D_layer = nn.Sequential() 96 | D_layer:add(nn.TemporalConvolution(D_feature_size * wind_size, 1000, 1)) 97 | D_layer:add(nn.Tanh()) 98 | D_layer:add(nn.TemporalMaxPooling(18)) 99 | D_layer:add(nn.Reshape(1000, true)) 100 | D_layer:add(nn.Linear(1000, 300)) 101 | 102 | model = nn.ParallelTable() 103 | model:add(Q_layer) 104 | model:add(D_layer) 105 | end 106 | 107 | self.model = model 108 | if opt.mode == 'gpu' then 109 | self.model:cuda() 110 | end 111 | -- 112 | parameters,gradParameters = self.model:getParameters() 113 | --print(Q_feature_size) 114 | --print(parameters:size()) 115 | if opt.optim == 'sgd' then 116 | self.optimState = { 117 | learningRate = opt.learning_rate, 118 | weightDecay = opt.weight_decay, 119 | momentum = opt.momentum, 120 | learningRateDecay = 1e-7 121 | } 122 | self.optimMethod = optim.sgd 123 | end 124 | if opt.optim == 'rmsprop' then 125 | self.optimState = { 126 | learningRate = nil, 127 | alpha = nil, 128 | epsilon = nil 129 | } 130 | self.optimMethod = optim.rmsprop 131 | end 132 | 133 | end 134 | 135 | 136 | function DSSM_Train:Training(qData, dData, opt, epoch) 137 | 138 | if epoch > opt.lr_batch then 139 | self.optimState.learningRate = opt.lr 140 | end 141 | print('learning rate: '..self.optimState.learningRate) 142 | if opt.loadmodel ~= '' then 143 | self.model = torch.load(opt.loadmodel) 144 | end 145 | self.model:training() 146 | self.PairStream:Init_Batch() 147 | local trainingLoss = 0 148 | local total_err = 0 149 | 150 | for i = 1, self.PairStream.qStream.batch_num do 151 | local flag, qData, dData = self.PairStream:Next_Batch(qData, dData, srcNorm, tgtNorm, opt) 152 | 153 | xlua.progress(i, self.PairStream.qStream.batch_num) 154 | -- doing the forward process 155 | local qTensor, dTensor 156 | if opt.mode == 'gpu' then 157 | qTensor = qData.data_matrix:cuda() 158 | dTensor = dData.data_matrix:cuda() 159 | else 160 | qTensor = qData.data_matrix:double() 161 | dTensor = dData.data_matrix:double() 162 | end 163 | -- negtive samping index 164 | local batch_size = self.PairStream.qStream.dataFun.batch_size 165 | local feval = function(x) 166 | if x ~= parameters then 167 | parameters:copy(x) 168 | end 169 | gradParameters:zero() 170 | -- get the cosine similarity for all positive and negtive. 171 | local output = self.model:forward({qTensor, dTensor}) 172 | -- initialize the criterion function here, in order we can intilized the negative sampling 173 | -- differently for each batch 174 | 175 | if opt.objective == 'NCE' then 176 | -- load the doc distance for NCE Training. 177 | end 178 | --if opt.mode == 'gpu' then 179 | -- output = {output[1]:double(), output[2]:double()} 180 | --end 181 | if opt.objective == 'MMI' then 182 | self.criterion = nn.DSSM_MMI_Criterion(batch_size, opt.ntrial, opt.gamma, opt.batch_size) 183 | end 184 | if opt.mode == 'gpu' then 185 | self.criterion:cuda() 186 | end 187 | 188 | --print(output) 189 | -- forward the criterion 190 | local err = self.criterion:updateOutput(output) 191 | local do_df = self.criterion:updateGradInput() 192 | self.model:backward({qTensor, dTensor}, do_df) 193 | gradParameters:div(batch_size) 194 | total_err = total_err + err 195 | err = err / batch_size 196 | return err, gradParameters 197 | end 198 | self.optimMethod(feval, parameters, self.optimState) 199 | end 200 | print('training Loss: ' .. total_err / self.PairStream.qStream.batch_num) 201 | 202 | 203 | return self.model 204 | end 205 | 206 | 207 | function DSSM_Train:evaluate(qData, dData, opt, model) 208 | model:evaluate() 209 | 210 | -- load label 211 | self.PairStream:Init_Batch() 212 | local trainingLoss = 0 213 | local total_err = 0 214 | -- get the encoding of each answer. 215 | 216 | for i = 1, self.PairStream.qStream.batch_num do 217 | xlua.progress(i, self.PairStream.qStream.batch_num) 218 | local flag, qData, dData = self.PairStream:Next_Batch(qData, dData, srcNorm, tgtNorm, opt) 219 | 220 | -- doing the forward process 221 | local qTensor, dTensor 222 | if opt.mode == 'gpu' then 223 | qTensor = qData.data_matrix:cuda() 224 | dTensor = dData.data_matrix:cuda() 225 | else 226 | qTensor = qData.data_matrix:double() 227 | dTensor = dData.data_matrix:double() 228 | end 229 | -- negtive samping index 230 | local batch_size = self.PairStream.qStream.dataFun.batch_size 231 | -- get the cosine similarity for all positive and negtive. 232 | -- 233 | local output = model:forward({qTensor, dTensor}) 234 | 235 | if opt.objective == 'NCE' then 236 | -- load the doc distance for NCE Training. 237 | end 238 | if opt.objective == 'MMI' then 239 | self.criterion = nn.DSSM_MMI_Criterion(batch_size, opt.ntrial, opt.gamma, opt.batch_size) 240 | end 241 | 242 | -- forward the criterion 243 | local err = self.criterion:updateOutput(output) 244 | total_err = total_err + err 245 | err = err / batch_size 246 | end 247 | total_err = total_err / self.PairStream.qStream.batch_num 248 | print('validation loss: ' .. total_err) 249 | return total_err 250 | end 251 | 252 | 253 | function DSSM_Train:predict(qData, dData, opt, model, question_id, MC_ans) 254 | self.model = model 255 | self.model:evaluate() 256 | 257 | -- load label 258 | self.PairStream:Init_Batch() 259 | -- get the encoding of each answer. 260 | 261 | local result_table = {} 262 | count = 1 263 | for i = 1,self.PairStream.qStream.batch_num do 264 | xlua.progress(i, self.PairStream.qStream.batch_num) 265 | local flag, qData, dData = self.PairStream:Next_Batch(qData, dData, srcNorm, tgtNorm, opt) 266 | 267 | -- doing the forward process 268 | local qTensor, dTensor 269 | if opt.mode == 'gpu' then 270 | qTensor = qData.data_matrix:cuda() 271 | dTensor = dData.data_matrix:cuda() 272 | else 273 | qTensor = qData.data_matrix:double() 274 | dTensor = dData.data_matrix:double() 275 | end 276 | -- negtive samping index 277 | local batch_size = self.PairStream.qStream.dataFun.batch_size 278 | -- get the cosine similarity for all positive and negtive. 279 | -- 280 | local output = self.model:forward({qTensor, dTensor}) 281 | 282 | if opt.objective == 'NCE' then 283 | -- load the doc distance for NCE Training. 284 | end 285 | prediction = DSSM_Predict:updateOutput(output, 17) 286 | 287 | for j = 1, batch_size do 288 | local tmp = {} 289 | tmp['question_id'] = question_id[count] 290 | tmp['answer'] = MC_ans[count][prediction[j][1]] 291 | table.insert(result_table, tmp) 292 | count = count + 1 293 | end 294 | end 295 | return result_table 296 | end 297 | return DSSM_Train 298 | -------------------------------------------------------------------------------- /CDSSM_sparse/nn/SparseLinearBatch.c: -------------------------------------------------------------------------------- 1 | #ifndef TH_GENERIC_FILE 2 | #define TH_GENERIC_FILE "generic/SparseLinearBatch.c" 3 | #else 4 | 5 | #ifdef _OPENMP 6 | #include 7 | #endif 8 | 9 | static int nn_(checkInput3D)(THTensor* t) { 10 | return t->nDimension == 2 && t->size[1] == 3; 11 | } 12 | 13 | static int nn_(SparseLinearBatch_updateOutput)(lua_State *L) 14 | { 15 | long i; 16 | THTensor * input = luaT_checkudata(L, 2, torch_Tensor); 17 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 18 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 19 | THTensor * output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); 20 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 21 | //printf("%d\n", batchSize); 22 | 23 | long outDim = weight->size[0]; 24 | long inDim = weight->size[1]; 25 | 26 | luaL_argcheck(L, nn_(checkInput3D)(input), 2, "input size must be nnz x 3"); 27 | luaL_argcheck(L, nn_(checkSize2D)(output, (long)batchSize, outDim), 1, "output size wrong"); 28 | luaL_argcheck(L, nn_(checkSize1D)(bias, outDim), 1, "bias size wrong"); 29 | 30 | lua_getfield(L, 1, "shardBuffer"); 31 | if (!lua_isnil(L, -1)) { 32 | THTensor *buffer = 33 | luaT_getfieldcheckudata(L, 1, "shardBuffer", torch_Tensor); 34 | long num_shards = buffer->size[2]; 35 | luaL_argcheck(L, 36 | buffer->nDimension == 3 && buffer->size[1] == outDim && 37 | num_shards > 0 && buffer->size[0] == (long)batchSize, 38 | 1, 39 | "shardBuffer size wrong"); 40 | 41 | THTensor_(zero)(buffer); 42 | #pragma omp parallel for private(i) schedule(static) num_threads(num_shards) 43 | for (i = 0; i < input->size[0]; i++) { 44 | #ifdef _OPENMP 45 | int shardId = omp_get_thread_num(); 46 | #else 47 | int shardId = 1; 48 | #endif 49 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 50 | long offset = (long)(THTensor_(get2d)(input, i, 1)) - 1; 51 | 52 | if (offset >= 0 && offset < inDim) { 53 | THBlas_(axpy)(outDim, 54 | THTensor_(get2d)(input, i, 2), 55 | THTensor_(data)(weight) + offset * weight->stride[1], 56 | weight->stride[0], 57 | THTensor_(data)(buffer) + idx * buffer->stride[0] + shardId * buffer->stride[2], 58 | buffer->stride[1]); 59 | } else { 60 | luaL_error(L, "index out of bound. updateOutput: \ 61 | %ld not between 1 and %ld", offset + 1, inDim); 62 | } 63 | } 64 | 65 | THTensor_(sum)(output, buffer, 2); 66 | THTensor_(resize2d)(output, batchSize, outDim); 67 | for(i=0; istride[0], 74 | THTensor_(data)(output) + i * output->stride[0], 75 | output->stride[1]); 76 | } 77 | 78 | lua_getfield(L, 1, "output"); 79 | return 1; 80 | } 81 | 82 | for(i = 0; i < input->size[0]; i++) 83 | { 84 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 85 | long offset = (long)(THTensor_(get2d)(input, i, 1)) - 1; 86 | 87 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 88 | { 89 | real val = THTensor_(get2d)(input, i, 2); 90 | THBlas_(axpy)(output->size[1], 91 | val, 92 | THTensor_(data)(weight)+offset*weight->stride[1], 93 | weight->stride[0], 94 | THTensor_(data)(output)+idx*output->stride[0], 95 | output->stride[1]); 96 | } 97 | else { 98 | luaL_error(L, "index out of bound. updateOutput: \ 99 | %ld not between 1 and %ld", offset + 1, inDim); 100 | } 101 | } 102 | for(i=0; istride[0], 109 | THTensor_(data)(output) + i * output->stride[0], 110 | output->stride[1]); 111 | } 112 | lua_getfield(L, 1, "output"); 113 | return 1; 114 | } 115 | 116 | static int nn_(SparseLinearBatch_accGradParameters)(lua_State *L) 117 | { 118 | long i; 119 | THTensor * input = luaT_checkudata(L, 2, torch_Tensor); 120 | THTensor * gradOutput = luaT_checkudata(L, 3, torch_Tensor); 121 | real scale = luaL_optnumber(L, 4, 1); 122 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 123 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 124 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 125 | THTensor * gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); 126 | real weightDecay = luaT_getfieldchecknumber(L, 1, "weightDecay"); 127 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 128 | 129 | long nnz = input->size[0]; 130 | long outDim = weight->size[0]; 131 | long inDim = weight->size[1]; 132 | 133 | luaL_argcheck(L, nn_(checkInput3D)(input), 2, "input size must be nnz x 3"); 134 | luaL_argcheck( 135 | L, nn_(checkSize2D)(gradOutput, (long)batchSize, outDim), 3, "gradOutput size wrong"); 136 | luaL_argcheck( 137 | L, nn_(checkSize2D)(gradWeight, outDim, inDim), 1, "gradWeight size wrong"); 138 | luaL_argcheck( 139 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 140 | 141 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 142 | for(i = 0; i < nnz; i++) 143 | { 144 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 145 | long offset = (long)(THTensor_(get2d)(input, i, 1)) - 1; 146 | 147 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 148 | { 149 | real val = scale*THTensor_(get2d)(input, i, 2); 150 | 151 | THBlas_(axpy)(outDim, 152 | val, 153 | THTensor_(data)(gradOutput)+idx*gradOutput->stride[0], 154 | gradOutput->stride[1], 155 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], 156 | gradWeight->stride[0]); 157 | } 158 | else { 159 | luaL_error(L, "index out of bound. accGradParameters: \ 160 | %ld not between 1 and %ld", offset + 1, inDim); 161 | } 162 | } 163 | 164 | THTensor_(sum)(gradOutput, gradOutput, 0); 165 | THTensor_(resize1d)(gradOutput, outDim); 166 | 167 | THTensor_(cadd)(gradBias, gradBias, scale, gradOutput); 168 | 169 | if(weightDecay != 0) { 170 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 171 | for(i = 0; i < nnz; i++) { 172 | long offset = (long)(THTensor_(get2d)(input, i, 1)) - 1; 173 | THBlas_(axpy)(outDim, 174 | weightDecay, 175 | THTensor_(data)(weight) + offset*weight->stride[1], 176 | weight->stride[0], 177 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], 178 | gradWeight->stride[0]); 179 | } 180 | 181 | THTensor_(cadd)(gradBias, gradBias, weightDecay, bias); 182 | } 183 | 184 | return 0; 185 | } 186 | 187 | int nn_(SparseLinearBatch_updateParameters)(lua_State *L) 188 | { 189 | long i; 190 | real learningRate = luaL_checknumber(L, 2); 191 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 192 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 193 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 194 | THTensor * gradWeight = luaT_getfieldcheckudata( 195 | L, 1, "gradWeight", torch_Tensor); 196 | THTensor * lastInput = luaT_getfieldcheckudata( 197 | L, 1, "lastInput", torch_Tensor); 198 | 199 | long nnz = lastInput->size[0]; 200 | long outDim = weight->size[0]; 201 | long inDim = weight->size[1]; 202 | 203 | luaL_argcheck( 204 | L, nn_(checkSize2D)(gradWeight, outDim, inDim), 1, "gradWeight size wrong"); 205 | luaL_argcheck( 206 | L, nn_(checkSize1D)(bias, outDim), 1, "bias size wrong"); 207 | luaL_argcheck( 208 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 209 | 210 | THTensor_(cadd)(bias, bias, -learningRate, gradBias); 211 | 212 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 50000) 213 | for(i = 0; i < nnz; i++) 214 | { 215 | long offset = (long)(THTensor_(get2d)(lastInput, i, 1)) - 1; 216 | 217 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 218 | { 219 | real* pGradWeight = 220 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1]; 221 | THBlas_(axpy)(outDim, 222 | -learningRate, 223 | pGradWeight, 224 | gradWeight->stride[0], 225 | THTensor_(data)(weight)+offset*weight->stride[1], 226 | weight->stride[0]); 227 | } 228 | else { 229 | luaL_error(L, "index out of bound. updateParameters: \ 230 | %ld not between 1 and %ld", offset + 1, inDim); 231 | } 232 | } 233 | return 0; 234 | } 235 | 236 | int nn_(SparseLinearBatch_zeroGradParameters)(lua_State *L) 237 | { 238 | long i; 239 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 240 | THTensor * gradWeight = luaT_getfieldcheckudata( 241 | L, 1, "gradWeight", torch_Tensor); 242 | THTensor * lastInput = luaT_getfieldcheckudata( 243 | L, 1, "lastInput", torch_Tensor); 244 | 245 | long nnz = lastInput->size[0]; 246 | long outDim = gradWeight->size[0]; 247 | long inDim = gradWeight->size[1]; 248 | 249 | luaL_argcheck( 250 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 251 | 252 | THTensor_(zero)(gradBias); 253 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 50000) 254 | for(i = 0; i < nnz; i++) 255 | { 256 | long offset = (long)(THTensor_(get2d)(lastInput, i, 1)) - 1; 257 | 258 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 259 | { 260 | real* pGradWeight = 261 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1]; 262 | if(gradWeight->stride[0] == 1) { 263 | THVector_(fill)(pGradWeight, 0, outDim); 264 | } else { 265 | long j; 266 | for(j = 0; j < outDim; ++j) { 267 | pGradWeight[j * gradWeight->stride[0]] = 0; 268 | } 269 | } 270 | } 271 | else { 272 | luaL_error(L, "index out of bound. zeroGradParameters: \ 273 | %ld not between 1 and %ld", offset + 1, inDim); 274 | } 275 | } 276 | return 0; 277 | } 278 | 279 | static int nn_(SparseLinearBatch_updateGradInput)(lua_State *L) { 280 | THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 281 | THTensor *gradInput = 282 | luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor); 283 | THTensor *input = luaT_checkudata(L, 2, torch_Tensor); 284 | THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); 285 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 286 | 287 | long i; 288 | long nnz = input->size[0]; 289 | long outDim = weight->size[0]; 290 | long inDim = weight->size[1]; 291 | 292 | luaL_argcheck( 293 | L, nn_(checkInput3D)(input), 2, "input must be an nnz x 3 tensor"); 294 | luaL_argcheck( 295 | L, nn_(checkSize2D)(gradOutput, (long)batchSize, outDim), 3, "gradOutput size wrong"); 296 | 297 | THTensor_(resize2d)(gradInput, input->size[0], input->size[1]); 298 | 299 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 300 | for (i = 0; i < nnz; ++i) { 301 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 302 | long offset = (long)(THTensor_(get2d)(input, i, 1)) - 1; 303 | THTensor_(set2d)(gradInput, i, 0, idx + 1); 304 | THTensor_(set2d)(gradInput, i, 1, offset + 1); 305 | 306 | if (offset >= 0 && offset < inDim) { 307 | real val = 308 | THBlas_(dot)(outDim, 309 | THTensor_(data)(gradOutput) + idx * gradOutput->stride[0], 310 | gradOutput->stride[1], 311 | THTensor_(data)(weight) + offset * weight->stride[1], 312 | weight->stride[0]); 313 | THTensor_(set2d)(gradInput, i, 2, val); 314 | } else { 315 | luaL_error(L, "index out of bound. updateGradInput: \ 316 | %ld not between 1 and %ld", offset + 1, inDim); 317 | } 318 | } 319 | return 0; 320 | } 321 | 322 | static const struct luaL_Reg nn_(SparseLinearBatch__) [] = { 323 | {"SparseLinearBatch_updateOutput", nn_(SparseLinearBatch_updateOutput)}, 324 | {"SparseLinearBatch_accGradParameters", nn_(SparseLinearBatch_accGradParameters)}, 325 | {"SparseLinearBatch_updateParameters", nn_(SparseLinearBatch_updateParameters)}, 326 | {"SparseLinearBatch_zeroGradParameters", nn_(SparseLinearBatch_zeroGradParameters)}, 327 | {"SparseLinearBatch_updateGradInput", nn_(SparseLinearBatch_updateGradInput)}, 328 | {NULL, NULL} 329 | }; 330 | 331 | void nn_(SparseLinearBatch_init)(lua_State *L) 332 | { 333 | luaT_pushmetatable(L, torch_Tensor); 334 | luaT_registeratname(L, nn_(SparseLinearBatch__), "nn"); 335 | lua_pop(L,1); 336 | } 337 | 338 | #endif 339 | -------------------------------------------------------------------------------- /CDSSM_sparse/nn/CDSSMSparseConvolution.c: -------------------------------------------------------------------------------- 1 | #ifndef TH_GENERIC_FILE 2 | #define TH_GENERIC_FILE "generic/CDSSMSparseConvolution.c" 3 | #else 4 | 5 | #ifdef _OPENMP 6 | #include 7 | #endif 8 | 9 | static int nn_(checkInput4)(THTensor* t) { 10 | return t->nDimension == 2 && t->size[1] == 4; 11 | } 12 | 13 | static int nn_(checkSize3D)(THTensor* t, long size0, long size1, long size2) { 14 | return t->nDimension == 3 && t->size[0] == size0 && t->size[1] == size1 && t->size[2] == size2; 15 | } 16 | 17 | static int nn_(CDSSMSparseConvolution_updateOutput)(lua_State *L) 18 | { 19 | long i, j; 20 | THTensor * input = luaT_checkudata(L, 2, torch_Tensor); 21 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 22 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 23 | THTensor * output = luaT_getfieldcheckudata(L, 1, "output", torch_Tensor); 24 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 25 | int winSize = luaT_getfieldcheckint(L, 1, "winSize"); 26 | 27 | //printf("%d\n", batchSize); 28 | 29 | long outDim = weight->size[0]; 30 | long inDim = weight->size[1]; 31 | 32 | luaL_argcheck(L, nn_(checkInput4)(input), 2, "input size must be nnz x 4"); 33 | luaL_argcheck(L, nn_(checkSize3D)(output, (long)batchSize, (long)winSize, outDim), 1, "output size wrong"); 34 | luaL_argcheck(L, nn_(checkSize1D)(bias, outDim), 1, "bias size wrong"); 35 | 36 | lua_getfield(L, 1, "shardBuffer"); 37 | if (!lua_isnil(L, -1)) { 38 | THTensor *buffer = 39 | luaT_getfieldcheckudata(L, 1, "shardBuffer", torch_Tensor); 40 | long num_shards = buffer->size[3]; 41 | luaL_argcheck(L, 42 | buffer->nDimension == 4 && buffer->size[2] == outDim && 43 | num_shards > 0 && buffer->size[0] == (long)batchSize && 44 | buffer->size[1] == (long)winSize, 45 | 1, 46 | "shardBuffer size wrong"); 47 | 48 | THTensor_(zero)(buffer); 49 | #pragma omp parallel for private(i) schedule(static) num_threads(num_shards) 50 | for (i = 0; i < input->size[0]; i++) { 51 | #ifdef _OPENMP 52 | int shardId = omp_get_thread_num(); 53 | #else 54 | int shardId = 0; 55 | #endif 56 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 57 | long win = (long)(THTensor_(get2d)(input, i, 1)) - 1; 58 | long offset = (long)(THTensor_(get2d)(input, i, 2)) - 1; 59 | if (offset >= 0 && offset < inDim) { 60 | THBlas_(axpy)(outDim, 61 | THTensor_(get2d)(input, i, 3), 62 | THTensor_(data)(weight) + offset * weight->stride[1], 63 | weight->stride[0], 64 | THTensor_(data)(buffer) + idx * buffer->stride[0] + win * buffer->stride[1] + shardId * buffer->stride[3], 65 | buffer->stride[2]); 66 | } else { 67 | luaL_error(L, "index out of bound. updateOutput: \ 68 | %ld not between 1 and %ld", offset + 1, inDim); 69 | } 70 | } 71 | 72 | THTensor_(sum)(output, buffer, 3); 73 | THTensor_(resize3d)(output, batchSize, winSize, outDim); 74 | for(i=0; istride[0], 82 | THTensor_(data)(output) + i * output->stride[0] + j*output->stride[1], 83 | output->stride[2]); 84 | } 85 | lua_getfield(L, 1, "output"); 86 | return 1; 87 | } 88 | for(i = 0; i < input->size[0]; i++) 89 | { 90 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 91 | long win = (long)(THTensor_(get2d)(input, i, 1)) - 1; 92 | long offset = (long)(THTensor_(get2d)(input, i, 2)) - 1; 93 | 94 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 95 | { 96 | real val = THTensor_(get2d)(input, i, 3); 97 | THBlas_(axpy)(output->size[1], 98 | val, 99 | THTensor_(data)(weight)+offset*weight->stride[1], 100 | weight->stride[0], 101 | THTensor_(data)(output)+idx*output->stride[0] + win*output->stride[1], 102 | output->stride[2]); 103 | } 104 | else { 105 | luaL_error(L, "index out of bound. updateOutput: \ 106 | %ld not between 1 and %ld", offset + 1, inDim); 107 | } 108 | } 109 | for(i=0; istride[0], 117 | THTensor_(data)(output) + i * output->stride[0] + j*output->stride[1], 118 | output->stride[2]); 119 | } 120 | lua_getfield(L, 1, "output"); 121 | return 1; 122 | } 123 | 124 | static int nn_(CDSSMSparseConvolution_accGradParameters)(lua_State *L) 125 | { 126 | long i; 127 | THTensor * input = luaT_checkudata(L, 2, torch_Tensor); 128 | THTensor * gradOutput = luaT_checkudata(L, 3, torch_Tensor); 129 | real scale = luaL_optnumber(L, 4, 1); 130 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 131 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 132 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 133 | THTensor * gradWeight = luaT_getfieldcheckudata(L, 1, "gradWeight", torch_Tensor); 134 | real weightDecay = luaT_getfieldchecknumber(L, 1, "weightDecay"); 135 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 136 | int winSize = luaT_getfieldcheckint(L, 1, "winSize"); 137 | 138 | long nnz = input->size[0]; 139 | long outDim = weight->size[0]; 140 | long inDim = weight->size[1]; 141 | 142 | luaL_argcheck(L, nn_(checkInput4)(input), 2, "input size must be nnz x 4"); 143 | luaL_argcheck( 144 | L, nn_(checkSize3D)(gradOutput, (long)batchSize, (long)winSize, outDim), 3, "gradOutput size wrong"); 145 | luaL_argcheck( 146 | L, nn_(checkSize2D)(gradWeight, outDim, inDim), 1, "gradWeight size wrong"); 147 | luaL_argcheck( 148 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 149 | 150 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 151 | for(i = 0; i < nnz; i++) 152 | { 153 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 154 | long win = (long)(THTensor_(get2d)(input, i, 1)) - 1; 155 | long offset = (long)(THTensor_(get2d)(input, i, 2)) - 1; 156 | 157 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 158 | { 159 | real val = scale*THTensor_(get2d)(input, i, 3); 160 | 161 | THBlas_(axpy)(outDim, 162 | val, 163 | THTensor_(data)(gradOutput)+idx*gradOutput->stride[0]+win*gradOutput->stride[1], 164 | gradOutput->stride[2], 165 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], 166 | gradWeight->stride[0]); 167 | } 168 | else { 169 | luaL_error(L, "index out of bound. accGradParameters: \ 170 | %ld not between 1 and %ld", offset + 1, inDim); 171 | } 172 | } 173 | 174 | THTensor_(sum)(gradOutput, gradOutput, 0); 175 | THTensor_(sum)(gradOutput, gradOutput, 1); 176 | THTensor_(resize1d)(gradOutput, outDim); 177 | 178 | THTensor_(cadd)(gradBias, gradBias, scale, gradOutput); 179 | 180 | if(weightDecay != 0) { 181 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 182 | for(i = 0; i < nnz; i++) { 183 | long offset = (long)(THTensor_(get2d)(input, i, 2)) - 1; 184 | THBlas_(axpy)(outDim, 185 | weightDecay, 186 | THTensor_(data)(weight) + offset*weight->stride[1], 187 | weight->stride[0], 188 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1], 189 | gradWeight->stride[0]); 190 | } 191 | 192 | THTensor_(cadd)(gradBias, gradBias, weightDecay, bias); 193 | } 194 | 195 | return 0; 196 | } 197 | 198 | int nn_(CDSSMSparseConvolution_updateParameters)(lua_State *L) 199 | { 200 | long i; 201 | real learningRate = luaL_checknumber(L, 2); 202 | THTensor * weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 203 | THTensor * bias = luaT_getfieldcheckudata(L, 1, "bias", torch_Tensor); 204 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 205 | THTensor * gradWeight = luaT_getfieldcheckudata( 206 | L, 1, "gradWeight", torch_Tensor); 207 | THTensor * lastInput = luaT_getfieldcheckudata( 208 | L, 1, "lastInput", torch_Tensor); 209 | 210 | long nnz = lastInput->size[0]; 211 | long outDim = weight->size[0]; 212 | long inDim = weight->size[1]; 213 | 214 | luaL_argcheck( 215 | L, nn_(checkSize2D)(gradWeight, outDim, inDim), 1, "gradWeight size wrong"); 216 | luaL_argcheck( 217 | L, nn_(checkSize1D)(bias, outDim), 1, "bias size wrong"); 218 | luaL_argcheck( 219 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 220 | 221 | THTensor_(cadd)(bias, bias, -learningRate, gradBias); 222 | 223 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 50000) 224 | for(i = 0; i < nnz; i++) 225 | { 226 | long offset = (long)(THTensor_(get2d)(lastInput, i, 2)) - 1; 227 | 228 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 229 | { 230 | real* pGradWeight = 231 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1]; 232 | THBlas_(axpy)(outDim, 233 | -learningRate, 234 | pGradWeight, 235 | gradWeight->stride[0], 236 | THTensor_(data)(weight)+offset*weight->stride[1], 237 | weight->stride[0]); 238 | } 239 | else { 240 | luaL_error(L, "index out of bound. updateParameters: \ 241 | %ld not between 1 and %ld", offset + 1, inDim); 242 | } 243 | } 244 | return 0; 245 | } 246 | 247 | int nn_(CDSSMSparseConvolution_zeroGradParameters)(lua_State *L) 248 | { 249 | long i; 250 | THTensor * gradBias = luaT_getfieldcheckudata(L, 1, "gradBias", torch_Tensor); 251 | THTensor * gradWeight = luaT_getfieldcheckudata( 252 | L, 1, "gradWeight", torch_Tensor); 253 | THTensor * lastInput = luaT_getfieldcheckudata( 254 | L, 1, "lastInput", torch_Tensor); 255 | 256 | long nnz = lastInput->size[0]; 257 | long outDim = gradWeight->size[0]; 258 | long inDim = gradWeight->size[1]; 259 | 260 | luaL_argcheck( 261 | L, nn_(checkSize1D)(gradBias, outDim), 1, "gradBias size wrong"); 262 | 263 | THTensor_(zero)(gradBias); 264 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 50000) 265 | for(i = 0; i < nnz; i++) 266 | { 267 | long offset = (long)(THTensor_(get2d)(lastInput, i, 2)) - 1; 268 | 269 | if(offset >= 0 && offset < inDim) /* make sure indices are in bounds.. */ 270 | { 271 | real* pGradWeight = 272 | THTensor_(data)(gradWeight)+offset*gradWeight->stride[1]; 273 | if(gradWeight->stride[0] == 1) { 274 | THVector_(fill)(pGradWeight, 0, outDim); 275 | } else { 276 | long j; 277 | for(j = 0; j < outDim; ++j) { 278 | pGradWeight[j * gradWeight->stride[0]] = 0; 279 | } 280 | } 281 | } 282 | else { 283 | luaL_error(L, "index out of bound. zeroGradParameters: \ 284 | %ld not between 1 and %ld", offset + 1, inDim); 285 | } 286 | } 287 | return 0; 288 | } 289 | 290 | static int nn_(CDSSMSparseConvolution_updateGradInput)(lua_State *L) { 291 | THTensor *weight = luaT_getfieldcheckudata(L, 1, "weight", torch_Tensor); 292 | THTensor *gradInput = 293 | luaT_getfieldcheckudata(L, 1, "gradInput", torch_Tensor); 294 | THTensor *input = luaT_checkudata(L, 2, torch_Tensor); 295 | THTensor *gradOutput = luaT_checkudata(L, 3, torch_Tensor); 296 | int batchSize = luaT_getfieldcheckint(L, 1, "batchSize"); 297 | int winSize = luaT_getfieldcheckint(L, 1, "winSize"); 298 | 299 | long i; 300 | long nnz = input->size[0]; 301 | long outDim = weight->size[0]; 302 | long inDim = weight->size[1]; 303 | 304 | luaL_argcheck( 305 | L, nn_(checkInput4)(input), 2, "input must be an nnz x 4 tensor"); 306 | luaL_argcheck( 307 | L, nn_(checkSize3D)(gradOutput, (long)batchSize, (long)winSize, outDim), 3, "gradOutput size wrong"); 308 | 309 | THTensor_(resize2d)(gradInput, input->size[0], input->size[1]); 310 | 311 | #pragma omp parallel for private(i) schedule(static) if(outDim * nnz > 100000) 312 | for (i = 0; i < nnz; ++i) { 313 | long idx = (long)(THTensor_(get2d)(input, i, 0)) - 1; 314 | long win = (long)(THTensor_(get2d)(input, i, 1)) - 1; 315 | long offset = (long)(THTensor_(get2d)(input, i, 2)) - 1; 316 | THTensor_(set2d)(gradInput, i, 0, idx + 1); 317 | THTensor_(set2d)(gradInput, i, 1, win + 1); 318 | THTensor_(set2d)(gradInput, i, 2, offset + 1); 319 | 320 | if (offset >= 0 && offset < inDim) { 321 | real val = 322 | THBlas_(dot)(outDim, 323 | THTensor_(data)(gradOutput) + idx * gradOutput->stride[0] + win*gradOutput->stride[1], 324 | gradOutput->stride[2], 325 | THTensor_(data)(weight) + offset * weight->stride[1], 326 | weight->stride[0]); 327 | THTensor_(set2d)(gradInput, i, 3, val); 328 | } else { 329 | luaL_error(L, "index out of bound. updateGradInput: \ 330 | %ld not between 1 and %ld", offset + 1, inDim); 331 | } 332 | } 333 | return 0; 334 | } 335 | 336 | static const struct luaL_Reg nn_(CDSSMSparseConvolution__) [] = { 337 | {"CDSSMSparseConvolution_updateOutput", nn_(CDSSMSparseConvolution_updateOutput)}, 338 | {"CDSSMSparseConvolution_accGradParameters", nn_(CDSSMSparseConvolution_accGradParameters)}, 339 | {"CDSSMSparseConvolution_updateParameters", nn_(CDSSMSparseConvolution_updateParameters)}, 340 | {"CDSSMSparseConvolution_zeroGradParameters", nn_(CDSSMSparseConvolution_zeroGradParameters)}, 341 | {"CDSSMSparseConvolution_updateGradInput", nn_(CDSSMSparseConvolution_updateGradInput)}, 342 | {NULL, NULL} 343 | }; 344 | 345 | void nn_(CDSSMSparseConvolution_init)(lua_State *L) 346 | { 347 | luaT_pushmetatable(L, torch_Tensor); 348 | luaT_registeratname(L, nn_(CDSSMSparseConvolution__), "nn"); 349 | lua_pop(L,1); 350 | } 351 | 352 | #endif 353 | -------------------------------------------------------------------------------- /CDSSM_dense/data/l3g.txt: -------------------------------------------------------------------------------- 1 | #un 2 | _tr 3 | ri# 4 | _ty 5 | dri 6 | ip_ 7 | ars 8 | arr 9 | dre 10 | sw# 11 | jer 12 | lls 13 | dra 14 | pst 15 | #az 16 | di# 17 | asy 18 | lle 19 | ef# 20 | ass 21 | lla 22 | lln 23 | llo 24 | eit 25 | ast 26 | ask 27 | lli 28 | ash 29 | ein 30 | fl_ 31 | uen 32 | asc 33 | haf 34 | eor 35 | eop 36 | ase 37 | hai 38 | ued 39 | g_m 40 | hig 41 | eog 42 | ts# 43 | al# 44 | dit 45 | gla 46 | car 47 | #fr 48 | #fu 49 | din 50 | dio 51 | #fi 52 | fly 53 | tel 54 | cam 55 | #fl 56 | #fo 57 | can 58 | zon 59 | dol 60 | #fc 61 | usl 62 | #fe 63 | amu 64 | doi 65 | flo 66 | doe 67 | vir 68 | mps 69 | mpr 70 | fle 71 | er# 72 | amb 73 | cie 74 | bar 75 | ks# 76 | ame 77 | aly 78 | 166 79 | mpe 80 | ust 81 | als 82 | mpa 83 | mpo 84 | bit 85 | ski 86 | alu 87 | alt 88 | alk 89 | mpi 90 | ali 91 | lpt 92 | alm 93 | all 94 | alc 95 | alb 96 | ala 97 | bic 98 | 98# 99 | ale 100 | y_a 101 | #10 102 | #13 103 | y_f 104 | y_e 105 | al_ 106 | y_k 107 | t50 108 | y_i 109 | tsu 110 | new 111 | ds# 112 | y_m 113 | ds_ 114 | y_p 115 | osp 116 | may 117 | mid 118 | he# 119 | est 120 | ess 121 | ww1 122 | esp 123 | ww2 124 | is# 125 | iqu 126 | em# 127 | esi 128 | osc 129 | osb 130 | lit 131 | ese 132 | ier 133 | esc 134 | rks 135 | ap# 136 | es_ 137 | nst 138 | nsu 139 | 886 140 | #jr 141 | #ju 142 | lif 143 | #ji 144 | sce 145 | el# 146 | #ja 147 | gat 148 | hqu 149 | nsf 150 | #je 151 | dsh 152 | s_l 153 | gar 154 | ba# 155 | gal 156 | her 157 | s_c 158 | s_b 159 | nsh 160 | rs_ 161 | iss 162 | gag 163 | ems 164 | hey 165 | isu 166 | s_p 167 | gin 168 | lal 169 | aps 170 | app 171 | hee 172 | eme 173 | isa 174 | ema 175 | api 176 | aph 177 | fal 178 | isk 179 | isi 180 | elt 181 | els 182 | r_e 183 | isl 184 | elo 185 | ape 186 | cep 187 | ell 188 | cer 189 | eli 190 | fil 191 | you 192 | r_r 193 | eld 194 | cem 195 | ela 196 | r_w 197 | el_ 198 | cea 199 | fif 200 | cec 201 | #gr 202 | #gu 203 | ced 204 | #gi 205 | #gh 206 | #gm 207 | #gl 208 | #go 209 | #gn 210 | #ga 211 | ber 212 | #ge 213 | #gd 214 | _ye 215 | dle 216 | ida 217 | bea 218 | bec 219 | bed 220 | bee 221 | bef 222 | twi 223 | aid 224 | twe 225 | ep# 226 | le_ 227 | gch 228 | cs_ 229 | ymb 230 | bst 231 | mec 232 | sor 233 | mea 234 | yme 235 | men 236 | ce# 237 | mel 238 | ls# 239 | ymn 240 | rom 241 | roo 242 | w_g 243 | ymp 244 | ler 245 | #bo 246 | niq 247 | rof 248 | rog 249 | roy 250 | #6# 251 | erb 252 | ley 253 | wsp 254 | leg 255 | led 256 | own 257 | epr 258 | nia 259 | len 260 | epl 261 | lem 262 | epi 263 | unn 264 | v_e 265 | v_g 266 | fe_ 267 | v_a 268 | ws_ 269 | lsv 270 | 3th 271 | dus 272 | da# 273 | ugs 274 | #ki 275 | gel 276 | geo 277 | v_p 278 | #kl 279 | #ko 280 | #kn 281 | sma 282 | ged 283 | #ke 284 | ita 285 | eap 286 | ayo 287 | ean 288 | eam 289 | 188 290 | #lu 291 | ws# 292 | ro# 293 | rm_ 294 | wiz 295 | 181 296 | 180 297 | ps_ 298 | eac 299 | eiv 300 | fer 301 | rmi 302 | eir 303 | fen 304 | rmo 305 | day 306 | rma 307 | fed 308 | eau 309 | dau 310 | rme 311 | 86# 312 | dap 313 | feb 314 | rmy 315 | as# 316 | p_c 317 | sm_ 318 | shm 319 | rms 320 | dad 321 | p_r 322 | ea_ 323 | wou 324 | uke 325 | civ 326 | sm# 327 | vol 328 | snl 329 | k_u 330 | us# 331 | voc 332 | sci 333 | efe 334 | sca 335 | k_o 336 | asu 337 | rm# 338 | asi 339 | aso 340 | fe# 341 | 0_4 342 | gs# 343 | wo# 344 | pie 345 | zes 346 | oks 347 | uck 348 | uci 349 | _tl 350 | #fa 351 | usi 352 | rcr 353 | oki 354 | fs# 355 | poe 356 | sku 357 | equ 358 | pos 359 | sc# 360 | tch 361 | t_n 362 | t_o 363 | adj 364 | adi 365 | t_m 366 | ado 367 | de# 368 | t_h 369 | t_i 370 | t_g 371 | lte 372 | t_b 373 | rke 374 | #hu 375 | ti# 376 | eer 377 | #hi 378 | t_w 379 | dqu 380 | t_u 381 | koh 382 | #ho 383 | imi 384 | sba 385 | kon 386 | #he 387 | jor 388 | eeb 389 | o_s 390 | ike 391 | gto 392 | to# 393 | dev 394 | 665 395 | 964 396 | o_a 397 | der 398 | des 399 | dep 400 | aw# 401 | den 402 | del 403 | joi 404 | rbo 405 | n_t 406 | eso 407 | fit 408 | ded 409 | n_p 410 | dec 411 | nsw 412 | rby 413 | nk# 414 | fin 415 | esa 416 | n_d 417 | #oz 418 | nco 419 | n_b 420 | n_m 421 | _vo 422 | n_o 423 | um# 424 | n_i 425 | #ov 426 | i_p 427 | atr 428 | ckn 429 | #on 430 | toc 431 | ss_ 432 | i_i 433 | tow 434 | tou 435 | tos 436 | top 437 | h_p 438 | h_t 439 | svi 440 | #ka 441 | 08# 442 | umm 443 | rge 444 | h_a 445 | h_g 446 | ume 447 | hsa 448 | umb 449 | uma 450 | h_o 451 | nke 452 | not 453 | nou 454 | nif 455 | rvi 456 | ycl 457 | ums 458 | c_s 459 | #pu 460 | sui 461 | c_i 462 | ex# 463 | rve 464 | c_m 465 | #pl 466 | c_n 467 | tme 468 | c_c 469 | et# 470 | tma 471 | um_ 472 | c_g 473 | xce 474 | nok 475 | xch 476 | lke 477 | reh 478 | b_s 479 | omp 480 | tn# 481 | rum 482 | omy 483 | sch 484 | iod 485 | ome 486 | rue 487 | sco 488 | oma 489 | m_v 490 | iol 491 | omb 492 | omm 493 | m_s 494 | m_r 495 | omi 496 | eiz 497 | ak# 498 | rus 499 | #ts 500 | scr 501 | hoc 502 | m_c 503 | om_ 504 | una 505 | rch 506 | gma 507 | m_n 508 | m_m 509 | m_l 510 | nme 511 | rco 512 | sza 513 | uca 514 | hom 515 | l_t 516 | rce 517 | l_p 518 | v_n 519 | cop 520 | x20 521 | imp 522 | cot 523 | l_f 524 | cov 525 | l_d 526 | coh 527 | l_b 528 | l_a 529 | col 530 | _ad 531 | con 532 | _af 533 | ub# 534 | _ac 535 | _ab 536 | om# 537 | cog 538 | _an 539 | _ai 540 | imo 541 | _at 542 | g_t 543 | g_w 544 | _ap 545 | onv 546 | _ar 547 | g_r 548 | ons 549 | g_l 550 | g_o 551 | g_i 552 | g_h 553 | uys 554 | wea 555 | bon 556 | one 557 | g_f 558 | aga 559 | g_b 560 | hme 561 | oce 562 | mot 563 | mos 564 | mor 565 | wes 566 | mna 567 | swo 568 | f_r 569 | ubl 570 | ubj 571 | vek 572 | ubd 573 | f_f 574 | moo 575 | nne 576 | f_a 577 | f_c 578 | #8a 579 | uto 580 | lot 581 | ubu 582 | a_s 583 | ubs 584 | acr 585 | ek# 586 | ug# 587 | lid 588 | lon 589 | lm# 590 | ub_ 591 | lom 592 | ing 593 | jsc 594 | ink 595 | tbr 596 | got 597 | sim 598 | jil 599 | lm_ 600 | _un 601 | gol 602 | mme 603 | gne 604 | obe 605 | _us 606 | mmo 607 | goa 608 | oba 609 | gig 610 | obb 611 | obl 612 | pes 613 | lms 614 | tv# 615 | nbu 616 | pus 617 | _ne 618 | fol 619 | _na 620 | lme 621 | lma 622 | ugu 623 | ccu 624 | nba 625 | _nu 626 | ibs 627 | #el 628 | gmt 629 | tv_ 630 | ibu 631 | ny# 632 | 24# 633 | yto 634 | aoh 635 | _fe 636 | am# 637 | ytv 638 | ibe 639 | _fl 640 | _fo 641 | _fi 642 | e_z 643 | _fu 644 | ibl 645 | e_u 646 | e_r 647 | my# 648 | xte 649 | ogy 650 | e_l 651 | uem 652 | e_i 653 | xtr 654 | xts 655 | e_e 656 | nsp 657 | tc# 658 | nb# 659 | amy 660 | hbo 661 | wwe 662 | sti 663 | ams 664 | ove 665 | stl 666 | amp 667 | 04# 668 | ly# 669 | d_n 670 | ucl 671 | d_l 672 | ste 673 | ami 674 | uch 675 | amo 676 | d_i 677 | uce 678 | d_g 679 | ucc 680 | #ha 681 | ama 682 | d_c 683 | rt_ 684 | w2# 685 | amd 686 | str 687 | stu 688 | tud 689 | am_ 690 | rth 691 | rti 692 | uct 693 | nve 694 | rtn 695 | rto 696 | tco 697 | rta 698 | bbr 699 | rte 700 | nvo 701 | rty 702 | bbi 703 | st_ 704 | lyr 705 | lys 706 | do# 707 | rts 708 | bbe 709 | rtw 710 | sul 711 | sun 712 | iva 713 | 0_2 714 | 0_3 715 | hsc 716 | st# 717 | ouc 718 | ivi 719 | 08_ 720 | by# 721 | _ju 722 | mbe 723 | ic# 724 | mbo 725 | oca 726 | occ 727 | mbi 728 | oci 729 | lbu 730 | ock 731 | gym 732 | dow 733 | _bi 734 | rt# 735 | dop 736 | ay# 737 | eid 738 | dom 739 | ncy 740 | _br 741 | iz# 742 | nce 743 | iet 744 | _by 745 | anu 746 | ncl 747 | emy 748 | ani 749 | nch 750 | nci 751 | ics 752 | icr 753 | emp 754 | emo 755 | ict 756 | ien 757 | ane 758 | emi 759 | cvg 760 | #dy 761 | _ge 762 | emb 763 | ica 764 | het 765 | two 766 | zy# 767 | ice 768 | ick 769 | #du 770 | ici 771 | ich 772 | #di 773 | _gu 774 | huy 775 | pt# 776 | #do 777 | bid 778 | #da 779 | #dc 780 | #de 781 | dmi 782 | ic_ 783 | pm# 784 | gy# 785 | iza 786 | ize 787 | 2nd 788 | ows 789 | owr 790 | aby 791 | cei 792 | cus 793 | abr 794 | cel 795 | cen 796 | owa 797 | abu 798 | owl 799 | cul 800 | abo 801 | abl 802 | abb 803 | aba 804 | cco 805 | ce_ 806 | abe 807 | buy 808 | cce 809 | ow_ 810 | pto 811 | lve 812 | bei 813 | pta 814 | pte 815 | lvi 816 | 998 817 | ag# 818 | env 819 | ktr 820 | enn 821 | _2# 822 | ut# 823 | ubb 824 | ow# 825 | _kn 826 | _fa 827 | mum 828 | 19# 829 | cc# 830 | #e1 831 | me_ 832 | mci 833 | 199 834 | _fr 835 | agr 836 | 192 837 | hwe 838 | sol 839 | 196 840 | agn 841 | agm 842 | sym 843 | lco 844 | uti 845 | uth 846 | _0# 847 | ute 848 | ebt 849 | ger 850 | ebr 851 | ry_ 852 | gen 853 | ave 854 | sys 855 | ryi 856 | #ex 857 | uts 858 | gue 859 | tto 860 | ttl 861 | #ep 862 | #es 863 | #er 864 | #eu 865 | #et 866 | dby 867 | #ev 868 | tte 869 | #em 870 | sy_ 871 | ut_ 872 | #en 873 | dbo 874 | #ec 875 | y_o 876 | #ed 877 | #eg 878 | ful 879 | ttr 880 | az# 881 | ott 882 | gza 883 | ots 884 | acy 885 | nt_ 886 | es# 887 | ote 888 | #16 889 | ota 890 | act 891 | ack 892 | aci 893 | ach 894 | aco 895 | oti 896 | oth 897 | acl 898 | acc 899 | aca 900 | adg 901 | ry# 902 | ace 903 | ntr 904 | nts 905 | _bu 906 | azy 907 | osi 908 | nty 909 | ht# 910 | s_n 911 | nta 912 | #4# 913 | ntm 914 | ego 915 | nto 916 | nth 917 | nti 918 | its 919 | evo 920 | ip# 921 | fc# 922 | itt 923 | evi 924 | 8am 925 | _he 926 | nso 927 | _ha 928 | ot# 929 | #is 930 | _ho 931 | _hi 932 | ith 933 | ito 934 | sdi 935 | itl 936 | #im 937 | #il 938 | tua 939 | #in 940 | nt2 941 | #ic 942 | #id 943 | dvi 944 | #if 945 | os# 946 | dve 947 | vot 948 | 0s# 949 | nt# 950 | yno 951 | rdi 952 | aws 953 | ez# 954 | dc# 955 | bt_ 956 | cts 957 | ctu 958 | awo 959 | awn 960 | cti 961 | ecu 962 | ect 963 | awa 964 | ctl 965 | cto 966 | eco 967 | een 968 | cte 969 | eci 970 | ech 971 | pon 972 | ece 973 | eca 974 | ct_ 975 | op# 976 | rro 977 | pyr 978 | dcu 979 | lt# 980 | ezo 981 | num 982 | kos 983 | kor 984 | mtv 985 | uy# 986 | dca 987 | lt_ 988 | bsu 989 | yri 990 | bsi 991 | ew# 992 | zar 993 | ct# 994 | xt_ 995 | ltu 996 | ty# 997 | _vi 998 | lty 999 | job 1000 | v_s 1001 | uyn 1002 | #5# 1003 | lti 1004 | 819 1005 | ft_ 1006 | izo 1007 | uoi 1008 | ty_ 1009 | ewm 1010 | ewi 1011 | zai 1012 | #ny 1013 | gth 1014 | at# 1015 | ew_ 1016 | tyl 1017 | zat 1018 | p_i 1019 | #nu 1020 | #ni 1021 | too 1022 | mpu 1023 | #no 1024 | #ns 1025 | #na 1026 | _ta 1027 | #nb 1028 | #ne 1029 | #nf 1030 | tyt 1031 | fti 1032 | lpe 1033 | typ 1034 | rra 1035 | fte 1036 | lpa 1037 | aty 1038 | ny_ 1039 | ats 1040 | oye 1041 | oop 1042 | oyf 1043 | oya 1044 | atu 1045 | att 1046 | req 1047 | oym 1048 | ati 1049 | ath 1050 | ato 1051 | atl 1052 | atc 1053 | ata 1054 | ate 1055 | #ut 1056 | at_ 1057 | hy# 1058 | yna 1059 | nyc 1060 | ho# 1061 | pra 1062 | ils 1063 | ft# 1064 | non 1065 | oy# 1066 | #ru 1067 | #ri 1068 | #ro 1069 | #ra 1070 | #rc 1071 | oo# 1072 | #re 1073 | wen 1074 | 0_0 1075 | iom 1076 | hor 1077 | hos 1078 | e3# 1079 | gm# 1080 | udi 1081 | how 1082 | ews 1083 | yah 1084 | yal 1085 | sac 1086 | t_v 1087 | udy 1088 | by_ 1089 | hod 1090 | etw 1091 | hon 1092 | xac 1093 | hol 1094 | ets 1095 | etr 1096 | rab 1097 | eto 1098 | uro 1099 | 23# 1100 | cyc 1101 | xan 1102 | eti 1103 | eth 1104 | urg 1105 | xas 1106 | o_c 1107 | urd 1108 | etb 1109 | eta 1110 | cy_ 1111 | et_ 1112 | #op 1113 | com 1114 | #or 1115 | #ou 1116 | #ot 1117 | #ow 1118 | ury 1119 | sai 1120 | #oh 1121 | sak 1122 | dtr 1123 | sam 1124 | sal 1125 | box 1126 | san 1127 | urp 1128 | 13# 1129 | tro 1130 | sab 1131 | sat 1132 | #od 1133 | 01# 1134 | #of 1135 | say 1136 | bos 1137 | ode 1138 | bou 1139 | ly_ 1140 | sas 1141 | odl 1142 | sau 1143 | jr# 1144 | 013 1145 | kna 1146 | rah 1147 | rai 1148 | ndt 1149 | rak 1150 | ral 1151 | ram 1152 | ran 1153 | rao 1154 | cy# 1155 | rac 1156 | rad 1157 | od_ 1158 | raf 1159 | #it 1160 | ors 1161 | ray 1162 | raz 1163 | nde 1164 | 09# 1165 | ory 1166 | py# 1167 | rap 1168 | ndb 1169 | ir# 1170 | ras 1171 | rat 1172 | #s# 1173 | rav 1174 | ora 1175 | llv 1176 | ids 1177 | mom 1178 | lyn 1179 | sa# 1180 | lym 1181 | #2p 1182 | exu 1183 | ext 1184 | xua 1185 | exp 1186 | od# 1187 | nre 1188 | gyp 1189 | idi 1190 | #2n 1191 | exh 1192 | #sy 1193 | #sz 1194 | bo# 1195 | exa 1196 | ra# 1197 | #sp 1198 | #su 1199 | #st 1200 | #sw 1201 | #si 1202 | #sh 1203 | #sk 1204 | #sm 1205 | #sl 1206 | #so 1207 | #sn 1208 | sua 1209 | c_a 1210 | suc 1211 | sub 1212 | #se 1213 | wal 1214 | suf 1215 | or# 1216 | yne 1217 | gog 1218 | wag 1219 | way 1220 | sup 1221 | sus 1222 | sur 1223 | rui 1224 | yfr 1225 | ays 1226 | wat 1227 | md# 1228 | wav 1229 | rua 1230 | rub 1231 | was 1232 | war 1233 | val 1234 | vam 1235 | ayi 1236 | b_f 1237 | ayn 1238 | fla 1239 | ayl 1240 | vad 1241 | m_t 1242 | teo 1243 | aye 1244 | vac 1245 | vat 1246 | ay_ 1247 | #ly 1248 | bdi 1249 | cru 1250 | ld# 1251 | m_a 1252 | vas 1253 | tex 1254 | 64# 1255 | #li 1256 | sno 1257 | sna 1258 | #ve 1259 | ija 1260 | #lo 1261 | #la 1262 | fy# 1263 | #le 1264 | 00s 1265 | oz# 1266 | rn_ 1267 | bru 1268 | pad 1269 | bri 1270 | l_n 1271 | pai 1272 | pan 1273 | #mt 1274 | l_j 1275 | rno 1276 | ey# 1277 | rfo 1278 | fl# 1279 | rfe 1280 | g_u 1281 | ova 1282 | par 1283 | pas 1284 | pap 1285 | pat 1286 | lde 1287 | ovi 1288 | ldc 1289 | 8_p 1290 | _de 1291 | 65# 1292 | dy# 1293 | cum 1294 | dy_ 1295 | _do 1296 | f_y 1297 | _di 1298 | wip 1299 | gdp 1300 | kai 1301 | f_p 1302 | eyt 1303 | gdo 1304 | irc 1305 | urk 1306 | _dr 1307 | he_ 1308 | jay 1309 | eyh 1310 | zoo 1311 | #py 1312 | ief 1313 | nt5 1314 | ta# 1315 | ey_ 1316 | jac 1317 | #pr 1318 | qui 1319 | quo 1320 | unk 1321 | ive 1322 | #ph 1323 | jai 1324 | qua 1325 | jan 1326 | #po 1327 | dyn 1328 | #pa 1329 | dyl 1330 | ual 1331 | uak 1332 | #pe 1333 | gri 1334 | uag 1335 | elm 1336 | uad 1337 | gra 1338 | ta_ 1339 | ka# 1340 | hel 1341 | pun 1342 | adv 1343 | adu 1344 | pub 1345 | uar 1346 | uat 1347 | wnt 1348 | tan 1349 | tal 1350 | tam 1351 | wns 1352 | tak 1353 | ada 1354 | tai 1355 | taf 1356 | tag 1357 | don 1358 | pur 1359 | tab 1360 | fre 1361 | tad 1362 | put 1363 | kup 1364 | dog 1365 | ary 1366 | tay 1367 | wn_ 1368 | taw 1369 | tat 1370 | tau 1371 | tar 1372 | tas 1373 | tap 1374 | tax 1375 | oat 1376 | ari 1377 | oar 1378 | kud 1379 | bes 1380 | bet 1381 | arm 1382 | arl 1383 | arc 1384 | ara 1385 | ia# 1386 | oad 1387 | bra 1388 | are 1389 | jur 1390 | oac 1391 | oam 1392 | oal 1393 | oan 1394 | nat 1395 | nau 1396 | pme 1397 | nar 1398 | jud 1399 | #ab 1400 | nas 1401 | nad 1402 | #19 1403 | #18 1404 | nag 1405 | rga 1406 | #i# 1407 | nac 1408 | nal 1409 | nam 1410 | nan 1411 | e_w 1412 | iar 1413 | er2 1414 | iat 1415 | rgu 1416 | _ed 1417 | _ef 1418 | tun 1419 | iag 1420 | un# 1421 | _em 1422 | _el 1423 | _en 1424 | edu 1425 | eds 1426 | ian 1427 | iam 1428 | ial 1429 | har 1430 | has 1431 | hap 1432 | _ep 1433 | hav 1434 | hat 1435 | edg 1436 | #d# 1437 | ede 1438 | _ex 1439 | ia_ 1440 | tut 1441 | tur 1442 | hab 1443 | tup 1444 | d_f 1445 | out 1446 | hag 1447 | had 1448 | na# 1449 | oup 1450 | ous 1451 | our 1452 | #qu 1453 | han 1454 | hal 1455 | ham 1456 | eru 1457 | iu# 1458 | ers 1459 | sko 1460 | cri 1461 | cap 1462 | cas 1463 | ske 1464 | oul 1465 | cat 1466 | oun 1467 | uff 1468 | cah 1469 | ere 1470 | ma# 1471 | cal 1472 | rk_ 1473 | era 1474 | fro 1475 | nur 1476 | nus 1477 | cab 1478 | cad 1479 | vg_ 1480 | fra 1481 | dru 1482 | nuf 1483 | cau 1484 | nuc 1485 | bas 1486 | bat 1487 | arv 1488 | bal 1489 | bam 1490 | ban 1491 | tne 1492 | bab 1493 | un_ 1494 | avi 1495 | roe 1496 | _id 1497 | mat 1498 | mas 1499 | mar 1500 | la# 1501 | map 1502 | ou# 1503 | r2# 1504 | _in 1505 | #eq 1506 | ig# 1507 | mag 1508 | ium 1509 | oga 1510 | mad 1511 | hus 1512 | mab 1513 | if# 1514 | 0_1 1515 | off 1516 | ca# 1517 | mal 1518 | mak 1519 | maj 1520 | mai 1521 | ong 1522 | onf 1523 | law 1524 | lat 1525 | hub 1526 | lar 1527 | las 1528 | lap 1529 | onl 1530 | ono 1531 | hud 1532 | rk# 1533 | man 1534 | lay 1535 | of_ 1536 | lag 1537 | hum 1538 | lav 1539 | lab 1540 | lac 1541 | nfe 1542 | lan 1543 | mac 1544 | vg# 1545 | cup 1546 | lak 1547 | nfl 1548 | lai 1549 | bad 1550 | nfo 1551 | nfi 1552 | #sa 1553 | gaz 1554 | #sc 1555 | ev# 1556 | k11 1557 | nni 1558 | gas 1559 | ins 1560 | gam 1561 | yho 1562 | gan 1563 | inv 1564 | int 1565 | ifa 1566 | gad 1567 | wom 1568 | iff 1569 | of# 1570 | bur 1571 | bus 1572 | inc 1573 | ifi 1574 | ina 1575 | ifo 1576 | _go 1577 | bui 1578 | ine 1579 | bul 1580 | fat 1581 | fau 1582 | ini 1583 | faw 1584 | inl 1585 | _gr 1586 | fam 1587 | fan 1588 | gac 1589 | mus 1590 | mur 1591 | mup 1592 | in_ 1593 | fac 1594 | gai 1595 | shi 1596 | fas 1597 | muc 1598 | aar 1599 | sho 1600 | sha 1601 | lam 1602 | ype 1603 | she 1604 | ugh 1605 | lut 1606 | nju 1607 | ga# 1608 | ypt 1609 | lud 1610 | lue 1611 | lub 1612 | luc 1613 | fad 1614 | tld 1615 | rho 1616 | lum 1617 | dem 1618 | luk 1619 | tba 1620 | guy 1621 | xpl 1622 | #dr 1623 | spl 1624 | gus 1625 | gur 1626 | gum 1627 | gul 1628 | guo 1629 | olv 1630 | gui 1631 | obs 1632 | uk# 1633 | ogt 1634 | #df 1635 | _no 1636 | spo 1637 | ogr 1638 | ole 1639 | ng_ 1640 | olf 1641 | spe 1642 | ogz 1643 | ea# 1644 | sky 1645 | olo 1646 | oln 1647 | oli 1648 | obj 1649 | fun 1650 | ogn 1651 | spr 1652 | spu 1653 | lfr 1654 | ngt 1655 | ngu 1656 | ngr 1657 | ngs 1658 | rpo 1659 | 003 1660 | og_ 1661 | nle 1662 | #wo 1663 | f_e 1664 | aus 1665 | ngd 1666 | nge 1667 | ywa 1668 | nga 1669 | aut 1670 | ngc 1671 | ngl 1672 | ibr 1673 | who 1674 | ngh 1675 | eav 1676 | aul 1677 | eat 1678 | eas 1679 | ear 1680 | aug 1681 | wha 1682 | igu 1683 | eal 1684 | eak 1685 | tla 1686 | eag 1687 | why 1688 | ilb 1689 | ead 1690 | ob# 1691 | rey 1692 | ile 1693 | igg 1694 | xt# 1695 | ige 1696 | ilh 1697 | igi 1698 | igh 1699 | igo 1700 | ign 1701 | dat 1702 | ffy 1703 | dar 1704 | an# 1705 | dan 1706 | dal 1707 | dam 1708 | bje 1709 | #tn 1710 | gn_ 1711 | _re 1712 | ffi 1713 | _ra 1714 | ffe 1715 | nl# 1716 | ng# 1717 | _ro 1718 | ox# 1719 | ufa 1720 | _ru 1721 | daw 1722 | _im 1723 | oke 1724 | oka 1725 | r_i 1726 | aft 1727 | djo 1728 | dwo 1729 | mb# 1730 | afl 1731 | ans 1732 | anr 1733 | hs# 1734 | aff 1735 | r_g 1736 | phi 1737 | ok_ 1738 | pho 1739 | ano 1740 | ann 1741 | azi 1742 | anl 1743 | anc 1744 | ana 1745 | 922 1746 | phe 1747 | gn# 1748 | phy 1749 | wel 1750 | dna 1751 | nki 1752 | sue 1753 | eur 1754 | eup 1755 | eun 1756 | eum 1757 | blo 1758 | stm 1759 | sto 1760 | ofe 1761 | ble 1762 | stc 1763 | _of 1764 | sty 1765 | _ob 1766 | ok# 1767 | ofi 1768 | ll_ 1769 | iki 1770 | nut 1771 | sts 1772 | _ow 1773 | mba 1774 | _op 1775 | dun 1776 | _or 1777 | #kt 1778 | nue 1779 | dub 1780 | duc 1781 | nfa 1782 | th# 1783 | en# 1784 | ppo 1785 | ppl 1786 | ppi 1787 | rtr 1788 | ifu 1789 | lbo 1790 | ify 1791 | cks 1792 | ajo 1793 | we# 1794 | efs 1795 | ife 1796 | efo 1797 | cke 1798 | hop 1799 | eff 1800 | ggi 1801 | up# 1802 | efa 1803 | ck_ 1804 | tho 1805 | thl 1806 | enr 1807 | thi 1808 | the 1809 | eni 1810 | enh 1811 | tha 1812 | pez 1813 | end 1814 | enb 1815 | dfh 1816 | thw 1817 | _ma 1818 | _se 1819 | thr 1820 | ths 1821 | _sa 1822 | _sc 1823 | ck1 1824 | _so 1825 | fli 1826 | em_ 1827 | _su 1828 | _st 1829 | _sq 1830 | _sp 1831 | cui 1832 | ck# 1833 | agu 1834 | cob 1835 | agi 1836 | oho 1837 | abi 1838 | ohi 1839 | alp 1840 | xhi 1841 | upa 1842 | e1# 1843 | age 1844 | ag_ 1845 | upt 1846 | ab_ 1847 | upr 1848 | lf# 1849 | upp 1850 | nha 1851 | #ct 1852 | #ci 1853 | _s# 1854 | 22# 1855 | #my 1856 | _le 1857 | axe 1858 | _la 1859 | opt 1860 | #os 1861 | oh# 1862 | _lo 1863 | #mu 1864 | _li 1865 | #mi 1866 | dju 1867 | #ol 1868 | #mo 1869 | ope 1870 | #ma 1871 | #oc 1872 | #ob 1873 | pti 1874 | #me 1875 | opm 1876 | dje 1877 | opi 1878 | oph 1879 | kfa 1880 | 123 1881 | nh# 1882 | ff_ 1883 | sh# 1884 | aks 1885 | 03# 1886 | 02# 1887 | aku 1888 | chr 1889 | aki 1890 | egy 1891 | chi 1892 | egu 1893 | 30# 1894 | egr 1895 | cho 1896 | chn 1897 | cha 1898 | ebs 1899 | che 1900 | egi 1901 | ak_ 1902 | ege 1903 | _be 1904 | ega 1905 | ch_ 1906 | _a_ 1907 | _3# 1908 | eba 1909 | ipe 1910 | yle 1911 | ipi 1912 | fou 1913 | yli 1914 | ipl 1915 | whi 1916 | _aw 1917 | _pe 1918 | dge 1919 | 200 1920 | _as 1921 | _pa 1922 | afr 1923 | _pl 1924 | _po 1925 | exi 1926 | _pi 1927 | dlo 1928 | _pu 1929 | exc 1930 | utc 1931 | _pp 1932 | o_p 1933 | _pr 1934 | ch# 1935 | 102 1936 | ibi 1937 | yde 1938 | aub 1939 | tti 1940 | owe 1941 | sla 1942 | lhe 1943 | tta 1944 | wme 1945 | sly 1946 | ff# 1947 | 13t 1948 | eks 1949 | ze_ 1950 | ght 1951 | n_a 1952 | rli 1953 | zed 1954 | ghs 1955 | n_h 1956 | rlo 1957 | #by 1958 | eke 1959 | #cl 1960 | ghe 1961 | #ca 1962 | it# 1963 | #cc 1964 | gha 1965 | #ce 1966 | #br 1967 | #bu 1968 | vma 1969 | otb 1970 | #bi 1971 | otl 1972 | nda 1973 | pha 1974 | #bl 1975 | fhu 1976 | #ba 1977 | ndo 1978 | #30 1979 | #be 1980 | nvi 1981 | mpl 1982 | yc# 1983 | rd_ 1984 | nte 1985 | lpr 1986 | #yo 1987 | lk# 1988 | #ya 1989 | 50_ 1990 | ntl 1991 | #ye 1992 | ahs 1993 | rdo 1994 | ake 1995 | ov# 1996 | oug 1997 | rde 1998 | itu 1999 | aho 2000 | ity 2001 | aha 2002 | _1# 2003 | rds 2004 | _4# 2005 | eou 2006 | ite 2007 | iti 2008 | hts 2009 | dfo 2010 | _ev 2011 | _eq 2012 | it_ 2013 | hte 2014 | dos 2015 | zil 2016 | zin 2017 | dur 2018 | zip 2019 | rd# 2020 | th_ 2021 | #vi 2022 | _sh 2023 | #vm 2024 | #vo 2025 | #va 2026 | thn 2027 | m_w 2028 | mt# 2029 | _du 2030 | _sy 2031 | yea 2032 | apt 2033 | yee 2034 | yed 2035 | apo 2036 | #gy 2037 | apa 2038 | yer 2039 | thq 2040 | m_h 2041 | #js 2042 | #cy 2043 | xes 2044 | _it 2045 | eha 2046 | 2pm 2047 | l_m 2048 | #cr 2049 | #cu 2050 | l_i 2051 | #cv 2052 | sei 2053 | #ch 2054 | lfe 2055 | sel 2056 | #co 2057 | sen 2058 | sea 2059 | sec 2060 | seb 2061 | see 2062 | sed 2063 | g_p 2064 | sey 2065 | sex 2066 | dhe 2067 | akf 2068 | ul# 2069 | re_ 2070 | ses 2071 | ser 2072 | seu 2073 | set 2074 | sew 2075 | #zi 2076 | rei 2077 | rek 2078 | rel 2079 | rem 2080 | #zo 2081 | rea 2082 | rec 2083 | red 2084 | ree 2085 | ref 2086 | reg 2087 | #e3 2088 | se_ 2089 | rep 2090 | dp_ 2091 | rer 2092 | res 2093 | ret 2094 | #20 2095 | rev 2096 | rew 2097 | yin 2098 | #24 2099 | ept 2100 | ud# 2101 | se# 2102 | #2# 2103 | _mu 2104 | rok 2105 | ulf 2106 | uld 2107 | _we 2108 | ula 2109 | chy 2110 | _wa 2111 | imu 2112 | xis 2113 | ult 2114 | chu 2115 | ryd 2116 | re# 2117 | #wr 2118 | #ww 2119 | #wi 2120 | #wh 2121 | y_c 2122 | y_b 2123 | tle 2124 | sio 2125 | sin 2126 | #wa 2127 | y_d 2128 | sic 2129 | sib 2130 | #we 2131 | sid 2132 | sig 2133 | udg 2134 | #ea 2135 | ude 2136 | y_s 2137 | y_r 2138 | wee 2139 | wed 2140 | y_v 2141 | y_u 2142 | y_t 2143 | sis 2144 | sir 2145 | web 2146 | sit 2147 | whe 2148 | ve_ 2149 | ril 2150 | rim 2151 | rin 2152 | rio 2153 | ria 2154 | rib 2155 | ric 2156 | wer 2157 | vel 2158 | vem 2159 | ven 2160 | riz 2161 | oll 2162 | ved 2163 | pe# 2164 | ud_ 2165 | rip 2166 | ris 2167 | rit 2168 | riu 2169 | riv 2170 | s_i 2171 | s_h 2172 | s_m 2173 | odu 2174 | s_o 2175 | nly 2176 | s_a 2177 | ver 2178 | ves 2179 | ods 2180 | s_e 2181 | s_d 2182 | s_g 2183 | s_f 2184 | nd_ 2185 | ody 2186 | 10f 2187 | id# 2188 | nli 2189 | oda 2190 | s_s 2191 | s_r 2192 | s_t 2193 | s_w 2194 | s_v 2195 | pe_ 2196 | odi 2197 | r_l 2198 | r_m 2199 | ndu 2200 | r_o 2201 | r_a 2202 | r_b 2203 | r_c 2204 | r_d 2205 | ila 2206 | r_f 2207 | pen 2208 | peo 2209 | ol# 2210 | ild 2211 | pec 2212 | ili 2213 | pea 2214 | ilo 2215 | r_p 2216 | ped 2217 | ve# 2218 | r_s 2219 | r_t 2220 | r_v 2221 | wil 2222 | 201 2223 | win 2224 | ndi 2225 | per 2226 | wik 2227 | ett 2228 | mma 2229 | pet 2230 | eve 2231 | kes 2232 | ker 2233 | ynh 2234 | ket 2235 | key 2236 | ete 2237 | ue# 2238 | wiv 2239 | ide 2240 | idd 2241 | wir 2242 | vil 2243 | #3# 2244 | vin 2245 | vio 2246 | 100 2247 | vid 2248 | vie 2249 | clu 2250 | ken 2251 | via 2252 | vic 2253 | zoe 2254 | id_ 2255 | 00# 2256 | sum 2257 | #ty 2258 | cla 2259 | _ce 2260 | te# 2261 | ke_ 2262 | _ca 2263 | 008 2264 | nd# 2265 | vis 2266 | jec 2267 | #tr 2268 | #tu 2269 | w_l 2270 | #tw 2271 | #tv 2272 | #ti 2273 | #th 2274 | jen 2275 | #to 2276 | blu 2277 | #ta 2278 | _cr 2279 | uel 2280 | #te 2281 | ueg 2282 | rul 2283 | uee 2284 | run 2285 | bla 2286 | w_t 2287 | ruc 2288 | te_ 2289 | ke# 2290 | rug 2291 | v_l 2292 | pin 2293 | pio 2294 | pil 2295 | rni 2296 | ues 2297 | uer 2298 | rnm 2299 | ten 2300 | pid 2301 | rna 2302 | tem 2303 | v_c 2304 | rne 2305 | #x2 2306 | teg 2307 | ted 2308 | tee 2309 | pis 2310 | tec 2311 | yra 2312 | tea 2313 | ue_ 2314 | pit 2315 | v_r 2316 | kip 2317 | #x# 2318 | tew 2319 | ter 2320 | tes 2321 | ui# 2322 | oet 2323 | ld_ 2324 | lly 2325 | oes 2326 | kie 2327 | kid 2328 | ne_ 2329 | oey 2330 | ie# 2331 | kin 2332 | jiu 2333 | p_n 2334 | p_o 2335 | ldw 2336 | p_m 2337 | ldr 2338 | net 2339 | neu 2340 | nev 2341 | p_d 2342 | rn# 2343 | ner 2344 | nes 2345 | glo 2346 | nex 2347 | ney 2348 | p_s 2349 | p_p 2350 | ned 2351 | nee 2352 | p_t 2353 | uin 2354 | nea 2355 | uil 2356 | k_s 2357 | nel 2358 | ldi 2359 | nen 2360 | nei 2361 | flu 2362 | ies 2363 | wne 2364 | wnf 2365 | iev 2366 | k_c 2367 | k_e 2368 | uiv 2369 | rri 2370 | uit 2371 | uis 2372 | yor 2373 | _on 2374 | ad# 2375 | tin 2376 | tio 2377 | til 2378 | tim 2379 | ied 2380 | rre 2381 | tif 2382 | rry 2383 | tie 2384 | iel 2385 | tic 2386 | hes 2387 | tia 2388 | _gi 2389 | nct 2390 | heu 2391 | sif 2392 | tiv 2393 | tit 2394 | tir 2395 | tis 2396 | ys# 2397 | hea 2398 | hef 2399 | hed 2400 | ne# 2401 | que 2402 | hei 2403 | hen 2404 | heo 2405 | ni_ 2406 | hem 2407 | #pi 2408 | som 2409 | adq 2410 | son 2411 | wn# 2412 | rid 2413 | soc 2414 | ces 2415 | rig 2416 | sod 2417 | adh 2418 | oin 2419 | adm 2420 | pul 2421 | adc 2422 | nit 2423 | me# 2424 | niv 2425 | ro_ 2426 | t_l 2427 | ade 2428 | add 2429 | nis 2430 | sou 2431 | cee 2432 | t_f 2433 | ad_ 2434 | t_d 2435 | t_e 2436 | niz 2437 | t_c 2438 | rol 2439 | t_a 2440 | ron 2441 | nig 2442 | roa 2443 | rob 2444 | roc 2445 | rod 2446 | nim 2447 | nin 2448 | wro 2449 | t_t 2450 | wri 2451 | t_r 2452 | t_s 2453 | t_p 2454 | bel 2455 | le# 2456 | ben 2457 | rop 2458 | ror 2459 | ros 2460 | rot 2461 | rou 2462 | rov 2463 | row 2464 | beg 2465 | wry 2466 | _me 2467 | yss 2468 | met 2469 | mes 2470 | mer 2471 | o_g 2472 | 800 2473 | yst 2474 | _mo 2475 | mez 2476 | jus 2477 | kni 2478 | med 2479 | his 2480 | hip 2481 | ed# 2482 | kno 2483 | n_w 2484 | hit 2485 | mem 2486 | n_r 2487 | n_s 2488 | lev 2489 | ely 2490 | let 2491 | hib 2492 | hic 2493 | les 2494 | n_e 2495 | n_f 2496 | n_g 2497 | gna 2498 | wif 2499 | elp 2500 | n_c 2501 | n_l 2502 | lex 2503 | hin 2504 | hio 2505 | hil 2506 | ssi 2507 | lee 2508 | leb 2509 | lec 2510 | ele 2511 | lea 2512 | wit 2513 | ssa 2514 | be# 2515 | cip 2516 | cis 2517 | sse 2518 | uni 2519 | cit 2520 | ung 2521 | won 2522 | une 2523 | und 2524 | unc 2525 | coa 2526 | yla 2527 | cil 2528 | get 2529 | cin 2530 | cia 2531 | edl 2532 | ges 2533 | pri 2534 | edi 2535 | pro 2536 | rsh 2537 | rsi 2538 | unt 2539 | uns 2540 | edb 2541 | eda 2542 | rso 2543 | ed_ 2544 | pre 2545 | bir 2546 | wor 2547 | rse 2548 | ddy 2549 | ge_ 2550 | voi 2551 | za# 2552 | ddr 2553 | bil 2554 | fet 2555 | bin 2556 | bio 2557 | bia 2558 | bib 2559 | rst 2560 | fes 2561 | fel 2562 | ddi 2563 | big 2564 | h_m 2565 | #ir 2566 | mit 2567 | mis 2568 | ur# 2569 | fee 2570 | ont 2571 | ss# 2572 | fea 2573 | c_p 2574 | fec 2575 | c_r 2576 | c_t 2577 | mig 2578 | ony 2579 | c_h 2580 | mic 2581 | ghb 2582 | in# 2583 | ond 2584 | pic 2585 | min 2586 | ona 2587 | mil 2588 | onc 2589 | onb 2590 | onm 2591 | c_d 2592 | liv 2593 | c_f 2594 | oni 2595 | lis 2596 | pir 2597 | rs# 2598 | ge# 2599 | liz 2600 | pok 2601 | poi 2602 | on_ 2603 | lig 2604 | pol 2605 | lie 2606 | lib 2607 | lic 2608 | t2# 2609 | lia 2610 | lin 2611 | lio 2612 | urn 2613 | nna 2614 | lik 2615 | rld 2616 | uri 2617 | rlf 2618 | nno 2619 | urf 2620 | ure 2621 | por 2622 | urc 2623 | pop 2624 | ura 2625 | pow 2626 | pot 2627 | pou 2628 | inu 2629 | rls 2630 | gis 2631 | gir 2632 | rup 2633 | gio 2634 | m_f 2635 | m_e 2636 | m_d 2637 | urr 2638 | inb 2639 | _ba 2640 | m_o 2641 | inf 2642 | on# 2643 | ind 2644 | _bo 2645 | inj 2646 | tri 2647 | sfe 2648 | inn 2649 | tre 2650 | l_u 2651 | l_r 2652 | l_s 2653 | tra 2654 | ee# 2655 | fiv 2656 | 10# 2657 | xpe 2658 | fir 2659 | try 2660 | jou 2661 | xpo 2662 | l_g 2663 | tru 2664 | l_e 2665 | rly 2666 | l_c 2667 | fid 2668 | fie 2669 | ort 2670 | l_o 2671 | l_l 2672 | 101 2673 | orp 2674 | fic 2675 | l_h 2676 | hni 2677 | _ga 2678 | ki# 2679 | spi 2680 | sph 2681 | spy 2682 | 0fi 2683 | ore 2684 | ord 2685 | org 2686 | cur 2687 | spa 2688 | de_ 2689 | orc 2690 | g_s 2691 | orm 2692 | orl 2693 | oro 2694 | orn 2695 | ori 2696 | 009 2697 | ork 2698 | eet 2699 | ees 2700 | g_e 2701 | g_d 2702 | g_g 2703 | to_ 2704 | g_a 2705 | icl 2706 | g_c 2707 | eek 2708 | or_ 2709 | pso 2710 | rph 2711 | rpi 2712 | uot 2713 | eed 2714 | eec 2715 | f_t 2716 | f_u 2717 | ton 2718 | f_w 2719 | but 2720 | tom 2721 | wls 2722 | f_s 2723 | rpe 2724 | rag 2725 | nro 2726 | f_o 2727 | tod 2728 | bum 2729 | irs 2730 | f_d 2731 | ln# 2732 | irp 2733 | f_g 2734 | deo 2735 | rie 2736 | f_b 2737 | buf 2738 | #dn 2739 | dei 2740 | def 2741 | deg 2742 | _ve 2743 | tor 2744 | deb 2745 | oic 2746 | _va 2747 | dea 2748 | oot 2749 | ire 2750 | rar 2751 | a_t 2752 | iri 2753 | a_i 2754 | iro 2755 | a_o 2756 | irm 2757 | irl 2758 | uba 2759 | a_c 2760 | io# 2761 | ood 2762 | scu 2763 | wl# 2764 | mui 2765 | nva 2766 | ool 2767 | oon 2768 | ook 2769 | nic 2770 | ks_ 2771 | nov 2772 | now 2773 | hre 2774 | nor 2775 | hri 2776 | pli 2777 | hro 2778 | plo 2779 | lne 2780 | nod 2781 | pla 2782 | usn 2783 | ple 2784 | nob 2785 | rcu 2786 | nom 2787 | ush 2788 | noo 2789 | use 2790 | usc 2791 | usb 2792 | usa 2793 | cro 2794 | ts_ 2795 | cra 2796 | iou 2797 | iot 2798 | gns 2799 | cre 2800 | klu 2801 | eig 2802 | gni 2803 | wan 2804 | usp 2805 | wie 2806 | gua 2807 | _cl 2808 | _co 2809 | _ci 2810 | _ch 2811 | div 2812 | ion 2813 | _cu 2814 | diu 2815 | dir 2816 | dis 2817 | hie 2818 | us_ 2819 | bro 2820 | 11# 2821 | hot 2822 | hou 2823 | umc 2824 | orh 2825 | _cy 2826 | him 2827 | dif 2828 | did 2829 | die 2830 | hob 2831 | dic 2832 | dia 2833 | ost 2834 | 112 2835 | cir 2836 | _zo 2837 | oss 2838 | hoi 2839 | any 2840 | hoo 2841 | ns_ 2842 | e_v 2843 | mi# 2844 | e_t 2845 | e_s 2846 | ose 2847 | aur 2848 | e_p 2849 | e_o 2850 | e_n 2851 | e_m 2852 | ant 2853 | ank 2854 | cor 2855 | cou 2856 | e_h 2857 | pp_ 2858 | e_f 2859 | ule 2860 | e_d 2861 | e_c 2862 | e_b 2863 | e_a 2864 | 012 2865 | ang 2866 | 010 2867 | 011 2868 | and 2869 | squ 2870 | uly 2871 | cod 2872 | an_ 2873 | d_w 2874 | d_t 2875 | boy 2876 | d_r 2877 | d_s 2878 | d_p 2879 | ulp 2880 | ren 2881 | d_o 2882 | ppe 2883 | d_m 2884 | bor 2885 | bie 2886 | d_h 2887 | sso 2888 | im# 2889 | bow 2890 | nsi 2891 | d_e 2892 | d_b 2893 | ppp 2894 | bol 2895 | d_a 2896 | isp 2897 | boo 2898 | pay 2899 | boa 2900 | ist 2901 | bod 2902 | tly 2903 | urt 2904 | gro 2905 | mow 2906 | mov 2907 | mou 2908 | isc 2909 | ssu 2910 | gre 2911 | olu 2912 | ise 2913 | isd 2914 | _wo 2915 | ols 2916 | _wi 2917 | ish 2918 | iso 2919 | ism 2920 | oly 2921 | mod 2922 | il# 2923 | old 2924 | _wr 2925 | mon 2926 | co# 2927 | olm 2928 | is_ 2929 | lov 2930 | low 2931 | fri 2932 | nds 2933 | lor 2934 | los 2935 | lop 2936 | ns# 2937 | #a# 2938 | ndf 2939 | loy 2940 | ol_ 2941 | log 2942 | ima 2943 | kil 2944 | loc 2945 | ime 2946 | loa 2947 | arp 2948 | loo 2949 | bs_ 2950 | giv 2951 | art 2952 | ark 2953 | upi 2954 | aro 2955 | w1# 2956 | upe 2957 | enu 2958 | ent 2959 | arb 2960 | ms# 2961 | gie 2962 | arg 2963 | eno 2964 | gov 2965 | ard 2966 | ilt 2967 | gor 2968 | ily 2969 | ar_ 2970 | eng 2971 | bre 2972 | ene 2973 | ups 2974 | enc 2975 | egn 2976 | ena 2977 | god 2978 | en_ 2979 | #ap 2980 | #as 2981 | #ar 2982 | #au 2983 | #at 2984 | #aw 2985 | #av 2986 | #ai 2987 | ilm 2988 | ill 2989 | #am 2990 | #al 2991 | up_ 2992 | #an 2993 | #aa 2994 | fig 2995 | #ac 2996 | for 2997 | _up 2998 | #ad 2999 | #ag 3000 | #af 3001 | foo 3002 | irt 3003 | opu 3004 | ais 3005 | air 3006 | ls_ 3007 | hle 3008 | 12# 3009 | ops 3010 | dua 3011 | aij 3012 | kyw 3013 | ain 3014 | ail 3015 | sta 3016 | cs# 3017 | aig 3018 | c_b 3019 | opp 3020 | opl 3021 | opo 3022 | ps# 3023 | cli 3024 | gh# 3025 | ms_ 3026 | clo 3027 | rif 3028 | go# 3029 | na_ 3030 | #u# 3031 | hur 3032 | cle 3033 | #04 3034 | lse 3035 | swe 3036 | dr# 3037 | pel 3038 | oid 3039 | bs# 3040 | ery 3041 | ar# 3042 | pee 3043 | erv 3044 | bli 3045 | ert 3046 | ips 3047 | err 3048 | ll# 3049 | erp 3050 | ero 3051 | ern 3052 | erm 3053 | erl 3054 | eri 3055 | erg 3056 | erf 3057 | _te 3058 | erd 3059 | erc 3060 | #jo 3061 | ipa 3062 | er_ 3063 | #up 3064 | #us 3065 | _to 3066 | _ti 3067 | _th 3068 | gs_ 3069 | #uk 3070 | nio 3071 | _tv 3072 | #ul 3073 | ked 3074 | -------------------------------------------------------------------------------- /CDSSM_sparse/data/l3g.txt: -------------------------------------------------------------------------------- 1 | #un 2 | _tr 3 | ri# 4 | _ty 5 | dri 6 | ip_ 7 | ars 8 | arr 9 | dre 10 | sw# 11 | jer 12 | lls 13 | dra 14 | pst 15 | #az 16 | di# 17 | asy 18 | lle 19 | ef# 20 | ass 21 | lla 22 | lln 23 | llo 24 | eit 25 | ast 26 | ask 27 | lli 28 | ash 29 | ein 30 | fl_ 31 | uen 32 | asc 33 | haf 34 | eor 35 | eop 36 | ase 37 | hai 38 | ued 39 | g_m 40 | hig 41 | eog 42 | ts# 43 | al# 44 | dit 45 | gla 46 | car 47 | #fr 48 | #fu 49 | din 50 | dio 51 | #fi 52 | fly 53 | tel 54 | cam 55 | #fl 56 | #fo 57 | can 58 | zon 59 | dol 60 | #fc 61 | usl 62 | #fe 63 | amu 64 | doi 65 | flo 66 | doe 67 | vir 68 | mps 69 | mpr 70 | fle 71 | er# 72 | amb 73 | cie 74 | bar 75 | ks# 76 | ame 77 | aly 78 | 166 79 | mpe 80 | ust 81 | als 82 | mpa 83 | mpo 84 | bit 85 | ski 86 | alu 87 | alt 88 | alk 89 | mpi 90 | ali 91 | lpt 92 | alm 93 | all 94 | alc 95 | alb 96 | ala 97 | bic 98 | 98# 99 | ale 100 | y_a 101 | #10 102 | #13 103 | y_f 104 | y_e 105 | al_ 106 | y_k 107 | t50 108 | y_i 109 | tsu 110 | new 111 | ds# 112 | y_m 113 | ds_ 114 | y_p 115 | osp 116 | may 117 | mid 118 | he# 119 | est 120 | ess 121 | ww1 122 | esp 123 | ww2 124 | is# 125 | iqu 126 | em# 127 | esi 128 | osc 129 | osb 130 | lit 131 | ese 132 | ier 133 | esc 134 | rks 135 | ap# 136 | es_ 137 | nst 138 | nsu 139 | 886 140 | #jr 141 | #ju 142 | lif 143 | #ji 144 | sce 145 | el# 146 | #ja 147 | gat 148 | hqu 149 | nsf 150 | #je 151 | dsh 152 | s_l 153 | gar 154 | ba# 155 | gal 156 | her 157 | s_c 158 | s_b 159 | nsh 160 | rs_ 161 | iss 162 | gag 163 | ems 164 | hey 165 | isu 166 | s_p 167 | gin 168 | lal 169 | aps 170 | app 171 | hee 172 | eme 173 | isa 174 | ema 175 | api 176 | aph 177 | fal 178 | isk 179 | isi 180 | elt 181 | els 182 | r_e 183 | isl 184 | elo 185 | ape 186 | cep 187 | ell 188 | cer 189 | eli 190 | fil 191 | you 192 | r_r 193 | eld 194 | cem 195 | ela 196 | r_w 197 | el_ 198 | cea 199 | fif 200 | cec 201 | #gr 202 | #gu 203 | ced 204 | #gi 205 | #gh 206 | #gm 207 | #gl 208 | #go 209 | #gn 210 | #ga 211 | ber 212 | #ge 213 | #gd 214 | _ye 215 | dle 216 | ida 217 | bea 218 | bec 219 | bed 220 | bee 221 | bef 222 | twi 223 | aid 224 | twe 225 | ep# 226 | le_ 227 | gch 228 | cs_ 229 | ymb 230 | bst 231 | mec 232 | sor 233 | mea 234 | yme 235 | men 236 | ce# 237 | mel 238 | ls# 239 | ymn 240 | rom 241 | roo 242 | w_g 243 | ymp 244 | ler 245 | #bo 246 | niq 247 | rof 248 | rog 249 | roy 250 | #6# 251 | erb 252 | ley 253 | wsp 254 | leg 255 | led 256 | own 257 | epr 258 | nia 259 | len 260 | epl 261 | lem 262 | epi 263 | unn 264 | v_e 265 | v_g 266 | fe_ 267 | v_a 268 | ws_ 269 | lsv 270 | 3th 271 | dus 272 | da# 273 | ugs 274 | #ki 275 | gel 276 | geo 277 | v_p 278 | #kl 279 | #ko 280 | #kn 281 | sma 282 | ged 283 | #ke 284 | ita 285 | eap 286 | ayo 287 | ean 288 | eam 289 | 188 290 | #lu 291 | ws# 292 | ro# 293 | rm_ 294 | wiz 295 | 181 296 | 180 297 | ps_ 298 | eac 299 | eiv 300 | fer 301 | rmi 302 | eir 303 | fen 304 | rmo 305 | day 306 | rma 307 | fed 308 | eau 309 | dau 310 | rme 311 | 86# 312 | dap 313 | feb 314 | rmy 315 | as# 316 | p_c 317 | sm_ 318 | shm 319 | rms 320 | dad 321 | p_r 322 | ea_ 323 | wou 324 | uke 325 | civ 326 | sm# 327 | vol 328 | snl 329 | k_u 330 | us# 331 | voc 332 | sci 333 | efe 334 | sca 335 | k_o 336 | asu 337 | rm# 338 | asi 339 | aso 340 | fe# 341 | 0_4 342 | gs# 343 | wo# 344 | pie 345 | zes 346 | oks 347 | uck 348 | uci 349 | _tl 350 | #fa 351 | usi 352 | rcr 353 | oki 354 | fs# 355 | poe 356 | sku 357 | equ 358 | pos 359 | sc# 360 | tch 361 | t_n 362 | t_o 363 | adj 364 | adi 365 | t_m 366 | ado 367 | de# 368 | t_h 369 | t_i 370 | t_g 371 | lte 372 | t_b 373 | rke 374 | #hu 375 | ti# 376 | eer 377 | #hi 378 | t_w 379 | dqu 380 | t_u 381 | koh 382 | #ho 383 | imi 384 | sba 385 | kon 386 | #he 387 | jor 388 | eeb 389 | o_s 390 | ike 391 | gto 392 | to# 393 | dev 394 | 665 395 | 964 396 | o_a 397 | der 398 | des 399 | dep 400 | aw# 401 | den 402 | del 403 | joi 404 | rbo 405 | n_t 406 | eso 407 | fit 408 | ded 409 | n_p 410 | dec 411 | nsw 412 | rby 413 | nk# 414 | fin 415 | esa 416 | n_d 417 | #oz 418 | nco 419 | n_b 420 | n_m 421 | _vo 422 | n_o 423 | um# 424 | n_i 425 | #ov 426 | i_p 427 | atr 428 | ckn 429 | #on 430 | toc 431 | ss_ 432 | i_i 433 | tow 434 | tou 435 | tos 436 | top 437 | h_p 438 | h_t 439 | svi 440 | #ka 441 | 08# 442 | umm 443 | rge 444 | h_a 445 | h_g 446 | ume 447 | hsa 448 | umb 449 | uma 450 | h_o 451 | nke 452 | not 453 | nou 454 | nif 455 | rvi 456 | ycl 457 | ums 458 | c_s 459 | #pu 460 | sui 461 | c_i 462 | ex# 463 | rve 464 | c_m 465 | #pl 466 | c_n 467 | tme 468 | c_c 469 | et# 470 | tma 471 | um_ 472 | c_g 473 | xce 474 | nok 475 | xch 476 | lke 477 | reh 478 | b_s 479 | omp 480 | tn# 481 | rum 482 | omy 483 | sch 484 | iod 485 | ome 486 | rue 487 | sco 488 | oma 489 | m_v 490 | iol 491 | omb 492 | omm 493 | m_s 494 | m_r 495 | omi 496 | eiz 497 | ak# 498 | rus 499 | #ts 500 | scr 501 | hoc 502 | m_c 503 | om_ 504 | una 505 | rch 506 | gma 507 | m_n 508 | m_m 509 | m_l 510 | nme 511 | rco 512 | sza 513 | uca 514 | hom 515 | l_t 516 | rce 517 | l_p 518 | v_n 519 | cop 520 | x20 521 | imp 522 | cot 523 | l_f 524 | cov 525 | l_d 526 | coh 527 | l_b 528 | l_a 529 | col 530 | _ad 531 | con 532 | _af 533 | ub# 534 | _ac 535 | _ab 536 | om# 537 | cog 538 | _an 539 | _ai 540 | imo 541 | _at 542 | g_t 543 | g_w 544 | _ap 545 | onv 546 | _ar 547 | g_r 548 | ons 549 | g_l 550 | g_o 551 | g_i 552 | g_h 553 | uys 554 | wea 555 | bon 556 | one 557 | g_f 558 | aga 559 | g_b 560 | hme 561 | oce 562 | mot 563 | mos 564 | mor 565 | wes 566 | mna 567 | swo 568 | f_r 569 | ubl 570 | ubj 571 | vek 572 | ubd 573 | f_f 574 | moo 575 | nne 576 | f_a 577 | f_c 578 | #8a 579 | uto 580 | lot 581 | ubu 582 | a_s 583 | ubs 584 | acr 585 | ek# 586 | ug# 587 | lid 588 | lon 589 | lm# 590 | ub_ 591 | lom 592 | ing 593 | jsc 594 | ink 595 | tbr 596 | got 597 | sim 598 | jil 599 | lm_ 600 | _un 601 | gol 602 | mme 603 | gne 604 | obe 605 | _us 606 | mmo 607 | goa 608 | oba 609 | gig 610 | obb 611 | obl 612 | pes 613 | lms 614 | tv# 615 | nbu 616 | pus 617 | _ne 618 | fol 619 | _na 620 | lme 621 | lma 622 | ugu 623 | ccu 624 | nba 625 | _nu 626 | ibs 627 | #el 628 | gmt 629 | tv_ 630 | ibu 631 | ny# 632 | 24# 633 | yto 634 | aoh 635 | _fe 636 | am# 637 | ytv 638 | ibe 639 | _fl 640 | _fo 641 | _fi 642 | e_z 643 | _fu 644 | ibl 645 | e_u 646 | e_r 647 | my# 648 | xte 649 | ogy 650 | e_l 651 | uem 652 | e_i 653 | xtr 654 | xts 655 | e_e 656 | nsp 657 | tc# 658 | nb# 659 | amy 660 | hbo 661 | wwe 662 | sti 663 | ams 664 | ove 665 | stl 666 | amp 667 | 04# 668 | ly# 669 | d_n 670 | ucl 671 | d_l 672 | ste 673 | ami 674 | uch 675 | amo 676 | d_i 677 | uce 678 | d_g 679 | ucc 680 | #ha 681 | ama 682 | d_c 683 | rt_ 684 | w2# 685 | amd 686 | str 687 | stu 688 | tud 689 | am_ 690 | rth 691 | rti 692 | uct 693 | nve 694 | rtn 695 | rto 696 | tco 697 | rta 698 | bbr 699 | rte 700 | nvo 701 | rty 702 | bbi 703 | st_ 704 | lyr 705 | lys 706 | do# 707 | rts 708 | bbe 709 | rtw 710 | sul 711 | sun 712 | iva 713 | 0_2 714 | 0_3 715 | hsc 716 | st# 717 | ouc 718 | ivi 719 | 08_ 720 | by# 721 | _ju 722 | mbe 723 | ic# 724 | mbo 725 | oca 726 | occ 727 | mbi 728 | oci 729 | lbu 730 | ock 731 | gym 732 | dow 733 | _bi 734 | rt# 735 | dop 736 | ay# 737 | eid 738 | dom 739 | ncy 740 | _br 741 | iz# 742 | nce 743 | iet 744 | _by 745 | anu 746 | ncl 747 | emy 748 | ani 749 | nch 750 | nci 751 | ics 752 | icr 753 | emp 754 | emo 755 | ict 756 | ien 757 | ane 758 | emi 759 | cvg 760 | #dy 761 | _ge 762 | emb 763 | ica 764 | het 765 | two 766 | zy# 767 | ice 768 | ick 769 | #du 770 | ici 771 | ich 772 | #di 773 | _gu 774 | huy 775 | pt# 776 | #do 777 | bid 778 | #da 779 | #dc 780 | #de 781 | dmi 782 | ic_ 783 | pm# 784 | gy# 785 | iza 786 | ize 787 | 2nd 788 | ows 789 | owr 790 | aby 791 | cei 792 | cus 793 | abr 794 | cel 795 | cen 796 | owa 797 | abu 798 | owl 799 | cul 800 | abo 801 | abl 802 | abb 803 | aba 804 | cco 805 | ce_ 806 | abe 807 | buy 808 | cce 809 | ow_ 810 | pto 811 | lve 812 | bei 813 | pta 814 | pte 815 | lvi 816 | 998 817 | ag# 818 | env 819 | ktr 820 | enn 821 | _2# 822 | ut# 823 | ubb 824 | ow# 825 | _kn 826 | _fa 827 | mum 828 | 19# 829 | cc# 830 | #e1 831 | me_ 832 | mci 833 | 199 834 | _fr 835 | agr 836 | 192 837 | hwe 838 | sol 839 | 196 840 | agn 841 | agm 842 | sym 843 | lco 844 | uti 845 | uth 846 | _0# 847 | ute 848 | ebt 849 | ger 850 | ebr 851 | ry_ 852 | gen 853 | ave 854 | sys 855 | ryi 856 | #ex 857 | uts 858 | gue 859 | tto 860 | ttl 861 | #ep 862 | #es 863 | #er 864 | #eu 865 | #et 866 | dby 867 | #ev 868 | tte 869 | #em 870 | sy_ 871 | ut_ 872 | #en 873 | dbo 874 | #ec 875 | y_o 876 | #ed 877 | #eg 878 | ful 879 | ttr 880 | az# 881 | ott 882 | gza 883 | ots 884 | acy 885 | nt_ 886 | es# 887 | ote 888 | #16 889 | ota 890 | act 891 | ack 892 | aci 893 | ach 894 | aco 895 | oti 896 | oth 897 | acl 898 | acc 899 | aca 900 | adg 901 | ry# 902 | ace 903 | ntr 904 | nts 905 | _bu 906 | azy 907 | osi 908 | nty 909 | ht# 910 | s_n 911 | nta 912 | #4# 913 | ntm 914 | ego 915 | nto 916 | nth 917 | nti 918 | its 919 | evo 920 | ip# 921 | fc# 922 | itt 923 | evi 924 | 8am 925 | _he 926 | nso 927 | _ha 928 | ot# 929 | #is 930 | _ho 931 | _hi 932 | ith 933 | ito 934 | sdi 935 | itl 936 | #im 937 | #il 938 | tua 939 | #in 940 | nt2 941 | #ic 942 | #id 943 | dvi 944 | #if 945 | os# 946 | dve 947 | vot 948 | 0s# 949 | nt# 950 | yno 951 | rdi 952 | aws 953 | ez# 954 | dc# 955 | bt_ 956 | cts 957 | ctu 958 | awo 959 | awn 960 | cti 961 | ecu 962 | ect 963 | awa 964 | ctl 965 | cto 966 | eco 967 | een 968 | cte 969 | eci 970 | ech 971 | pon 972 | ece 973 | eca 974 | ct_ 975 | op# 976 | rro 977 | pyr 978 | dcu 979 | lt# 980 | ezo 981 | num 982 | kos 983 | kor 984 | mtv 985 | uy# 986 | dca 987 | lt_ 988 | bsu 989 | yri 990 | bsi 991 | ew# 992 | zar 993 | ct# 994 | xt_ 995 | ltu 996 | ty# 997 | _vi 998 | lty 999 | job 1000 | v_s 1001 | uyn 1002 | #5# 1003 | lti 1004 | 819 1005 | ft_ 1006 | izo 1007 | uoi 1008 | ty_ 1009 | ewm 1010 | ewi 1011 | zai 1012 | #ny 1013 | gth 1014 | at# 1015 | ew_ 1016 | tyl 1017 | zat 1018 | p_i 1019 | #nu 1020 | #ni 1021 | too 1022 | mpu 1023 | #no 1024 | #ns 1025 | #na 1026 | _ta 1027 | #nb 1028 | #ne 1029 | #nf 1030 | tyt 1031 | fti 1032 | lpe 1033 | typ 1034 | rra 1035 | fte 1036 | lpa 1037 | aty 1038 | ny_ 1039 | ats 1040 | oye 1041 | oop 1042 | oyf 1043 | oya 1044 | atu 1045 | att 1046 | req 1047 | oym 1048 | ati 1049 | ath 1050 | ato 1051 | atl 1052 | atc 1053 | ata 1054 | ate 1055 | #ut 1056 | at_ 1057 | hy# 1058 | yna 1059 | nyc 1060 | ho# 1061 | pra 1062 | ils 1063 | ft# 1064 | non 1065 | oy# 1066 | #ru 1067 | #ri 1068 | #ro 1069 | #ra 1070 | #rc 1071 | oo# 1072 | #re 1073 | wen 1074 | 0_0 1075 | iom 1076 | hor 1077 | hos 1078 | e3# 1079 | gm# 1080 | udi 1081 | how 1082 | ews 1083 | yah 1084 | yal 1085 | sac 1086 | t_v 1087 | udy 1088 | by_ 1089 | hod 1090 | etw 1091 | hon 1092 | xac 1093 | hol 1094 | ets 1095 | etr 1096 | rab 1097 | eto 1098 | uro 1099 | 23# 1100 | cyc 1101 | xan 1102 | eti 1103 | eth 1104 | urg 1105 | xas 1106 | o_c 1107 | urd 1108 | etb 1109 | eta 1110 | cy_ 1111 | et_ 1112 | #op 1113 | com 1114 | #or 1115 | #ou 1116 | #ot 1117 | #ow 1118 | ury 1119 | sai 1120 | #oh 1121 | sak 1122 | dtr 1123 | sam 1124 | sal 1125 | box 1126 | san 1127 | urp 1128 | 13# 1129 | tro 1130 | sab 1131 | sat 1132 | #od 1133 | 01# 1134 | #of 1135 | say 1136 | bos 1137 | ode 1138 | bou 1139 | ly_ 1140 | sas 1141 | odl 1142 | sau 1143 | jr# 1144 | 013 1145 | kna 1146 | rah 1147 | rai 1148 | ndt 1149 | rak 1150 | ral 1151 | ram 1152 | ran 1153 | rao 1154 | cy# 1155 | rac 1156 | rad 1157 | od_ 1158 | raf 1159 | #it 1160 | ors 1161 | ray 1162 | raz 1163 | nde 1164 | 09# 1165 | ory 1166 | py# 1167 | rap 1168 | ndb 1169 | ir# 1170 | ras 1171 | rat 1172 | #s# 1173 | rav 1174 | ora 1175 | llv 1176 | ids 1177 | mom 1178 | lyn 1179 | sa# 1180 | lym 1181 | #2p 1182 | exu 1183 | ext 1184 | xua 1185 | exp 1186 | od# 1187 | nre 1188 | gyp 1189 | idi 1190 | #2n 1191 | exh 1192 | #sy 1193 | #sz 1194 | bo# 1195 | exa 1196 | ra# 1197 | #sp 1198 | #su 1199 | #st 1200 | #sw 1201 | #si 1202 | #sh 1203 | #sk 1204 | #sm 1205 | #sl 1206 | #so 1207 | #sn 1208 | sua 1209 | c_a 1210 | suc 1211 | sub 1212 | #se 1213 | wal 1214 | suf 1215 | or# 1216 | yne 1217 | gog 1218 | wag 1219 | way 1220 | sup 1221 | sus 1222 | sur 1223 | rui 1224 | yfr 1225 | ays 1226 | wat 1227 | md# 1228 | wav 1229 | rua 1230 | rub 1231 | was 1232 | war 1233 | val 1234 | vam 1235 | ayi 1236 | b_f 1237 | ayn 1238 | fla 1239 | ayl 1240 | vad 1241 | m_t 1242 | teo 1243 | aye 1244 | vac 1245 | vat 1246 | ay_ 1247 | #ly 1248 | bdi 1249 | cru 1250 | ld# 1251 | m_a 1252 | vas 1253 | tex 1254 | 64# 1255 | #li 1256 | sno 1257 | sna 1258 | #ve 1259 | ija 1260 | #lo 1261 | #la 1262 | fy# 1263 | #le 1264 | 00s 1265 | oz# 1266 | rn_ 1267 | bru 1268 | pad 1269 | bri 1270 | l_n 1271 | pai 1272 | pan 1273 | #mt 1274 | l_j 1275 | rno 1276 | ey# 1277 | rfo 1278 | fl# 1279 | rfe 1280 | g_u 1281 | ova 1282 | par 1283 | pas 1284 | pap 1285 | pat 1286 | lde 1287 | ovi 1288 | ldc 1289 | 8_p 1290 | _de 1291 | 65# 1292 | dy# 1293 | cum 1294 | dy_ 1295 | _do 1296 | f_y 1297 | _di 1298 | wip 1299 | gdp 1300 | kai 1301 | f_p 1302 | eyt 1303 | gdo 1304 | irc 1305 | urk 1306 | _dr 1307 | he_ 1308 | jay 1309 | eyh 1310 | zoo 1311 | #py 1312 | ief 1313 | nt5 1314 | ta# 1315 | ey_ 1316 | jac 1317 | #pr 1318 | qui 1319 | quo 1320 | unk 1321 | ive 1322 | #ph 1323 | jai 1324 | qua 1325 | jan 1326 | #po 1327 | dyn 1328 | #pa 1329 | dyl 1330 | ual 1331 | uak 1332 | #pe 1333 | gri 1334 | uag 1335 | elm 1336 | uad 1337 | gra 1338 | ta_ 1339 | ka# 1340 | hel 1341 | pun 1342 | adv 1343 | adu 1344 | pub 1345 | uar 1346 | uat 1347 | wnt 1348 | tan 1349 | tal 1350 | tam 1351 | wns 1352 | tak 1353 | ada 1354 | tai 1355 | taf 1356 | tag 1357 | don 1358 | pur 1359 | tab 1360 | fre 1361 | tad 1362 | put 1363 | kup 1364 | dog 1365 | ary 1366 | tay 1367 | wn_ 1368 | taw 1369 | tat 1370 | tau 1371 | tar 1372 | tas 1373 | tap 1374 | tax 1375 | oat 1376 | ari 1377 | oar 1378 | kud 1379 | bes 1380 | bet 1381 | arm 1382 | arl 1383 | arc 1384 | ara 1385 | ia# 1386 | oad 1387 | bra 1388 | are 1389 | jur 1390 | oac 1391 | oam 1392 | oal 1393 | oan 1394 | nat 1395 | nau 1396 | pme 1397 | nar 1398 | jud 1399 | #ab 1400 | nas 1401 | nad 1402 | #19 1403 | #18 1404 | nag 1405 | rga 1406 | #i# 1407 | nac 1408 | nal 1409 | nam 1410 | nan 1411 | e_w 1412 | iar 1413 | er2 1414 | iat 1415 | rgu 1416 | _ed 1417 | _ef 1418 | tun 1419 | iag 1420 | un# 1421 | _em 1422 | _el 1423 | _en 1424 | edu 1425 | eds 1426 | ian 1427 | iam 1428 | ial 1429 | har 1430 | has 1431 | hap 1432 | _ep 1433 | hav 1434 | hat 1435 | edg 1436 | #d# 1437 | ede 1438 | _ex 1439 | ia_ 1440 | tut 1441 | tur 1442 | hab 1443 | tup 1444 | d_f 1445 | out 1446 | hag 1447 | had 1448 | na# 1449 | oup 1450 | ous 1451 | our 1452 | #qu 1453 | han 1454 | hal 1455 | ham 1456 | eru 1457 | iu# 1458 | ers 1459 | sko 1460 | cri 1461 | cap 1462 | cas 1463 | ske 1464 | oul 1465 | cat 1466 | oun 1467 | uff 1468 | cah 1469 | ere 1470 | ma# 1471 | cal 1472 | rk_ 1473 | era 1474 | fro 1475 | nur 1476 | nus 1477 | cab 1478 | cad 1479 | vg_ 1480 | fra 1481 | dru 1482 | nuf 1483 | cau 1484 | nuc 1485 | bas 1486 | bat 1487 | arv 1488 | bal 1489 | bam 1490 | ban 1491 | tne 1492 | bab 1493 | un_ 1494 | avi 1495 | roe 1496 | _id 1497 | mat 1498 | mas 1499 | mar 1500 | la# 1501 | map 1502 | ou# 1503 | r2# 1504 | _in 1505 | #eq 1506 | ig# 1507 | mag 1508 | ium 1509 | oga 1510 | mad 1511 | hus 1512 | mab 1513 | if# 1514 | 0_1 1515 | off 1516 | ca# 1517 | mal 1518 | mak 1519 | maj 1520 | mai 1521 | ong 1522 | onf 1523 | law 1524 | lat 1525 | hub 1526 | lar 1527 | las 1528 | lap 1529 | onl 1530 | ono 1531 | hud 1532 | rk# 1533 | man 1534 | lay 1535 | of_ 1536 | lag 1537 | hum 1538 | lav 1539 | lab 1540 | lac 1541 | nfe 1542 | lan 1543 | mac 1544 | vg# 1545 | cup 1546 | lak 1547 | nfl 1548 | lai 1549 | bad 1550 | nfo 1551 | nfi 1552 | #sa 1553 | gaz 1554 | #sc 1555 | ev# 1556 | k11 1557 | nni 1558 | gas 1559 | ins 1560 | gam 1561 | yho 1562 | gan 1563 | inv 1564 | int 1565 | ifa 1566 | gad 1567 | wom 1568 | iff 1569 | of# 1570 | bur 1571 | bus 1572 | inc 1573 | ifi 1574 | ina 1575 | ifo 1576 | _go 1577 | bui 1578 | ine 1579 | bul 1580 | fat 1581 | fau 1582 | ini 1583 | faw 1584 | inl 1585 | _gr 1586 | fam 1587 | fan 1588 | gac 1589 | mus 1590 | mur 1591 | mup 1592 | in_ 1593 | fac 1594 | gai 1595 | shi 1596 | fas 1597 | muc 1598 | aar 1599 | sho 1600 | sha 1601 | lam 1602 | ype 1603 | she 1604 | ugh 1605 | lut 1606 | nju 1607 | ga# 1608 | ypt 1609 | lud 1610 | lue 1611 | lub 1612 | luc 1613 | fad 1614 | tld 1615 | rho 1616 | lum 1617 | dem 1618 | luk 1619 | tba 1620 | guy 1621 | xpl 1622 | #dr 1623 | spl 1624 | gus 1625 | gur 1626 | gum 1627 | gul 1628 | guo 1629 | olv 1630 | gui 1631 | obs 1632 | uk# 1633 | ogt 1634 | #df 1635 | _no 1636 | spo 1637 | ogr 1638 | ole 1639 | ng_ 1640 | olf 1641 | spe 1642 | ogz 1643 | ea# 1644 | sky 1645 | olo 1646 | oln 1647 | oli 1648 | obj 1649 | fun 1650 | ogn 1651 | spr 1652 | spu 1653 | lfr 1654 | ngt 1655 | ngu 1656 | ngr 1657 | ngs 1658 | rpo 1659 | 003 1660 | og_ 1661 | nle 1662 | #wo 1663 | f_e 1664 | aus 1665 | ngd 1666 | nge 1667 | ywa 1668 | nga 1669 | aut 1670 | ngc 1671 | ngl 1672 | ibr 1673 | who 1674 | ngh 1675 | eav 1676 | aul 1677 | eat 1678 | eas 1679 | ear 1680 | aug 1681 | wha 1682 | igu 1683 | eal 1684 | eak 1685 | tla 1686 | eag 1687 | why 1688 | ilb 1689 | ead 1690 | ob# 1691 | rey 1692 | ile 1693 | igg 1694 | xt# 1695 | ige 1696 | ilh 1697 | igi 1698 | igh 1699 | igo 1700 | ign 1701 | dat 1702 | ffy 1703 | dar 1704 | an# 1705 | dan 1706 | dal 1707 | dam 1708 | bje 1709 | #tn 1710 | gn_ 1711 | _re 1712 | ffi 1713 | _ra 1714 | ffe 1715 | nl# 1716 | ng# 1717 | _ro 1718 | ox# 1719 | ufa 1720 | _ru 1721 | daw 1722 | _im 1723 | oke 1724 | oka 1725 | r_i 1726 | aft 1727 | djo 1728 | dwo 1729 | mb# 1730 | afl 1731 | ans 1732 | anr 1733 | hs# 1734 | aff 1735 | r_g 1736 | phi 1737 | ok_ 1738 | pho 1739 | ano 1740 | ann 1741 | azi 1742 | anl 1743 | anc 1744 | ana 1745 | 922 1746 | phe 1747 | gn# 1748 | phy 1749 | wel 1750 | dna 1751 | nki 1752 | sue 1753 | eur 1754 | eup 1755 | eun 1756 | eum 1757 | blo 1758 | stm 1759 | sto 1760 | ofe 1761 | ble 1762 | stc 1763 | _of 1764 | sty 1765 | _ob 1766 | ok# 1767 | ofi 1768 | ll_ 1769 | iki 1770 | nut 1771 | sts 1772 | _ow 1773 | mba 1774 | _op 1775 | dun 1776 | _or 1777 | #kt 1778 | nue 1779 | dub 1780 | duc 1781 | nfa 1782 | th# 1783 | en# 1784 | ppo 1785 | ppl 1786 | ppi 1787 | rtr 1788 | ifu 1789 | lbo 1790 | ify 1791 | cks 1792 | ajo 1793 | we# 1794 | efs 1795 | ife 1796 | efo 1797 | cke 1798 | hop 1799 | eff 1800 | ggi 1801 | up# 1802 | efa 1803 | ck_ 1804 | tho 1805 | thl 1806 | enr 1807 | thi 1808 | the 1809 | eni 1810 | enh 1811 | tha 1812 | pez 1813 | end 1814 | enb 1815 | dfh 1816 | thw 1817 | _ma 1818 | _se 1819 | thr 1820 | ths 1821 | _sa 1822 | _sc 1823 | ck1 1824 | _so 1825 | fli 1826 | em_ 1827 | _su 1828 | _st 1829 | _sq 1830 | _sp 1831 | cui 1832 | ck# 1833 | agu 1834 | cob 1835 | agi 1836 | oho 1837 | abi 1838 | ohi 1839 | alp 1840 | xhi 1841 | upa 1842 | e1# 1843 | age 1844 | ag_ 1845 | upt 1846 | ab_ 1847 | upr 1848 | lf# 1849 | upp 1850 | nha 1851 | #ct 1852 | #ci 1853 | _s# 1854 | 22# 1855 | #my 1856 | _le 1857 | axe 1858 | _la 1859 | opt 1860 | #os 1861 | oh# 1862 | _lo 1863 | #mu 1864 | _li 1865 | #mi 1866 | dju 1867 | #ol 1868 | #mo 1869 | ope 1870 | #ma 1871 | #oc 1872 | #ob 1873 | pti 1874 | #me 1875 | opm 1876 | dje 1877 | opi 1878 | oph 1879 | kfa 1880 | 123 1881 | nh# 1882 | ff_ 1883 | sh# 1884 | aks 1885 | 03# 1886 | 02# 1887 | aku 1888 | chr 1889 | aki 1890 | egy 1891 | chi 1892 | egu 1893 | 30# 1894 | egr 1895 | cho 1896 | chn 1897 | cha 1898 | ebs 1899 | che 1900 | egi 1901 | ak_ 1902 | ege 1903 | _be 1904 | ega 1905 | ch_ 1906 | _a_ 1907 | _3# 1908 | eba 1909 | ipe 1910 | yle 1911 | ipi 1912 | fou 1913 | yli 1914 | ipl 1915 | whi 1916 | _aw 1917 | _pe 1918 | dge 1919 | 200 1920 | _as 1921 | _pa 1922 | afr 1923 | _pl 1924 | _po 1925 | exi 1926 | _pi 1927 | dlo 1928 | _pu 1929 | exc 1930 | utc 1931 | _pp 1932 | o_p 1933 | _pr 1934 | ch# 1935 | 102 1936 | ibi 1937 | yde 1938 | aub 1939 | tti 1940 | owe 1941 | sla 1942 | lhe 1943 | tta 1944 | wme 1945 | sly 1946 | ff# 1947 | 13t 1948 | eks 1949 | ze_ 1950 | ght 1951 | n_a 1952 | rli 1953 | zed 1954 | ghs 1955 | n_h 1956 | rlo 1957 | #by 1958 | eke 1959 | #cl 1960 | ghe 1961 | #ca 1962 | it# 1963 | #cc 1964 | gha 1965 | #ce 1966 | #br 1967 | #bu 1968 | vma 1969 | otb 1970 | #bi 1971 | otl 1972 | nda 1973 | pha 1974 | #bl 1975 | fhu 1976 | #ba 1977 | ndo 1978 | #30 1979 | #be 1980 | nvi 1981 | mpl 1982 | yc# 1983 | rd_ 1984 | nte 1985 | lpr 1986 | #yo 1987 | lk# 1988 | #ya 1989 | 50_ 1990 | ntl 1991 | #ye 1992 | ahs 1993 | rdo 1994 | ake 1995 | ov# 1996 | oug 1997 | rde 1998 | itu 1999 | aho 2000 | ity 2001 | aha 2002 | _1# 2003 | rds 2004 | _4# 2005 | eou 2006 | ite 2007 | iti 2008 | hts 2009 | dfo 2010 | _ev 2011 | _eq 2012 | it_ 2013 | hte 2014 | dos 2015 | zil 2016 | zin 2017 | dur 2018 | zip 2019 | rd# 2020 | th_ 2021 | #vi 2022 | _sh 2023 | #vm 2024 | #vo 2025 | #va 2026 | thn 2027 | m_w 2028 | mt# 2029 | _du 2030 | _sy 2031 | yea 2032 | apt 2033 | yee 2034 | yed 2035 | apo 2036 | #gy 2037 | apa 2038 | yer 2039 | thq 2040 | m_h 2041 | #js 2042 | #cy 2043 | xes 2044 | _it 2045 | eha 2046 | 2pm 2047 | l_m 2048 | #cr 2049 | #cu 2050 | l_i 2051 | #cv 2052 | sei 2053 | #ch 2054 | lfe 2055 | sel 2056 | #co 2057 | sen 2058 | sea 2059 | sec 2060 | seb 2061 | see 2062 | sed 2063 | g_p 2064 | sey 2065 | sex 2066 | dhe 2067 | akf 2068 | ul# 2069 | re_ 2070 | ses 2071 | ser 2072 | seu 2073 | set 2074 | sew 2075 | #zi 2076 | rei 2077 | rek 2078 | rel 2079 | rem 2080 | #zo 2081 | rea 2082 | rec 2083 | red 2084 | ree 2085 | ref 2086 | reg 2087 | #e3 2088 | se_ 2089 | rep 2090 | dp_ 2091 | rer 2092 | res 2093 | ret 2094 | #20 2095 | rev 2096 | rew 2097 | yin 2098 | #24 2099 | ept 2100 | ud# 2101 | se# 2102 | #2# 2103 | _mu 2104 | rok 2105 | ulf 2106 | uld 2107 | _we 2108 | ula 2109 | chy 2110 | _wa 2111 | imu 2112 | xis 2113 | ult 2114 | chu 2115 | ryd 2116 | re# 2117 | #wr 2118 | #ww 2119 | #wi 2120 | #wh 2121 | y_c 2122 | y_b 2123 | tle 2124 | sio 2125 | sin 2126 | #wa 2127 | y_d 2128 | sic 2129 | sib 2130 | #we 2131 | sid 2132 | sig 2133 | udg 2134 | #ea 2135 | ude 2136 | y_s 2137 | y_r 2138 | wee 2139 | wed 2140 | y_v 2141 | y_u 2142 | y_t 2143 | sis 2144 | sir 2145 | web 2146 | sit 2147 | whe 2148 | ve_ 2149 | ril 2150 | rim 2151 | rin 2152 | rio 2153 | ria 2154 | rib 2155 | ric 2156 | wer 2157 | vel 2158 | vem 2159 | ven 2160 | riz 2161 | oll 2162 | ved 2163 | pe# 2164 | ud_ 2165 | rip 2166 | ris 2167 | rit 2168 | riu 2169 | riv 2170 | s_i 2171 | s_h 2172 | s_m 2173 | odu 2174 | s_o 2175 | nly 2176 | s_a 2177 | ver 2178 | ves 2179 | ods 2180 | s_e 2181 | s_d 2182 | s_g 2183 | s_f 2184 | nd_ 2185 | ody 2186 | 10f 2187 | id# 2188 | nli 2189 | oda 2190 | s_s 2191 | s_r 2192 | s_t 2193 | s_w 2194 | s_v 2195 | pe_ 2196 | odi 2197 | r_l 2198 | r_m 2199 | ndu 2200 | r_o 2201 | r_a 2202 | r_b 2203 | r_c 2204 | r_d 2205 | ila 2206 | r_f 2207 | pen 2208 | peo 2209 | ol# 2210 | ild 2211 | pec 2212 | ili 2213 | pea 2214 | ilo 2215 | r_p 2216 | ped 2217 | ve# 2218 | r_s 2219 | r_t 2220 | r_v 2221 | wil 2222 | 201 2223 | win 2224 | ndi 2225 | per 2226 | wik 2227 | ett 2228 | mma 2229 | pet 2230 | eve 2231 | kes 2232 | ker 2233 | ynh 2234 | ket 2235 | key 2236 | ete 2237 | ue# 2238 | wiv 2239 | ide 2240 | idd 2241 | wir 2242 | vil 2243 | #3# 2244 | vin 2245 | vio 2246 | 100 2247 | vid 2248 | vie 2249 | clu 2250 | ken 2251 | via 2252 | vic 2253 | zoe 2254 | id_ 2255 | 00# 2256 | sum 2257 | #ty 2258 | cla 2259 | _ce 2260 | te# 2261 | ke_ 2262 | _ca 2263 | 008 2264 | nd# 2265 | vis 2266 | jec 2267 | #tr 2268 | #tu 2269 | w_l 2270 | #tw 2271 | #tv 2272 | #ti 2273 | #th 2274 | jen 2275 | #to 2276 | blu 2277 | #ta 2278 | _cr 2279 | uel 2280 | #te 2281 | ueg 2282 | rul 2283 | uee 2284 | run 2285 | bla 2286 | w_t 2287 | ruc 2288 | te_ 2289 | ke# 2290 | rug 2291 | v_l 2292 | pin 2293 | pio 2294 | pil 2295 | rni 2296 | ues 2297 | uer 2298 | rnm 2299 | ten 2300 | pid 2301 | rna 2302 | tem 2303 | v_c 2304 | rne 2305 | #x2 2306 | teg 2307 | ted 2308 | tee 2309 | pis 2310 | tec 2311 | yra 2312 | tea 2313 | ue_ 2314 | pit 2315 | v_r 2316 | kip 2317 | #x# 2318 | tew 2319 | ter 2320 | tes 2321 | ui# 2322 | oet 2323 | ld_ 2324 | lly 2325 | oes 2326 | kie 2327 | kid 2328 | ne_ 2329 | oey 2330 | ie# 2331 | kin 2332 | jiu 2333 | p_n 2334 | p_o 2335 | ldw 2336 | p_m 2337 | ldr 2338 | net 2339 | neu 2340 | nev 2341 | p_d 2342 | rn# 2343 | ner 2344 | nes 2345 | glo 2346 | nex 2347 | ney 2348 | p_s 2349 | p_p 2350 | ned 2351 | nee 2352 | p_t 2353 | uin 2354 | nea 2355 | uil 2356 | k_s 2357 | nel 2358 | ldi 2359 | nen 2360 | nei 2361 | flu 2362 | ies 2363 | wne 2364 | wnf 2365 | iev 2366 | k_c 2367 | k_e 2368 | uiv 2369 | rri 2370 | uit 2371 | uis 2372 | yor 2373 | _on 2374 | ad# 2375 | tin 2376 | tio 2377 | til 2378 | tim 2379 | ied 2380 | rre 2381 | tif 2382 | rry 2383 | tie 2384 | iel 2385 | tic 2386 | hes 2387 | tia 2388 | _gi 2389 | nct 2390 | heu 2391 | sif 2392 | tiv 2393 | tit 2394 | tir 2395 | tis 2396 | ys# 2397 | hea 2398 | hef 2399 | hed 2400 | ne# 2401 | que 2402 | hei 2403 | hen 2404 | heo 2405 | ni_ 2406 | hem 2407 | #pi 2408 | som 2409 | adq 2410 | son 2411 | wn# 2412 | rid 2413 | soc 2414 | ces 2415 | rig 2416 | sod 2417 | adh 2418 | oin 2419 | adm 2420 | pul 2421 | adc 2422 | nit 2423 | me# 2424 | niv 2425 | ro_ 2426 | t_l 2427 | ade 2428 | add 2429 | nis 2430 | sou 2431 | cee 2432 | t_f 2433 | ad_ 2434 | t_d 2435 | t_e 2436 | niz 2437 | t_c 2438 | rol 2439 | t_a 2440 | ron 2441 | nig 2442 | roa 2443 | rob 2444 | roc 2445 | rod 2446 | nim 2447 | nin 2448 | wro 2449 | t_t 2450 | wri 2451 | t_r 2452 | t_s 2453 | t_p 2454 | bel 2455 | le# 2456 | ben 2457 | rop 2458 | ror 2459 | ros 2460 | rot 2461 | rou 2462 | rov 2463 | row 2464 | beg 2465 | wry 2466 | _me 2467 | yss 2468 | met 2469 | mes 2470 | mer 2471 | o_g 2472 | 800 2473 | yst 2474 | _mo 2475 | mez 2476 | jus 2477 | kni 2478 | med 2479 | his 2480 | hip 2481 | ed# 2482 | kno 2483 | n_w 2484 | hit 2485 | mem 2486 | n_r 2487 | n_s 2488 | lev 2489 | ely 2490 | let 2491 | hib 2492 | hic 2493 | les 2494 | n_e 2495 | n_f 2496 | n_g 2497 | gna 2498 | wif 2499 | elp 2500 | n_c 2501 | n_l 2502 | lex 2503 | hin 2504 | hio 2505 | hil 2506 | ssi 2507 | lee 2508 | leb 2509 | lec 2510 | ele 2511 | lea 2512 | wit 2513 | ssa 2514 | be# 2515 | cip 2516 | cis 2517 | sse 2518 | uni 2519 | cit 2520 | ung 2521 | won 2522 | une 2523 | und 2524 | unc 2525 | coa 2526 | yla 2527 | cil 2528 | get 2529 | cin 2530 | cia 2531 | edl 2532 | ges 2533 | pri 2534 | edi 2535 | pro 2536 | rsh 2537 | rsi 2538 | unt 2539 | uns 2540 | edb 2541 | eda 2542 | rso 2543 | ed_ 2544 | pre 2545 | bir 2546 | wor 2547 | rse 2548 | ddy 2549 | ge_ 2550 | voi 2551 | za# 2552 | ddr 2553 | bil 2554 | fet 2555 | bin 2556 | bio 2557 | bia 2558 | bib 2559 | rst 2560 | fes 2561 | fel 2562 | ddi 2563 | big 2564 | h_m 2565 | #ir 2566 | mit 2567 | mis 2568 | ur# 2569 | fee 2570 | ont 2571 | ss# 2572 | fea 2573 | c_p 2574 | fec 2575 | c_r 2576 | c_t 2577 | mig 2578 | ony 2579 | c_h 2580 | mic 2581 | ghb 2582 | in# 2583 | ond 2584 | pic 2585 | min 2586 | ona 2587 | mil 2588 | onc 2589 | onb 2590 | onm 2591 | c_d 2592 | liv 2593 | c_f 2594 | oni 2595 | lis 2596 | pir 2597 | rs# 2598 | ge# 2599 | liz 2600 | pok 2601 | poi 2602 | on_ 2603 | lig 2604 | pol 2605 | lie 2606 | lib 2607 | lic 2608 | t2# 2609 | lia 2610 | lin 2611 | lio 2612 | urn 2613 | nna 2614 | lik 2615 | rld 2616 | uri 2617 | rlf 2618 | nno 2619 | urf 2620 | ure 2621 | por 2622 | urc 2623 | pop 2624 | ura 2625 | pow 2626 | pot 2627 | pou 2628 | inu 2629 | rls 2630 | gis 2631 | gir 2632 | rup 2633 | gio 2634 | m_f 2635 | m_e 2636 | m_d 2637 | urr 2638 | inb 2639 | _ba 2640 | m_o 2641 | inf 2642 | on# 2643 | ind 2644 | _bo 2645 | inj 2646 | tri 2647 | sfe 2648 | inn 2649 | tre 2650 | l_u 2651 | l_r 2652 | l_s 2653 | tra 2654 | ee# 2655 | fiv 2656 | 10# 2657 | xpe 2658 | fir 2659 | try 2660 | jou 2661 | xpo 2662 | l_g 2663 | tru 2664 | l_e 2665 | rly 2666 | l_c 2667 | fid 2668 | fie 2669 | ort 2670 | l_o 2671 | l_l 2672 | 101 2673 | orp 2674 | fic 2675 | l_h 2676 | hni 2677 | _ga 2678 | ki# 2679 | spi 2680 | sph 2681 | spy 2682 | 0fi 2683 | ore 2684 | ord 2685 | org 2686 | cur 2687 | spa 2688 | de_ 2689 | orc 2690 | g_s 2691 | orm 2692 | orl 2693 | oro 2694 | orn 2695 | ori 2696 | 009 2697 | ork 2698 | eet 2699 | ees 2700 | g_e 2701 | g_d 2702 | g_g 2703 | to_ 2704 | g_a 2705 | icl 2706 | g_c 2707 | eek 2708 | or_ 2709 | pso 2710 | rph 2711 | rpi 2712 | uot 2713 | eed 2714 | eec 2715 | f_t 2716 | f_u 2717 | ton 2718 | f_w 2719 | but 2720 | tom 2721 | wls 2722 | f_s 2723 | rpe 2724 | rag 2725 | nro 2726 | f_o 2727 | tod 2728 | bum 2729 | irs 2730 | f_d 2731 | ln# 2732 | irp 2733 | f_g 2734 | deo 2735 | rie 2736 | f_b 2737 | buf 2738 | #dn 2739 | dei 2740 | def 2741 | deg 2742 | _ve 2743 | tor 2744 | deb 2745 | oic 2746 | _va 2747 | dea 2748 | oot 2749 | ire 2750 | rar 2751 | a_t 2752 | iri 2753 | a_i 2754 | iro 2755 | a_o 2756 | irm 2757 | irl 2758 | uba 2759 | a_c 2760 | io# 2761 | ood 2762 | scu 2763 | wl# 2764 | mui 2765 | nva 2766 | ool 2767 | oon 2768 | ook 2769 | nic 2770 | ks_ 2771 | nov 2772 | now 2773 | hre 2774 | nor 2775 | hri 2776 | pli 2777 | hro 2778 | plo 2779 | lne 2780 | nod 2781 | pla 2782 | usn 2783 | ple 2784 | nob 2785 | rcu 2786 | nom 2787 | ush 2788 | noo 2789 | use 2790 | usc 2791 | usb 2792 | usa 2793 | cro 2794 | ts_ 2795 | cra 2796 | iou 2797 | iot 2798 | gns 2799 | cre 2800 | klu 2801 | eig 2802 | gni 2803 | wan 2804 | usp 2805 | wie 2806 | gua 2807 | _cl 2808 | _co 2809 | _ci 2810 | _ch 2811 | div 2812 | ion 2813 | _cu 2814 | diu 2815 | dir 2816 | dis 2817 | hie 2818 | us_ 2819 | bro 2820 | 11# 2821 | hot 2822 | hou 2823 | umc 2824 | orh 2825 | _cy 2826 | him 2827 | dif 2828 | did 2829 | die 2830 | hob 2831 | dic 2832 | dia 2833 | ost 2834 | 112 2835 | cir 2836 | _zo 2837 | oss 2838 | hoi 2839 | any 2840 | hoo 2841 | ns_ 2842 | e_v 2843 | mi# 2844 | e_t 2845 | e_s 2846 | ose 2847 | aur 2848 | e_p 2849 | e_o 2850 | e_n 2851 | e_m 2852 | ant 2853 | ank 2854 | cor 2855 | cou 2856 | e_h 2857 | pp_ 2858 | e_f 2859 | ule 2860 | e_d 2861 | e_c 2862 | e_b 2863 | e_a 2864 | 012 2865 | ang 2866 | 010 2867 | 011 2868 | and 2869 | squ 2870 | uly 2871 | cod 2872 | an_ 2873 | d_w 2874 | d_t 2875 | boy 2876 | d_r 2877 | d_s 2878 | d_p 2879 | ulp 2880 | ren 2881 | d_o 2882 | ppe 2883 | d_m 2884 | bor 2885 | bie 2886 | d_h 2887 | sso 2888 | im# 2889 | bow 2890 | nsi 2891 | d_e 2892 | d_b 2893 | ppp 2894 | bol 2895 | d_a 2896 | isp 2897 | boo 2898 | pay 2899 | boa 2900 | ist 2901 | bod 2902 | tly 2903 | urt 2904 | gro 2905 | mow 2906 | mov 2907 | mou 2908 | isc 2909 | ssu 2910 | gre 2911 | olu 2912 | ise 2913 | isd 2914 | _wo 2915 | ols 2916 | _wi 2917 | ish 2918 | iso 2919 | ism 2920 | oly 2921 | mod 2922 | il# 2923 | old 2924 | _wr 2925 | mon 2926 | co# 2927 | olm 2928 | is_ 2929 | lov 2930 | low 2931 | fri 2932 | nds 2933 | lor 2934 | los 2935 | lop 2936 | ns# 2937 | #a# 2938 | ndf 2939 | loy 2940 | ol_ 2941 | log 2942 | ima 2943 | kil 2944 | loc 2945 | ime 2946 | loa 2947 | arp 2948 | loo 2949 | bs_ 2950 | giv 2951 | art 2952 | ark 2953 | upi 2954 | aro 2955 | w1# 2956 | upe 2957 | enu 2958 | ent 2959 | arb 2960 | ms# 2961 | gie 2962 | arg 2963 | eno 2964 | gov 2965 | ard 2966 | ilt 2967 | gor 2968 | ily 2969 | ar_ 2970 | eng 2971 | bre 2972 | ene 2973 | ups 2974 | enc 2975 | egn 2976 | ena 2977 | god 2978 | en_ 2979 | #ap 2980 | #as 2981 | #ar 2982 | #au 2983 | #at 2984 | #aw 2985 | #av 2986 | #ai 2987 | ilm 2988 | ill 2989 | #am 2990 | #al 2991 | up_ 2992 | #an 2993 | #aa 2994 | fig 2995 | #ac 2996 | for 2997 | _up 2998 | #ad 2999 | #ag 3000 | #af 3001 | foo 3002 | irt 3003 | opu 3004 | ais 3005 | air 3006 | ls_ 3007 | hle 3008 | 12# 3009 | ops 3010 | dua 3011 | aij 3012 | kyw 3013 | ain 3014 | ail 3015 | sta 3016 | cs# 3017 | aig 3018 | c_b 3019 | opp 3020 | opl 3021 | opo 3022 | ps# 3023 | cli 3024 | gh# 3025 | ms_ 3026 | clo 3027 | rif 3028 | go# 3029 | na_ 3030 | #u# 3031 | hur 3032 | cle 3033 | #04 3034 | lse 3035 | swe 3036 | dr# 3037 | pel 3038 | oid 3039 | bs# 3040 | ery 3041 | ar# 3042 | pee 3043 | erv 3044 | bli 3045 | ert 3046 | ips 3047 | err 3048 | ll# 3049 | erp 3050 | ero 3051 | ern 3052 | erm 3053 | erl 3054 | eri 3055 | erg 3056 | erf 3057 | _te 3058 | erd 3059 | erc 3060 | #jo 3061 | ipa 3062 | er_ 3063 | #up 3064 | #us 3065 | _to 3066 | _ti 3067 | _th 3068 | gs_ 3069 | #uk 3070 | nio 3071 | _tv 3072 | #ul 3073 | ked 3074 | --------------------------------------------------------------------------------