├── .gitignore ├── 400-120-relu-50N.png ├── README.md ├── dist ├── dist-dssm-time-tf.png ├── dist-plot.py ├── dist-tf.png ├── dist_dssm_tf.sh └── sync-dssm-dist.py ├── single ├── dssm-denser.py ├── dssm.py ├── dssm_v2.py ├── dssm_v3.py └── plot.py └── tf-datasize.png /.gitignore: -------------------------------------------------------------------------------- 1 | data/* 2 | -------------------------------------------------------------------------------- /400-120-relu-50N.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fwd4/dssm/18842289e2eb8447efca2405e1edea080d5f7bcb/400-120-relu-50N.png -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dssm on Tensorflow 2 | 3 | Document at http://liaha.github.io/models/2016/06/21/dssm-on-tensorflow.html 4 | 5 | Please refer to https://www.microsoft.com/en-us/research/project/dssm/ for more information. 6 | 7 | # Notice 8 | Thank everyone for your attention on this project. Two important things that we want to clarify: 9 | - This is not the official repository of DSSM project. 10 | - We are not authorized to provide any training or test data that belongs to original authors. 11 | 12 | Sorry for any inconveniences caused. 13 | -------------------------------------------------------------------------------- /dist/dist-dssm-time-tf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fwd4/dssm/18842289e2eb8447efca2405e1edea080d5f7bcb/dist/dist-dssm-time-tf.png -------------------------------------------------------------------------------- /dist/dist-plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | 6 | num_workers = range(1,8) 7 | time = [769, 400, 370, 300, 285, 260, 242] 8 | 9 | plt.grid() 10 | 11 | barwidth = 0.3 12 | x = np.arange(7) - barwidth / 2 + 1 13 | barlist = plt.bar(x, time, barwidth) 14 | 15 | plt.xticks(num_workers) 16 | plt.title('Training Time Per Epoch') 17 | plt.ylabel("Time (s)") 18 | plt.xlabel("Number of Workers") 19 | plt.xlim(0.5, len(num_workers) + 0.5) 20 | 21 | plt.show() 22 | -------------------------------------------------------------------------------- /dist/dist-tf.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fwd4/dssm/18842289e2eb8447efca2405e1edea080d5f7bcb/dist/dist-tf.png -------------------------------------------------------------------------------- /dist/dist_dssm_tf.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | declare -a HOSTS=("tf20-1" "tf21-1" "tf20-2" "tf21-2" "tf20-3" "tf21-3" "tf20-4" "tf21-4") 4 | declare -a IPS=("10.1.0.200" "10.1.0.204" "10.1.0.201" "10.1.0.205" "10.1.0.202" "10.1.0.206" "10.1.0.203" "10.1.0.207" ) 5 | PORT=2222 6 | USER=tensorflow-20 7 | 8 | # for HOSTNAME in ${HOSTS[@]}; do 9 | # sshpass -p 123 ssh -l ${USER} ${HOSTNAME} "echo 123 | sudo -S reboot" 10 | # done 11 | 12 | # sleep 20 13 | 14 | 15 | # Default Setting 16 | N_HOSTS=8 17 | N_WORKERS=4 18 | N_PS=2 19 | HALT=0 20 | 21 | WORK_DIR=git/tensorflow/tensorflow/models/dssm 22 | 23 | for i in "$@"; do 24 | case $i in 25 | -np=*|--num_ps=*) 26 | N_PS="${i#*=}" 27 | ;; 28 | -nw=*|--num_worker=*) 29 | N_WORKERS="${i#*=}" 30 | ;; 31 | --kill) 32 | HALT=1 33 | ;; 34 | *) 35 | ;; 36 | esac 37 | done 38 | 39 | 40 | if (( $((${N_WORKERS} + ${N_PS})) > ${N_HOSTS} )); then 41 | echo "Total #workers and #parameter servers exceeds available #machines: ${N_HOSTS}" 42 | exit 43 | fi 44 | 45 | 46 | PSS=(${HOSTS[@]:0:${N_PS}}) 47 | WORKERS=(${HOSTS[@]:${N_PS}:${N_WORKERS}}) 48 | 49 | 50 | PS_HOSTS="" 51 | for ((i = 0; i < N_PS - 1; ++i)) ; do 52 | PS_HOSTS="${PS_HOSTS}${IPS[$i]}:${PORT}," 53 | done 54 | PS_HOSTS="${PS_HOSTS}${IPS[$i]}:${PORT}" 55 | echo "ps_hosts=${PS_HOSTS}" 56 | 57 | WORKER_HOSTS="" 58 | for ((i = N_PS; i < N_PS + N_WORKERS - 1; ++i)) ; do 59 | WORKER_HOSTS="${WORKER_HOSTS}${IPS[$i]}:${PORT}," 60 | done 61 | WORKER_HOSTS="${WORKER_HOSTS}${IPS[$i]}:${PORT}" 62 | echo "worker_hosts=${WORKER_HOSTS}" 63 | 64 | echo ${PSS[@]} 65 | echo ${WORKERS[@]} 66 | 67 | # Kill All Existing Tasks 68 | for HOSTNAME in ${HOSTS[@]} ; do 69 | sshpass -p 123 ssh -l ${USER} ${HOSTNAME} "echo 123 | sudo -S fuser -k 2222/tcp" 70 | sshpass -p 123 ssh -l ${USER} ${HOSTNAME} "rm -rf /tmp/dssm-dist" 71 | # sshpass -p 123 ssh -l ${USER} ${HOSTNAME} "echo 123 | sudo -S apt-get update" 72 | # sshpass -p 123 ssh -l ${USER} ${HOSTNAME} "echo 123 | sudo -S apt-get -y upgrade" 73 | sshpass -p 123 scp ~/${WORK_DIR}/dist/sync-dssm-dist.py ${USER}@${HOSTNAME}:~/${WORK_DIR} 74 | echo "$HOSTNAME ready" 75 | done 76 | 77 | if (($HALT)); then 78 | exit 79 | fi 80 | 81 | # Start Parameter Servers 82 | temp=0 83 | for HOSTNAME in ${PSS[@]} ; do 84 | #echo "Starting parameter server on ${HOSTNAME}" 85 | sshpass -p 123 ssh -l ${USER} ${HOSTNAME} \ 86 | "cd git/tensorflow/tensorflow/models/dssm && \ 87 | python sync-dssm-dist.py --ps_hosts='${PS_HOSTS}' \ 88 | --worker_hosts='${WORKER_HOSTS}' \ 89 | --num_workers=${N_WORKERS} --job_name=ps --task_index=${temp}" & 90 | echo "Started parameter server ${temp} on ${HOSTNAME}" 91 | temp=$((temp+1)) 92 | done 93 | 94 | # Start Workers 95 | temp=0 96 | for HOSTNAME in ${WORKERS[@]} ; do 97 | # echo "python sync-dssm-dist.py --ps_hosts='${PS_HOSTS}' \ 98 | # --worker_hosts='${WORKER_HOSTS}' \ 99 | # --num_workers=${N_WORKERS} --job_name=worker --task_index=${temp}" 100 | sshpass -p 123 ssh -l ${USER} ${HOSTNAME} \ 101 | "cd git/tensorflow/tensorflow/models/dssm && \ 102 | python sync-dssm-dist.py --ps_hosts='${PS_HOSTS}' \ 103 | --worker_hosts='${WORKER_HOSTS}' \ 104 | --num_workers=${N_WORKERS} --job_name=worker --task_index=${temp} > run.log" & 105 | echo "Starting worker ${temp} on ${HOSTNAME}" 106 | temp=$((temp+1)) 107 | done 108 | -------------------------------------------------------------------------------- /dist/sync-dssm-dist.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import random 3 | import time 4 | 5 | import numpy as np 6 | 7 | import tensorflow as tf 8 | 9 | flags = tf.app.flags 10 | FLAGS = flags.FLAGS 11 | 12 | # Flags for defining the tf.train.ClusterSpec 13 | flags.DEFINE_string("ps_hosts", "", "Comma-separated list of hostname:port pairs") 14 | flags.DEFINE_string("worker_hosts", "", "Comma-separated list of hostname:port pairs") 15 | # Flags for defining the tf.train.Server 16 | flags.DEFINE_string("job_name", "", "One of 'ps', 'worker'") 17 | flags.DEFINE_integer("task_index", 0, "Index of task within the job") 18 | flags.DEFINE_integer("num_workers", "", "Number of workers ") 19 | 20 | flags.DEFINE_string('summaries_dir', '/tmp/dssm-dist', 'Summaries directory') 21 | flags.DEFINE_float('learning_rate', 0.1, 'Initial learning rate.') 22 | flags.DEFINE_integer('max_steps', 50000, 'Number of steps to run trainer.') 23 | flags.DEFINE_integer('epoch_steps', 1000, "Number of steps in one epoch") 24 | # FLAGS.max_steps = FLAGS.num_workers 25 | # FLAGS.epoch_steps = FLAGS.num_workers 26 | # flags.DEFINE_integer('gama', 20, 'Multiply cosine similarity by gama before softmax') 27 | flags.DEFINE_bool('gpu', 1, "Enable GPU or not") 28 | 29 | start = time.time() 30 | doc_train_data = pickle.load(open('doc.train.pickle', 'rb')).tocsr() 31 | query_train_data = pickle.load(open('query.train.pickle', 'rb')).tocsr() 32 | doc_test_data = pickle.load(open('doc.test.pickle', 'rb')).tocsr() 33 | query_test_data = pickle.load(open('query.test.pickle', 'rb')).tocsr() 34 | end = time.time() 35 | print("Number of batches per epoch per worker: %d " % FLAGS.epoch_steps) 36 | print("Loading data from HDD to memory: %f s" % (end - start)) 37 | 38 | FLAGS.learning_rate /= np.sqrt(FLAGS.num_workers) 39 | FLAGS.max_steps //= FLAGS.num_workers 40 | FLAGS.epoch_steps //= FLAGS.num_workers 41 | 42 | BS = 1024 # // FLAGS.num_workers 43 | 44 | TRIGRAM_D = doc_train_data.shape[1] 45 | NEG = 50 46 | 47 | L1_N = 400 48 | L2_N = 120 49 | GAMA = 20 50 | 51 | 52 | def pull_batch(query_data, doc_data, batch_idx): 53 | query_in = query_data[batch_idx * BS:(batch_idx + 1) * BS, :] 54 | doc_in = doc_data[batch_idx * BS:(batch_idx + 1) * BS, :] 55 | cols = np.unique(np.concatenate((query_in.tocoo().col.T, doc_in.tocoo().col.T), axis=0)) 56 | # print(query_in.shape) 57 | # print(doc_in.shape) 58 | query_in = query_in[:, cols].tocoo() 59 | doc_in = doc_in[:, cols].tocoo() 60 | 61 | query_in = tf.SparseTensorValue( 62 | np.transpose([np.array(query_in.row, dtype=np.int64), np.array(query_in.col, dtype=np.int64)]), 63 | np.array(query_in.data, dtype=np.float), 64 | np.array(query_in.shape, dtype=np.int64)) 65 | doc_in = tf.SparseTensorValue( 66 | np.transpose([np.array(doc_in.row, dtype=np.int64), np.array(doc_in.col, dtype=np.int64)]), 67 | np.array(doc_in.data, dtype=np.float), 68 | np.array(doc_in.shape, dtype=np.int64)) 69 | 70 | return query_in, doc_in, cols 71 | 72 | 73 | def feed_dict(Train, batch_idx): 74 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 75 | if Train: 76 | query_in, doc_in, cols = pull_batch(query_train_data, doc_train_data, batch_idx) 77 | else: 78 | query_in, doc_in, cols = pull_batch(query_test_data, doc_test_data, batch_idx) 79 | return {query_batch: query_in, doc_batch: doc_in, dense_cols: cols} 80 | 81 | 82 | def variable_summaries(var, name): 83 | """Attach a lot of summaries to a Tensor.""" 84 | with tf.name_scope('summaries'): 85 | mean = tf.reduce_mean(var) 86 | tf.scalar_summary('mean/' + name, mean) 87 | with tf.name_scope('stddev'): 88 | stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean))) 89 | tf.scalar_summary('sttdev/' + name, stddev) 90 | tf.scalar_summary('max/' + name, tf.reduce_max(var)) 91 | tf.scalar_summary('min/' + name, tf.reduce_min(var)) 92 | tf.histogram_summary(name, var) 93 | 94 | 95 | query_in_shape = np.array([BS, TRIGRAM_D]).astype('int64') 96 | doc_in_shape = np.array([BS, TRIGRAM_D]).astype('int64') 97 | 98 | ps_hosts = FLAGS.ps_hosts.split(",") 99 | worker_hosts = FLAGS.worker_hosts.split(",") 100 | 101 | # Create a cluster from the parameter server and worker hosts 102 | cluster = tf.train.ClusterSpec({"worker": worker_hosts, "ps": ps_hosts}) 103 | # Cteate and start a server for the local task. 104 | server = tf.train.Server(cluster, job_name=FLAGS.job_name, task_index=FLAGS.task_index) 105 | 106 | if FLAGS.job_name == "ps": 107 | server.join() 108 | elif FLAGS.job_name == "worker": 109 | with tf.device(tf.train.replica_device_setter( # cluster = cluster)): 110 | worker_device="/job:worker/task:%d" % FLAGS.task_index, cluster=cluster)): 111 | 112 | global_step = tf.Variable(0, name="global_step", trainable=False) 113 | # quit() 114 | with tf.name_scope('input'): 115 | # Shape [BS, TRIGRAM_D]. 116 | query_batch = tf.sparse_placeholder(tf.float32, shape=query_in_shape, name='QueryBatch') 117 | # Shape [BS, TRIGRAM_D] 118 | doc_batch = tf.sparse_placeholder(tf.float32, shape=doc_in_shape, name='DocBatch') 119 | 120 | dense_cols = tf.placeholder(tf.int64, shape=None, name='ActualCols') 121 | 122 | with tf.name_scope('L1'): 123 | # Hidden layer 1 [input: cols, output: 300] 124 | l1_par_range = np.sqrt(6.0 / (TRIGRAM_D + L1_N)) 125 | 126 | #partitioner = tf.variable_axis_size_partitioner(16<<20-1) 127 | #weight1 = tf.get_variable("weight", [TRIGRAM_D, L1_N], 128 | # initializer=tf.random_uniform_initializer( 129 | # minval=-l1_par_range, 130 | # maxval=l1_par_range, 131 | # dtype=tf.float32), 132 | # partitioner=partitioner) 133 | weight1 = tf.Variable(tf.random_uniform([TRIGRAM_D, L1_N], -l1_par_range, l1_par_range), 134 | name='weight') 135 | bias1 = tf.Variable(tf.random_uniform([L1_N], -l1_par_range, l1_par_range), 136 | name='bias') 137 | variable_summaries(weight1, 'L1_weights') 138 | variable_summaries(bias1, 'L1_biases') 139 | with tf.device(weight1.device): 140 | dense_w1 = tf.gather(weight1, dense_cols, name="L1_dense_w") 141 | # query_l1 = tf.matmul(tf.to_float(query_batch),weight1)+bias1 142 | query_l1 = tf.sparse_tensor_dense_matmul(query_batch, dense_w1) + bias1 143 | # doc_l1 = tf.matmul(tf.to_float(doc_batch),weight1)+bias1 144 | doc_l1 = tf.sparse_tensor_dense_matmul(doc_batch, dense_w1) + bias1 145 | 146 | query_l1_out = tf.nn.relu(query_l1) 147 | doc_l1_out = tf.nn.relu(doc_l1) 148 | 149 | with tf.name_scope('L2'): 150 | # Hidden layer 2 [input: 300, output: 300] 151 | l2_par_range = np.sqrt(6.0 / (L1_N + L2_N)) 152 | weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], -l2_par_range, l2_par_range), 153 | name='weight') 154 | bias2 = tf.Variable(tf.random_uniform([L2_N], -l2_par_range, l2_par_range), 155 | name='bias') 156 | variable_summaries(weight2, 'L2_weights') 157 | variable_summaries(bias2, 'L2_biases') 158 | 159 | query_l2 = tf.matmul(query_l1_out, weight2) + bias2 160 | doc_l2 = tf.matmul(doc_l1_out, weight2) + bias2 161 | query_y = tf.nn.relu(query_l2) 162 | doc_y = tf.nn.relu(doc_l2) 163 | 164 | with tf.name_scope('FD_rotate'): 165 | # Rotate FD+ to produce 50 FD- 166 | temp = tf.tile(doc_y, [1, 1]) 167 | for i in range(NEG): 168 | rand = int((random.random() + i) * BS / NEG) 169 | doc_y = tf.concat(0, 170 | [doc_y, 171 | tf.slice(temp, [rand, 0], [BS - rand, -1]), 172 | tf.slice(temp, [0, 0], [rand, -1])]) 173 | 174 | with tf.name_scope('Cosine_Similarity'): 175 | # Cosine similarity 176 | query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)), [NEG + 1, 1]) 177 | doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True)) 178 | 179 | prod = tf.reduce_sum(tf.mul(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1, True) 180 | norm_prod = tf.mul(query_norm, doc_norm) 181 | 182 | cos_sim_raw = tf.truediv(prod, norm_prod) 183 | cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw), [NEG + 1, BS])) * GAMA 184 | 185 | with tf.name_scope('Loss'): 186 | # Train Loss 187 | prob = tf.nn.softmax((cos_sim)) 188 | hit_prob = tf.slice(prob, [0, 0], [-1, 1]) 189 | loss = -tf.reduce_sum(tf.log(hit_prob)) / BS 190 | tf.scalar_summary('loss', loss) 191 | 192 | with tf.name_scope('Training'): 193 | # Optimizer 194 | # train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate)\ 195 | # .minimize(loss) 196 | opt = tf.train.AdagradOptimizer(FLAGS.learning_rate) 197 | opt = tf.train.SyncReplicasOptimizer(opt, 198 | replicas_to_aggregate=FLAGS.num_workers, 199 | total_num_replicas=FLAGS.num_workers, 200 | replica_id=FLAGS.task_index, 201 | name="dssm_sync_replicas") 202 | train_step = opt.minimize(loss, global_step=global_step) 203 | 204 | # merged = tf.merge_all_summaries() 205 | 206 | with tf.name_scope('Test'): 207 | average_loss = tf.placeholder(tf.float32) 208 | loss_summary = tf.scalar_summary('average_loss', average_loss) 209 | 210 | if (FLAGS.task_index == 0): 211 | chief_queue_runner = opt.get_chief_queue_runner() 212 | init_tokens_op = opt.get_init_tokens_op() 213 | 214 | init_op = tf.initialize_all_variables() 215 | 216 | saver = tf.train.Saver(tf.all_variables(), max_to_keep=50) 217 | 218 | sv = tf.train.Supervisor(is_chief=(FLAGS.task_index == 0), 219 | logdir="/tmp/dssm-dist", 220 | init_op=init_op, 221 | saver=None, 222 | global_step=global_step, 223 | summary_op=None) 224 | # save_model_secs=60) 225 | 226 | # if not FLAGS.gpu: 227 | # config = tf.ConfigProto(device_count= {'GPU' : 0}) 228 | # else: 229 | config = tf.ConfigProto() # log_device_placement=True) 230 | config.gpu_options.allow_growth = True 231 | 232 | # print (FLAGS.gpu) 233 | 234 | # iter = [] 235 | # train_loss = [] 236 | # test_loss = [] 237 | 238 | with sv.managed_session(server.target, config=config) as sess: 239 | if FLAGS.task_index == 0: 240 | print("Starting chief queue runner and running init_tokens_op") 241 | sv.start_queue_runners(sess, [chief_queue_runner]) 242 | sess.run(init_tokens_op) 243 | 244 | step = 0 245 | start = time.time() 246 | local_step = 0 247 | train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) 248 | test_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/test', sess.graph) 249 | 250 | logfile = open('dist.log','w') 251 | 252 | while not sv.should_stop() and step < FLAGS.max_steps: 253 | 254 | batch_idx = step % FLAGS.epoch_steps 255 | temp = feed_dict(True, batch_idx * FLAGS.num_workers + FLAGS.task_index) 256 | 257 | _, step = sess.run([train_step, global_step], feed_dict=temp) 258 | local_step += 1 259 | now = time.time() 260 | if step % 10 == 0: 261 | logfile.write("%.2f: Worker %d: training step %d done (global step: %d)\n" % 262 | (now, FLAGS.task_index, local_step, step)) 263 | if (step + 1) % FLAGS.epoch_steps == 0: # or (step < FLAGS.epoch_steps and step %100 ==0): 264 | end = time.time() 265 | print("PureTrainTime: %-4.3fs" % (end - start)) 266 | epoch_loss = 0 267 | for i in range(FLAGS.epoch_steps): 268 | loss_v = sess.run(loss, 269 | feed_dict= 270 | feed_dict(True, i * FLAGS.num_workers + FLAGS.task_index)) 271 | epoch_loss += loss_v 272 | 273 | epoch_loss /= FLAGS.epoch_steps 274 | #iter.append(1.0*step/FLAGS.epoch_steps) 275 | #train_loss.append(epoch_loss) 276 | train_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 277 | train_writer.add_summary(train_loss, step + 1) 278 | logfile.write("Epoch #%-2.3f | Train Loss: %-4.3f | PureTrainTime: %-3.3fs\n" % 279 | (step / FLAGS.epoch_steps, epoch_loss, end - start)) 280 | print ("Epoch #%-2.3f | Train Loss: %-4.3f | PureTrainTime: %-3.3fs" % 281 | (step / FLAGS.epoch_steps, epoch_loss, end - start)) 282 | 283 | epoch_loss = 0 284 | for i in range(FLAGS.epoch_steps): 285 | loss_v = sess.run(loss, 286 | feed_dict= 287 | feed_dict(False, i * FLAGS.num_workers + FLAGS.task_index)) 288 | epoch_loss += loss_v 289 | 290 | epoch_loss /= FLAGS.epoch_steps 291 | start = time.time() 292 | #test_loss.append(epoch_loss) 293 | 294 | test_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 295 | test_writer.add_summary(test_loss, step + 1) 296 | logfile.write("Epoch #%-5d | Test Loss: %-4.3f | Calc_LossTime: %-3.3fs\n" % 297 | (step / FLAGS.epoch_steps, epoch_loss, start - end)) 298 | print("Epoch #%-5d | Test Loss: %-4.3f | Calc_LossTime: %-3.3fs" % 299 | (step / FLAGS.epoch_steps, epoch_loss, start - end)) 300 | # save_path = saver.save(sess, "/tmp/dssm-dist/model.ckpt", global_step=step / FLAGS.epoch_steps) 301 | # print("Epoch %d trained in %.2fs. Model saved in file: %s" % ( 302 | # step / FLAGS.epoch_steps, end - start, save_path)) 303 | start = time.time() 304 | print("Worker %d done!\n" % FLAGS.task_index) 305 | 306 | sv.stop() 307 | 308 | # print("[") 309 | # for i in range(len(iter)): 310 | # print("%2.2f, " % iter[i]) 311 | # print("]\n[") 312 | # for i in range(len(iter)): 313 | # print("%3.3f, " % train_loss[i]) 314 | # print("]\n[") 315 | # for i in range(len(iter)): 316 | # print("%3.3f, " % test_loss[i]) 317 | # print("]") 318 | -------------------------------------------------------------------------------- /single/dssm-denser.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import random 3 | import time 4 | import sys 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | flags = tf.app.flags 9 | FLAGS = flags.FLAGS 10 | 11 | flags.DEFINE_string('summaries_dir', '/tmp/dssm-400-128-relu', 'Summaries directory') 12 | flags.DEFINE_float('learning_rate', 0.1, 'Initial learning rate.') 13 | flags.DEFINE_integer('max_steps', 900000, 'Number of steps to run trainer.') 14 | flags.DEFINE_integer('epoch_steps', 18000, "Number of steps in one epoch.") 15 | flags.DEFINE_integer('pack_size', 2000, "Number of batches in one pickle pack.") 16 | flags.DEFINE_bool('gpu', 1, "Enable GPU or not") 17 | 18 | start = time.time() 19 | 20 | doc_train_data = None 21 | query_train_data = None 22 | 23 | # load test data for now 24 | query_test_data = pickle.load(open('../data/query.test.1.pickle', 'rb')).tocsr() 25 | doc_test_data = pickle.load(open('../data/doc.test.1.pickle', 'rb')).tocsr() 26 | 27 | def load_train_data(pack_idx): 28 | global doc_train_data, query_train_data 29 | doc_train_data = None 30 | query_train_data = None 31 | start = time.time() 32 | doc_train_data = pickle.load(open('../data/doc.train.' + str(pack_idx)+ '.pickle', 'rb')).tocsr() 33 | query_train_data = pickle.load(open('../data/query.train.'+ str(pack_idx)+ '.pickle', 'rb')).tocsr() 34 | end = time.time() 35 | print ("\nTrain data %d/9 is loaded in %.2fs" % (pack_idx, end - start)) 36 | 37 | end = time.time() 38 | print("Loading data from HDD to memory: %.2fs" % (end - start)) 39 | 40 | TRIGRAM_D = 49284 41 | 42 | NEG = 50 43 | BS = 1024 44 | 45 | L1_N = 400 46 | L2_N = 120 47 | 48 | # query_in_shape = np.array([BS, TRIGRAM_D], np.int64) 49 | # doc_in_shape = np.array([BS, TRIGRAM_D], np.int64) 50 | 51 | 52 | def variable_summaries(var, name): 53 | """Attach a lot of summaries to a Tensor.""" 54 | with tf.name_scope('summaries'): 55 | mean = tf.reduce_mean(var) 56 | tf.scalar_summary('mean/' + name, mean) 57 | with tf.name_scope('stddev'): 58 | stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean))) 59 | tf.scalar_summary('sttdev/' + name, stddev) 60 | tf.scalar_summary('max/' + name, tf.reduce_max(var)) 61 | tf.scalar_summary('min/' + name, tf.reduce_min(var)) 62 | tf.histogram_summary(name, var) 63 | 64 | 65 | with tf.name_scope('input'): 66 | # Shape [BS, TRIGRAM_D]. 67 | query_batch = tf.sparse_placeholder(tf.float32, shape=None, name='QueryBatch') 68 | # Shape [BS, TRIGRAM_D] 69 | doc_batch = tf.sparse_placeholder(tf.float32, shape=None, name='DocBatch') 70 | 71 | dense_cols = tf.placeholder(tf.int64, shape=None, name='ActualCols') 72 | 73 | with tf.name_scope('L1'): 74 | l1_par_range = np.sqrt(6.0 / (TRIGRAM_D + L1_N)) 75 | weight1 = tf.Variable(tf.random_uniform([TRIGRAM_D, L1_N], -l1_par_range, l1_par_range)) 76 | bias1 = tf.Variable(tf.random_uniform([L1_N], -l1_par_range, l1_par_range)) 77 | variable_summaries(weight1, 'L1_weights') 78 | variable_summaries(bias1, 'L1_biases') 79 | 80 | dense_w1 = tf.gather(weight1, dense_cols, name="L1_dense_w") 81 | 82 | query_l1 = tf.sparse_tensor_dense_matmul(query_batch, dense_w1) + bias1 83 | doc_l1 = tf.sparse_tensor_dense_matmul(doc_batch, dense_w1) + bias1 84 | 85 | query_l1_out = tf.nn.relu(query_l1) 86 | doc_l1_out = tf.nn.relu(doc_l1) 87 | 88 | with tf.name_scope('L2'): 89 | l2_par_range = np.sqrt(6.0 / (L1_N + L2_N)) 90 | 91 | weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], -l2_par_range, l2_par_range)) 92 | bias2 = tf.Variable(tf.random_uniform([L2_N], -l2_par_range, l2_par_range)) 93 | variable_summaries(weight2, 'L2_weights') 94 | variable_summaries(bias2, 'L2_biases') 95 | 96 | query_l2 = tf.matmul(query_l1_out, weight2) + bias2 97 | doc_l2 = tf.matmul(doc_l1_out, weight2) + bias2 98 | query_y = tf.nn.relu(query_l2) 99 | doc_y = tf.nn.relu(doc_l2) 100 | 101 | with tf.name_scope('FD_rotate'): 102 | # Rotate FD+ to produce 50 FD- 103 | temp = tf.tile(doc_y, [1, 1]) 104 | 105 | for i in range(NEG): 106 | rand = int((random.random() + i) * BS / NEG) 107 | doc_y = tf.concat(0, 108 | [doc_y, 109 | tf.slice(temp, [rand, 0], [BS - rand, -1]), 110 | tf.slice(temp, [0, 0], [rand, -1])]) 111 | 112 | with tf.name_scope('Cosine_Similarity'): 113 | # Cosine similarity 114 | query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)), [NEG + 1, 1]) 115 | doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True)) 116 | 117 | prod = tf.reduce_sum(tf.mul(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1, True) 118 | norm_prod = tf.mul(query_norm, doc_norm) 119 | 120 | cos_sim_raw = tf.truediv(prod, norm_prod) 121 | cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw), [NEG + 1, BS])) * 20 122 | 123 | with tf.name_scope('Loss'): 124 | # Train Loss 125 | prob = tf.nn.softmax((cos_sim)) 126 | hit_prob = tf.slice(prob, [0, 0], [-1, 1]) 127 | loss = -tf.reduce_sum(tf.log(hit_prob)) / BS 128 | tf.scalar_summary('loss', loss) 129 | 130 | with tf.name_scope('Training'): 131 | # Optimizer 132 | train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(loss) 133 | 134 | # with tf.name_scope('Accuracy'): 135 | # correct_prediction = tf.equal(tf.argmax(prob, 1), 0) 136 | # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 137 | # tf.scalar_summary('accuracy', accuracy) 138 | 139 | merged = tf.merge_all_summaries() 140 | 141 | with tf.name_scope('Test'): 142 | average_loss = tf.placeholder(tf.float32) 143 | loss_summary = tf.scalar_summary('average_loss', average_loss) 144 | 145 | 146 | def pull_batch(query_data, doc_data, batch_idx): 147 | #start = time.time() 148 | query_in = query_data[batch_idx * BS:(batch_idx + 1) * BS, :] 149 | doc_in = doc_data[batch_idx * BS:(batch_idx + 1) * BS, :] 150 | cols = np.unique(np.concatenate((query_in.tocoo().col.T, doc_in.tocoo().col.T), axis=0)) 151 | query_in = query_in[:, cols].tocoo() 152 | doc_in = doc_in[:, cols].tocoo() 153 | 154 | #print(1.0 * len(query_in.data) / query_in.shape[0] / query_in.shape[1]) 155 | #print(1.0 * len(doc_in.data) / doc_in.shape[0] / doc_in.shape[1]) 156 | 157 | query_in = tf.SparseTensorValue( 158 | np.transpose([np.array(query_in.row, dtype=np.int64), np.array(query_in.col, dtype=np.int64)]), 159 | np.array(query_in.data, dtype=np.float), 160 | np.array(query_in.shape, dtype=np.int64)) 161 | doc_in = tf.SparseTensorValue( 162 | np.transpose([np.array(doc_in.row, dtype=np.int64), np.array(doc_in.col, dtype=np.int64)]), 163 | np.array(doc_in.data, dtype=np.float), 164 | np.array(doc_in.shape, dtype=np.int64)) 165 | #end = time.time() 166 | #print("Pull_batch time: %f" % (end - start)) 167 | 168 | return query_in, doc_in, cols 169 | 170 | 171 | def feed_dict(Train, batch_idx): 172 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 173 | if Train: 174 | query_in, doc_in, cols = pull_batch(query_train_data, doc_train_data, batch_idx) 175 | else: 176 | query_in, doc_in, cols = pull_batch(query_test_data, doc_test_data, batch_idx) 177 | return {query_batch: query_in, doc_batch: doc_in, dense_cols:cols} 178 | 179 | 180 | config = tf.ConfigProto() # log_device_placement=True) 181 | config.gpu_options.allow_growth = True 182 | 183 | with tf.Session(config=config) as sess: 184 | sess.run(tf.initialize_all_variables()) 185 | train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) 186 | test_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/test', sess.graph) 187 | 188 | # Actual execution 189 | start = time.time() 190 | # fp_time = 0 191 | # fbp_time = 0 192 | for step in range(FLAGS.max_steps): 193 | batch_idx = step % FLAGS.epoch_steps 194 | if batch_idx % FLAGS.pack_size == 0: 195 | load_train_data(batch_idx / FLAGS.pack_size + 1) 196 | 197 | # # setup toolbar 198 | # sys.stdout.write("[%s]" % (" " * toolbar_width)) 199 | # #sys.stdout.flush() 200 | # sys.stdout.write("\b" * (toolbar_width + 1)) # return to start of line, after '[' 201 | 202 | 203 | if batch_idx % (FLAGS.pack_size / 64) == 0: 204 | progress = 100.0 * batch_idx / FLAGS.epoch_steps 205 | sys.stdout.write("\r%.2f%% Epoch" % progress) 206 | sys.stdout.flush() 207 | 208 | # t1 = time.time() 209 | # sess.run(loss, feed_dict = feed_dict(True, batch_idx)) 210 | # t2 = time.time() 211 | # fp_time += t2 - t1 212 | # #print(t2-t1) 213 | # t1 = time.time() 214 | sess.run(train_step, feed_dict=feed_dict(True, batch_idx % FLAGS.pack_size)) 215 | # t2 = time.time() 216 | # fbp_time += t2 - t1 217 | # #print(t2 - t1) 218 | # if batch_idx % 2000 == 1999: 219 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 220 | # (fp_time / step, fbp_time / step)) 221 | 222 | 223 | if batch_idx == FLAGS.epoch_steps - 1: 224 | end = time.time() 225 | epoch_loss = 0 226 | for i in range(FLAGS.pack_size): 227 | loss_v = sess.run(loss, feed_dict=feed_dict(True, i)) 228 | epoch_loss += loss_v 229 | 230 | epoch_loss /= FLAGS.pack_size 231 | train_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 232 | train_writer.add_summary(train_loss, step + 1) 233 | 234 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 235 | # (fp_time / step, fbp_time / step)) 236 | # 237 | print ("\nEpoch #%-5d | Train Loss: %-4.3f | PureTrainTime: %-3.3fs" % 238 | (step / FLAGS.epoch_steps, epoch_loss, end - start)) 239 | 240 | epoch_loss = 0 241 | for i in range(FLAGS.pack_size): 242 | loss_v = sess.run(loss, feed_dict=feed_dict(False, i)) 243 | epoch_loss += loss_v 244 | 245 | epoch_loss /= FLAGS.pack_size 246 | 247 | test_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 248 | test_writer.add_summary(test_loss, step + 1) 249 | 250 | start = time.time() 251 | print ("Epoch #%-5d | Test Loss: %-4.3f | Calc_LossTime: %-3.3fs" % 252 | (step / FLAGS.epoch_steps, epoch_loss, start - end)) 253 | 254 | -------------------------------------------------------------------------------- /single/dssm.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import random 3 | import time 4 | import sys 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | flags = tf.app.flags 9 | FLAGS = flags.FLAGS 10 | 11 | flags.DEFINE_string('summaries_dir', '/tmp/dssm-400-120-relu', 'Summaries directory') 12 | flags.DEFINE_float('learning_rate', 0.1, 'Initial learning rate.') 13 | flags.DEFINE_integer('max_steps', 900000, 'Number of steps to run trainer.') 14 | flags.DEFINE_integer('epoch_steps', 18000, "Number of steps in one epoch.") 15 | flags.DEFINE_integer('pack_size', 2000, "Number of batches in one pickle pack.") 16 | flags.DEFINE_bool('gpu', 1, "Enable GPU or not") 17 | 18 | start = time.time() 19 | 20 | doc_train_data = None 21 | query_train_data = None 22 | 23 | # load test data for now 24 | query_test_data = pickle.load(open('../data/query.test.1.pickle', 'rb')).tocsr() 25 | doc_test_data = pickle.load(open('../data/doc.test.1.pickle', 'rb')).tocsr() 26 | 27 | def load_train_data(pack_idx): 28 | global doc_train_data, query_train_data 29 | doc_train_data = None 30 | query_train_data = None 31 | start = time.time() 32 | doc_train_data = pickle.load(open('../data/doc.train.' + str(pack_idx)+ '.pickle', 'rb')).tocsr() 33 | query_train_data = pickle.load(open('../data/query.train.'+ str(pack_idx)+ '.pickle', 'rb')).tocsr() 34 | end = time.time() 35 | print ("\nTrain data %d/9 is loaded in %.2fs" % (pack_idx, end - start)) 36 | 37 | end = time.time() 38 | print("Loading data from HDD to memory: %.2fs" % (end - start)) 39 | 40 | TRIGRAM_D = 49284 41 | 42 | NEG = 50 43 | BS = 1000 44 | 45 | L1_N = 400 46 | L2_N = 120 47 | 48 | query_in_shape = np.array([BS, TRIGRAM_D], np.int64) 49 | doc_in_shape = np.array([BS, TRIGRAM_D], np.int64) 50 | 51 | 52 | def variable_summaries(var, name): 53 | """Attach a lot of summaries to a Tensor.""" 54 | with tf.name_scope('summaries'): 55 | mean = tf.reduce_mean(var) 56 | tf.scalar_summary('mean/' + name, mean) 57 | with tf.name_scope('stddev'): 58 | stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean))) 59 | tf.scalar_summary('sttdev/' + name, stddev) 60 | tf.scalar_summary('max/' + name, tf.reduce_max(var)) 61 | tf.scalar_summary('min/' + name, tf.reduce_min(var)) 62 | tf.histogram_summary(name, var) 63 | 64 | 65 | with tf.name_scope('input'): 66 | # Shape [BS, TRIGRAM_D]. 67 | query_batch = tf.sparse_placeholder(tf.float32, shape=query_in_shape, name='QueryBatch') 68 | # Shape [BS, TRIGRAM_D] 69 | doc_batch = tf.sparse_placeholder(tf.float32, shape=doc_in_shape, name='DocBatch') 70 | 71 | with tf.name_scope('L1'): 72 | l1_par_range = np.sqrt(6.0 / (TRIGRAM_D + L1_N)) 73 | weight1 = tf.Variable(tf.random_uniform([TRIGRAM_D, L1_N], -l1_par_range, l1_par_range)) 74 | bias1 = tf.Variable(tf.random_uniform([L1_N], -l1_par_range, l1_par_range)) 75 | variable_summaries(weight1, 'L1_weights') 76 | variable_summaries(bias1, 'L1_biases') 77 | 78 | # query_l1 = tf.matmul(tf.to_float(query_batch),weight1)+bias1 79 | query_l1 = tf.sparse_tensor_dense_matmul(query_batch, weight1) + bias1 80 | # doc_l1 = tf.matmul(tf.to_float(doc_batch),weight1)+bias1 81 | doc_l1 = tf.sparse_tensor_dense_matmul(doc_batch, weight1) + bias1 82 | 83 | query_l1_out = tf.nn.relu(query_l1) 84 | doc_l1_out = tf.nn.relu(doc_l1) 85 | 86 | with tf.name_scope('L2'): 87 | l2_par_range = np.sqrt(6.0 / (L1_N + L2_N)) 88 | 89 | weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], -l2_par_range, l2_par_range)) 90 | bias2 = tf.Variable(tf.random_uniform([L2_N], -l2_par_range, l2_par_range)) 91 | variable_summaries(weight2, 'L2_weights') 92 | variable_summaries(bias2, 'L2_biases') 93 | 94 | query_l2 = tf.matmul(query_l1_out, weight2) + bias2 95 | doc_l2 = tf.matmul(doc_l1_out, weight2) + bias2 96 | query_y = tf.nn.relu(query_l2) 97 | doc_y = tf.nn.relu(doc_l2) 98 | 99 | with tf.name_scope('FD_rotate'): 100 | # Rotate FD+ to produce 50 FD- 101 | temp = tf.tile(doc_y, [1, 1]) 102 | 103 | for i in range(NEG): 104 | rand = int((random.random() + i) * BS / NEG) 105 | doc_y = tf.concat(0, 106 | [doc_y, 107 | tf.slice(temp, [rand, 0], [BS - rand, -1]), 108 | tf.slice(temp, [0, 0], [rand, -1])]) 109 | 110 | with tf.name_scope('Cosine_Similarity'): 111 | # Cosine similarity 112 | query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)), [NEG + 1, 1]) 113 | doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True)) 114 | 115 | prod = tf.reduce_sum(tf.mul(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1, True) 116 | norm_prod = tf.mul(query_norm, doc_norm) 117 | 118 | cos_sim_raw = tf.truediv(prod, norm_prod) 119 | cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw), [NEG + 1, BS])) * 20 120 | 121 | with tf.name_scope('Loss'): 122 | # Train Loss 123 | prob = tf.nn.softmax((cos_sim)) 124 | hit_prob = tf.slice(prob, [0, 0], [-1, 1]) 125 | loss = -tf.reduce_sum(tf.log(hit_prob)) / BS 126 | tf.scalar_summary('loss', loss) 127 | 128 | with tf.name_scope('Training'): 129 | # Optimizer 130 | train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(loss) 131 | 132 | # with tf.name_scope('Accuracy'): 133 | # correct_prediction = tf.equal(tf.argmax(prob, 1), 0) 134 | # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 135 | # tf.scalar_summary('accuracy', accuracy) 136 | 137 | merged = tf.merge_all_summaries() 138 | 139 | with tf.name_scope('Test'): 140 | average_loss = tf.placeholder(tf.float32) 141 | loss_summary = tf.scalar_summary('average_loss', average_loss) 142 | 143 | 144 | def pull_batch(query_data, doc_data, batch_idx): 145 | # start = time.time() 146 | query_in = query_data[batch_idx * BS:(batch_idx + 1) * BS, :] 147 | doc_in = doc_data[batch_idx * BS:(batch_idx + 1) * BS, :] 148 | query_in = query_in.tocoo() 149 | doc_in = doc_in.tocoo() 150 | 151 | 152 | 153 | query_in = tf.SparseTensorValue( 154 | np.transpose([np.array(query_in.row, dtype=np.int64), np.array(query_in.col, dtype=np.int64)]), 155 | np.array(query_in.data, dtype=np.float), 156 | np.array(query_in.shape, dtype=np.int64)) 157 | doc_in = tf.SparseTensorValue( 158 | np.transpose([np.array(doc_in.row, dtype=np.int64), np.array(doc_in.col, dtype=np.int64)]), 159 | np.array(doc_in.data, dtype=np.float), 160 | np.array(doc_in.shape, dtype=np.int64)) 161 | 162 | # end = time.time() 163 | # print("Pull_batch time: %f" % (end - start)) 164 | 165 | return query_in, doc_in 166 | 167 | 168 | def feed_dict(Train, batch_idx): 169 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 170 | if Train: 171 | query_in, doc_in = pull_batch(query_train_data, doc_train_data, batch_idx) 172 | else: 173 | query_in, doc_in = pull_batch(query_test_data, doc_test_data, batch_idx) 174 | return {query_batch: query_in, doc_batch: doc_in} 175 | 176 | 177 | config = tf.ConfigProto() # log_device_placement=True) 178 | config.gpu_options.allow_growth = True 179 | #if not FLAGS.gpu: 180 | #config = tf.ConfigProto(device_count= {'GPU' : 0}) 181 | 182 | with tf.Session(config=config) as sess: 183 | sess.run(tf.initialize_all_variables()) 184 | train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) 185 | test_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/test', sess.graph) 186 | 187 | # Actual execution 188 | start = time.time() 189 | # fp_time = 0 190 | # fbp_time = 0 191 | for step in range(FLAGS.max_steps): 192 | batch_idx = step % FLAGS.epoch_steps 193 | if batch_idx % FLAGS.pack_size == 0: 194 | load_train_data(batch_idx / FLAGS.pack_size + 1) 195 | 196 | # # setup toolbar 197 | # sys.stdout.write("[%s]" % (" " * toolbar_width)) 198 | # #sys.stdout.flush() 199 | # sys.stdout.write("\b" * (toolbar_width + 1)) # return to start of line, after '[' 200 | 201 | 202 | if batch_idx % (FLAGS.pack_size / 64) == 0: 203 | progress = 100.0 * batch_idx / FLAGS.epoch_steps 204 | sys.stdout.write("\r%.2f%% Epoch" % progress) 205 | sys.stdout.flush() 206 | 207 | # t1 = time.time() 208 | # sess.run(loss, feed_dict = feed_dict(True, batch_idx)) 209 | # t2 = time.time() 210 | # fp_time += t2 - t1 211 | # #print(t2-t1) 212 | # t1 = time.time() 213 | sess.run(train_step, feed_dict=feed_dict(True, batch_idx % FLAGS.pack_size)) 214 | # t2 = time.time() 215 | # fbp_time += t2 - t1 216 | # #print(t2 - t1) 217 | # if batch_idx % 2000 == 1999: 218 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 219 | # (fp_time / step, fbp_time / step)) 220 | 221 | 222 | if batch_idx == FLAGS.epoch_steps - 1: 223 | end = time.time() 224 | epoch_loss = 0 225 | for i in range(FLAGS.pack_size): 226 | loss_v = sess.run(loss, feed_dict=feed_dict(True, i)) 227 | epoch_loss += loss_v 228 | 229 | epoch_loss /= FLAGS.pack_size 230 | train_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 231 | train_writer.add_summary(train_loss, step + 1) 232 | 233 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 234 | # (fp_time / step, fbp_time / step)) 235 | # 236 | print ("\nEpoch #%-5d | Train Loss: %-4.3f | PureTrainTime: %-3.3fs" % 237 | (step / FLAGS.epoch_steps, epoch_loss, end - start)) 238 | 239 | epoch_loss = 0 240 | for i in range(FLAGS.pack_size): 241 | loss_v = sess.run(loss, feed_dict=feed_dict(False, i)) 242 | epoch_loss += loss_v 243 | 244 | epoch_loss /= FLAGS.pack_size 245 | 246 | test_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 247 | test_writer.add_summary(test_loss, step + 1) 248 | 249 | start = time.time() 250 | print ("Epoch #%-5d | Test Loss: %-4.3f | Calc_LossTime: %-3.3fs" % 251 | (step / FLAGS.epoch_steps, epoch_loss, start - end)) 252 | 253 | -------------------------------------------------------------------------------- /single/dssm_v2.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import random 3 | import time 4 | 5 | import numpy as np 6 | 7 | import tensorflow as tf 8 | 9 | flags = tf.app.flags 10 | FLAGS = flags.FLAGS 11 | 12 | flags.DEFINE_string('summaries_dir', '/tmp/dssm_dump', 'Summaries directory') 13 | flags.DEFINE_float('learning_rate', 0.3, 'Initial learning rate.') 14 | flags.DEFINE_integer('max_steps', 50000, 'Number of steps to run trainer.') 15 | flags.DEFINE_integer('epoch_steps', 1000, "Number of steps in one epoch") 16 | 17 | start = time.time() 18 | doc_train_data = pickle.load(open('doc.train.pickle', 'rb')).tocsr() 19 | query_train_data = pickle.load(open('query.train.pickle', 'rb')).tocsr() 20 | doc_test_data = pickle.load(open('doc.test.pickle', 'rb')).tocsr() 21 | query_test_data = pickle.load(open('query.test.pickle', 'rb')).tocsr() 22 | end = time.time() 23 | print("Loading data from HDD to memory: %f s" % (end - start)) 24 | 25 | TRIGRAM_D = doc_train_data.shape[1] 26 | 27 | NEG = 4 28 | BS = 1024 29 | 30 | L1_N = 600 31 | L2_N = 300 32 | 33 | query_in_shape = np.array([BS, TRIGRAM_D], np.int64) 34 | doc_in_shape = np.array([BS, TRIGRAM_D], np.int64) 35 | 36 | 37 | def variable_summaries(var, name): 38 | """Attach a lot of summaries to a Tensor.""" 39 | with tf.name_scope('summaries'): 40 | mean = tf.reduce_mean(var) 41 | tf.scalar_summary('mean/' + name, mean) 42 | with tf.name_scope('stddev'): 43 | stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean))) 44 | tf.scalar_summary('sttdev/' + name, stddev) 45 | tf.scalar_summary('max/' + name, tf.reduce_max(var)) 46 | tf.scalar_summary('min/' + name, tf.reduce_min(var)) 47 | tf.histogram_summary(name, var) 48 | 49 | 50 | with tf.name_scope('input'): 51 | # Shape [BS, TRIGRAM_D]. 52 | query_batch = tf.sparse_placeholder(tf.float32, shape=query_in_shape, name='QueryBatch') 53 | # Shape [BS, TRIGRAM_D] 54 | doc_batch = tf.sparse_placeholder(tf.float32, shape=doc_in_shape, name='DocBatch') 55 | 56 | with tf.name_scope('L1'): 57 | # Hidden layer 1 [input: cols, output: 300] 58 | l1_par_range = np.sqrt(6.0 / (TRIGRAM_D + L1_N)) 59 | 60 | weight1 = tf.Variable(tf.random_uniform([TRIGRAM_D, L1_N], -l1_par_range, l1_par_range)) 61 | bias1 = tf.Variable(tf.random_uniform([L1_N], -l1_par_range, l1_par_range)) 62 | variable_summaries(weight1, 'L1_weights') 63 | variable_summaries(bias1, 'L1_biases') 64 | 65 | # query_l1 = tf.matmul(tf.to_float(query_batch),weight1)+bias1 66 | query_l1 = tf.sparse_tensor_dense_matmul(query_batch, weight1) + bias1 67 | # doc_l1 = tf.matmul(tf.to_float(doc_batch),weight1)+bias1 68 | doc_l1 = tf.sparse_tensor_dense_matmul(doc_batch, weight1) + bias1 69 | 70 | query_l1_out = tf.nn.tanh(query_l1) 71 | doc_l1_out = tf.nn.tanh(doc_l1) 72 | 73 | with tf.name_scope('L2'): 74 | # Hidden layer 2 [input: 300, output: 300] 75 | l2_par_range = np.sqrt(6.0 / (L1_N + L2_N)) 76 | 77 | weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], -l2_par_range, l2_par_range)) 78 | bias2 = tf.Variable(tf.random_uniform([L2_N], -l2_par_range, l2_par_range)) 79 | variable_summaries(weight2, 'L2_weights') 80 | variable_summaries(bias2, 'L2_biases') 81 | 82 | query_l2 = tf.matmul(query_l1_out, weight2) + bias2 83 | doc_l2 = tf.matmul(doc_l1_out, weight2) + bias2 84 | query_y = tf.nn.tanh(query_l2) 85 | doc_y = tf.nn.tanh(doc_l2) 86 | 87 | # with tf.name_scope('L3'): 88 | # # Hidden layer 3 [input: 300, output: 128] 89 | # l3_par_range = np.sqrt(6.0 / (L2_N + L3_N)) 90 | # 91 | # weight3 = tf.Variable(tf.random_uniform([L2_N, L3_N], -l3_par_range, l3_par_range)) 92 | # bias3 = tf.Variable(tf.random_uniform([L3_N], -l3_par_range, l3_par_range)) 93 | # variable_summaries(weight3, 'L3_weights') 94 | # variable_summaries(bias3, 'L3_biases') 95 | # 96 | # query_l3 = tf.matmul(query_l2_out, weight3) + bias3 97 | # doc_l3 = tf.matmul(doc_l2_out, weight3) + bias3 98 | # query_y = tf.nn.relu(query_l3) 99 | # doc_y = tf.nn.relu(doc_l3) 100 | 101 | with tf.name_scope('FD_rotate'): 102 | # Rotate FD+ to produce 50 FD- 103 | temp = tf.tile(doc_y, [1, 1]) 104 | 105 | for i in range(NEG): 106 | rand = int((random.random() + i) * BS / NEG) 107 | doc_y = tf.concat(0, 108 | [doc_y, 109 | tf.slice(temp, [rand, 0], [BS - rand, -1]), 110 | tf.slice(temp, [0, 0], [rand, -1])]) 111 | 112 | with tf.name_scope('Cosine_Similarity'): 113 | # Cosine similarity 114 | query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)), [NEG + 1, 1]) 115 | doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True)) 116 | 117 | prod = tf.reduce_sum(tf.mul(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1, True) 118 | norm_prod = tf.mul(query_norm, doc_norm) 119 | 120 | cos_sim_raw = tf.truediv(prod, norm_prod) 121 | cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw), [NEG + 1, BS])) * 20 122 | 123 | with tf.name_scope('Loss'): 124 | # Train Loss 125 | prob = tf.nn.softmax((cos_sim)) 126 | hit_prob = tf.slice(prob, [0, 0], [-1, 1]) 127 | loss = -tf.reduce_sum(tf.log(hit_prob)) / BS 128 | tf.scalar_summary('loss', loss) 129 | 130 | with tf.name_scope('Training'): 131 | # Optimizer 132 | train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(loss) 133 | 134 | with tf.name_scope('Accuracy'): 135 | correct_prediction = tf.equal(tf.argmax(prob, 1), 0) 136 | accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 137 | tf.scalar_summary('accuracy', accuracy) 138 | 139 | merged = tf.merge_all_summaries() 140 | 141 | with tf.name_scope('Test'): 142 | average_loss = tf.placeholder(tf.float32) 143 | test_loss_summary = tf.scalar_summary('average_loss', average_loss) 144 | average_acc = tf.placeholder(tf.float32) 145 | test_acc_summary = tf.scalar_summary('average_acc', average_acc) 146 | 147 | 148 | def pull_batch(query_data, doc_data, batch_idx): 149 | query_in = query_data[batch_idx * BS:(batch_idx + 1) * BS, :] 150 | doc_in = doc_data[batch_idx * BS:(batch_idx + 1) * BS, :] 151 | query_in = query_in.tocoo() 152 | doc_in = doc_in.tocoo() 153 | 154 | print(query_in.data.shape) 155 | print(doc_in.data.shape) 156 | 157 | query_in = tf.SparseTensorValue( 158 | np.transpose([np.array(query_in.row, dtype=np.int64), np.array(query_in.col, dtype=np.int64)]), 159 | np.array(query_in.data, dtype=np.float), 160 | np.array(query_in.shape, dtype=np.int64)) 161 | doc_in = tf.SparseTensorValue( 162 | np.transpose([np.array(doc_in.row, dtype=np.int64), np.array(doc_in.col, dtype=np.int64)]), 163 | np.array(doc_in.data, dtype=np.float), 164 | np.array(doc_in.shape, dtype=np.int64)) 165 | 166 | return query_in, doc_in 167 | 168 | 169 | def feed_dict(Train, batch_idx): 170 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 171 | if Train: 172 | query_in, doc_in = pull_batch(query_train_data, doc_train_data, batch_idx) 173 | else: 174 | query_in, doc_in = pull_batch(query_test_data, doc_test_data, batch_idx) 175 | return {query_batch: query_in, doc_batch: doc_in} 176 | 177 | 178 | config = tf.ConfigProto() # log_device_placement=True) 179 | config.gpu_options.allow_growth = True 180 | 181 | with tf.Session(config=config) as sess: 182 | sess.run(tf.initialize_all_variables()) 183 | train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) 184 | test_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/test', sess.graph) 185 | 186 | # Actual execution 187 | for step in range(FLAGS.max_steps): 188 | batch_idx = step % FLAGS.epoch_steps 189 | 190 | start = time.time() 191 | loss_v = sess.run(loss, feed_dict=feed_dict(True, batch_idx)) 192 | end = time.time() 193 | 194 | start2 = time.time() 195 | sess.run(train_step, feed_dict=feed_dict(True, batch_idx)) 196 | end2 = time.time() 197 | 198 | acc_v = sess.run(accuracy, feed_dict=feed_dict(True, batch_idx)) 199 | if batch_idx % 100 == 0: 200 | summary = sess.run(merged, feed_dict=feed_dict(True, batch_idx)) 201 | train_writer.add_summary(summary, step) 202 | print("MiniBatch #%-5d | Acc %-3.2f%% | Loss: %-4.3f | FP: %1.4fs | FP+BP: %1.4fs" 203 | % (step + 1, acc_v * 100, loss_v, (end - start), end2 - start2)) 204 | 205 | if batch_idx == FLAGS.epoch_steps - 1: 206 | start = time.time() 207 | epoch_loss = 0 208 | acc = 0 209 | for i in range(FLAGS.epoch_steps): 210 | acc_v, loss_v = sess.run([accuracy, loss], feed_dict=feed_dict(False, i)) 211 | epoch_loss += loss_v 212 | acc += acc_v 213 | 214 | epoch_loss /= FLAGS.epoch_steps 215 | acc /= FLAGS.epoch_steps 216 | 217 | loss_summary = sess.run(test_loss_summary, feed_dict={average_loss: epoch_loss}) 218 | acc_summary = sess.run(test_acc_summary, feed_dict={average_acc: acc}) 219 | test_writer.add_summary(loss_summary, step + 1) 220 | test_writer.add_summary(acc_summary, step + 1) 221 | 222 | end = time.time() 223 | print ("Epoch #%-5d | Acc %-3.2f%% | Loss: %-4.3f | Val_Time: %2.4f" % 224 | (step / FLAGS.epoch_steps, acc * 100, epoch_loss, end - start)) 225 | -------------------------------------------------------------------------------- /single/dssm_v3.py: -------------------------------------------------------------------------------- 1 | import pickle 2 | import random 3 | import time 4 | import sys 5 | import numpy as np 6 | import tensorflow as tf 7 | 8 | flags = tf.app.flags 9 | FLAGS = flags.FLAGS 10 | 11 | flags.DEFINE_string('summaries_dir', '/tmp/dssm-400-120-relu', 'Summaries directory') 12 | flags.DEFINE_float('learning_rate', 0.1, 'Initial learning rate.') 13 | flags.DEFINE_integer('max_steps', 900000, 'Number of steps to run trainer.') 14 | flags.DEFINE_integer('epoch_steps', 18000, "Number of steps in one epoch.") 15 | flags.DEFINE_integer('pack_size', 2000, "Number of batches in one pickle pack.") 16 | flags.DEFINE_bool('gpu', 1, "Enable GPU or not") 17 | 18 | start = time.time() 19 | 20 | doc_train_data = None 21 | query_train_data = None 22 | 23 | # load test data for now 24 | query_test_data = pickle.load(open('../data/query.test.pickle', 'rb')).tocsr() 25 | doc_test_data = pickle.load(open('../data/doc.test.pickle', 'rb')).tocsr() 26 | 27 | doc_train_data = pickle.load(open('../data/doc.train.pickle', 'rb')).tocsr() 28 | query_train_data = pickle.load(open('../data/query.train.pickle', 'rb')).tocsr() 29 | 30 | end = time.time() 31 | print("Loading data from HDD to memory: %.2fs" % (end - start)) 32 | 33 | TRIGRAM_D = 49284 34 | 35 | NEG = 50 36 | BS = 1024 37 | 38 | L1_N = 400 39 | L2_N = 120 40 | 41 | query_in_shape = np.array([BS, TRIGRAM_D], np.int64) 42 | doc_in_shape = np.array([BS, TRIGRAM_D], np.int64) 43 | 44 | 45 | def variable_summaries(var, name): 46 | """Attach a lot of summaries to a Tensor.""" 47 | with tf.name_scope('summaries'): 48 | mean = tf.reduce_mean(var) 49 | tf.scalar_summary('mean/' + name, mean) 50 | with tf.name_scope('stddev'): 51 | stddev = tf.sqrt(tf.reduce_sum(tf.square(var - mean))) 52 | tf.scalar_summary('sttdev/' + name, stddev) 53 | tf.scalar_summary('max/' + name, tf.reduce_max(var)) 54 | tf.scalar_summary('min/' + name, tf.reduce_min(var)) 55 | tf.histogram_summary(name, var) 56 | 57 | 58 | with tf.name_scope('input'): 59 | # Shape [BS, TRIGRAM_D]. 60 | query_batch = tf.sparse_placeholder(tf.float32, shape=query_in_shape, name='QueryBatch') 61 | # Shape [BS, TRIGRAM_D] 62 | doc_batch = tf.sparse_placeholder(tf.float32, shape=doc_in_shape, name='DocBatch') 63 | 64 | with tf.name_scope('L1'): 65 | l1_par_range = np.sqrt(6.0 / (TRIGRAM_D + L1_N)) 66 | weight1 = tf.Variable(tf.random_uniform([TRIGRAM_D, L1_N], -l1_par_range, l1_par_range)) 67 | bias1 = tf.Variable(tf.random_uniform([L1_N], -l1_par_range, l1_par_range)) 68 | variable_summaries(weight1, 'L1_weights') 69 | variable_summaries(bias1, 'L1_biases') 70 | 71 | # query_l1 = tf.matmul(tf.to_float(query_batch),weight1)+bias1 72 | query_l1 = tf.sparse_tensor_dense_matmul(query_batch, weight1) + bias1 73 | # doc_l1 = tf.matmul(tf.to_float(doc_batch),weight1)+bias1 74 | doc_l1 = tf.sparse_tensor_dense_matmul(doc_batch, weight1) + bias1 75 | 76 | query_l1_out = tf.nn.relu(query_l1) 77 | doc_l1_out = tf.nn.relu(doc_l1) 78 | 79 | with tf.name_scope('L2'): 80 | l2_par_range = np.sqrt(6.0 / (L1_N + L2_N)) 81 | 82 | weight2 = tf.Variable(tf.random_uniform([L1_N, L2_N], -l2_par_range, l2_par_range)) 83 | bias2 = tf.Variable(tf.random_uniform([L2_N], -l2_par_range, l2_par_range)) 84 | variable_summaries(weight2, 'L2_weights') 85 | variable_summaries(bias2, 'L2_biases') 86 | 87 | query_l2 = tf.matmul(query_l1_out, weight2) + bias2 88 | doc_l2 = tf.matmul(doc_l1_out, weight2) + bias2 89 | query_y = tf.nn.relu(query_l2) 90 | doc_y = tf.nn.relu(doc_l2) 91 | 92 | with tf.name_scope('FD_rotate'): 93 | # Rotate FD+ to produce 50 FD- 94 | temp = tf.tile(doc_y, [1, 1]) 95 | 96 | for i in range(NEG): 97 | rand = int((random.random() + i) * BS / NEG) 98 | doc_y = tf.concat(0, 99 | [doc_y, 100 | tf.slice(temp, [rand, 0], [BS - rand, -1]), 101 | tf.slice(temp, [0, 0], [rand, -1])]) 102 | 103 | with tf.name_scope('Cosine_Similarity'): 104 | # Cosine similarity 105 | query_norm = tf.tile(tf.sqrt(tf.reduce_sum(tf.square(query_y), 1, True)), [NEG + 1, 1]) 106 | doc_norm = tf.sqrt(tf.reduce_sum(tf.square(doc_y), 1, True)) 107 | 108 | prod = tf.reduce_sum(tf.mul(tf.tile(query_y, [NEG + 1, 1]), doc_y), 1, True) 109 | norm_prod = tf.mul(query_norm, doc_norm) 110 | 111 | cos_sim_raw = tf.truediv(prod, norm_prod) 112 | cos_sim = tf.transpose(tf.reshape(tf.transpose(cos_sim_raw), [NEG + 1, BS])) * 20 113 | 114 | with tf.name_scope('Loss'): 115 | # Train Loss 116 | prob = tf.nn.softmax((cos_sim)) 117 | hit_prob = tf.slice(prob, [0, 0], [-1, 1]) 118 | loss = -tf.reduce_sum(tf.log(hit_prob)) / BS 119 | tf.scalar_summary('loss', loss) 120 | 121 | with tf.name_scope('Training'): 122 | # Optimizer 123 | train_step = tf.train.GradientDescentOptimizer(FLAGS.learning_rate).minimize(loss) 124 | 125 | # with tf.name_scope('Accuracy'): 126 | # correct_prediction = tf.equal(tf.argmax(prob, 1), 0) 127 | # accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32)) 128 | # tf.scalar_summary('accuracy', accuracy) 129 | 130 | merged = tf.merge_all_summaries() 131 | 132 | with tf.name_scope('Test'): 133 | average_loss = tf.placeholder(tf.float32) 134 | loss_summary = tf.scalar_summary('average_loss', average_loss) 135 | 136 | 137 | def pull_batch(query_data, doc_data, batch_idx): 138 | # start = time.time() 139 | query_in = query_data[batch_idx * BS:(batch_idx + 1) * BS, :] 140 | doc_in = doc_data[batch_idx * BS:(batch_idx + 1) * BS, :] 141 | 142 | if batch_idx == 0: 143 | print(query_in.getrow(53)) 144 | query_in = query_in.tocoo() 145 | doc_in = doc_in.tocoo() 146 | 147 | 148 | 149 | query_in = tf.SparseTensorValue( 150 | np.transpose([np.array(query_in.row, dtype=np.int64), np.array(query_in.col, dtype=np.int64)]), 151 | np.array(query_in.data, dtype=np.float), 152 | np.array(query_in.shape, dtype=np.int64)) 153 | doc_in = tf.SparseTensorValue( 154 | np.transpose([np.array(doc_in.row, dtype=np.int64), np.array(doc_in.col, dtype=np.int64)]), 155 | np.array(doc_in.data, dtype=np.float), 156 | np.array(doc_in.shape, dtype=np.int64)) 157 | 158 | # end = time.time() 159 | # print("Pull_batch time: %f" % (end - start)) 160 | 161 | return query_in, doc_in 162 | 163 | 164 | def feed_dict(Train, batch_idx): 165 | """Make a TensorFlow feed_dict: maps data onto Tensor placeholders.""" 166 | if Train: 167 | query_in, doc_in = pull_batch(query_train_data, doc_train_data, batch_idx) 168 | else: 169 | query_in, doc_in = pull_batch(query_test_data, doc_test_data, batch_idx) 170 | return {query_batch: query_in, doc_batch: doc_in} 171 | 172 | 173 | config = tf.ConfigProto() # log_device_placement=True) 174 | config.gpu_options.allow_growth = True 175 | #if not FLAGS.gpu: 176 | #config = tf.ConfigProto(device_count= {'GPU' : 0}) 177 | 178 | with tf.Session(config=config) as sess: 179 | sess.run(tf.initialize_all_variables()) 180 | train_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/train', sess.graph) 181 | test_writer = tf.train.SummaryWriter(FLAGS.summaries_dir + '/test', sess.graph) 182 | 183 | # Actual execution 184 | start = time.time() 185 | # fp_time = 0 186 | # fbp_time = 0 187 | for step in range(FLAGS.max_steps): 188 | batch_idx = step % FLAGS.epoch_steps 189 | # if batch_idx % FLAGS.pack_size == 0: 190 | # load_train_data(batch_idx / FLAGS.pack_size + 1) 191 | 192 | # # setup toolbar 193 | # sys.stdout.write("[%s]" % (" " * toolbar_width)) 194 | # #sys.stdout.flush() 195 | # sys.stdout.write("\b" * (toolbar_width + 1)) # return to start of line, after '[' 196 | if batch_idx == 0: 197 | temp = sess.run(query_y, feed_dict=feed_dict(True, 0)) 198 | print(np.count_nonzero(temp)) 199 | sys.exit() 200 | 201 | if batch_idx % (FLAGS.pack_size / 64) == 0: 202 | progress = 100.0 * batch_idx / FLAGS.epoch_steps 203 | sys.stdout.write("\r%.2f%% Epoch" % progress) 204 | sys.stdout.flush() 205 | 206 | # t1 = time.time() 207 | # sess.run(loss, feed_dict = feed_dict(True, batch_idx)) 208 | # t2 = time.time() 209 | # fp_time += t2 - t1 210 | # #print(t2-t1) 211 | # t1 = time.time() 212 | sess.run(train_step, feed_dict=feed_dict(True, batch_idx % FLAGS.pack_size)) 213 | # t2 = time.time() 214 | # fbp_time += t2 - t1 215 | # #print(t2 - t1) 216 | # if batch_idx % 2000 == 1999: 217 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 218 | # (fp_time / step, fbp_time / step)) 219 | 220 | 221 | if batch_idx == FLAGS.epoch_steps - 1: 222 | end = time.time() 223 | epoch_loss = 0 224 | for i in range(FLAGS.pack_size): 225 | loss_v = sess.run(loss, feed_dict=feed_dict(True, i)) 226 | epoch_loss += loss_v 227 | 228 | epoch_loss /= FLAGS.pack_size 229 | train_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 230 | train_writer.add_summary(train_loss, step + 1) 231 | 232 | # print ("MiniBatch: Average FP Time %f, Average FP+BP Time %f" % 233 | # (fp_time / step, fbp_time / step)) 234 | # 235 | print ("\nEpoch #%-5d | Train Loss: %-4.3f | PureTrainTime: %-3.3fs" % 236 | (step / FLAGS.epoch_steps, epoch_loss, end - start)) 237 | 238 | epoch_loss = 0 239 | for i in range(FLAGS.pack_size): 240 | loss_v = sess.run(loss, feed_dict=feed_dict(False, i)) 241 | epoch_loss += loss_v 242 | 243 | epoch_loss /= FLAGS.pack_size 244 | 245 | test_loss = sess.run(loss_summary, feed_dict={average_loss: epoch_loss}) 246 | test_writer.add_summary(test_loss, step + 1) 247 | 248 | start = time.time() 249 | print ("Epoch #%-5d | Test Loss: %-4.3f | Calc_LossTime: %-3.3fs" % 250 | (step / FLAGS.epoch_steps, epoch_loss, start - end)) 251 | 252 | -------------------------------------------------------------------------------- /single/plot.py: -------------------------------------------------------------------------------- 1 | import matplotlib.pyplot as plt 2 | import numpy as np 3 | 4 | 5 | 6 | tf_train_loss = {} 7 | tf_test_loss = {} 8 | with open('size-20M.log', 'r') as file: 9 | iter = 1 10 | i = 1 11 | for line in file: 12 | if i % 2 == 1: 13 | line = line.split('|')[1] 14 | line = float(line.replace("Train Loss: ", "")) 15 | tf_train_loss[iter] = line 16 | else: 17 | line = line.split('|')[1] 18 | line = float(line.replace("Test Loss: ", "")) 19 | tf_test_loss[iter] = line 20 | iter += 1 21 | i += 1 22 | 23 | #print(tf_train_loss.values()) 24 | #print(tf_test_loss.values()) 25 | 26 | epochs = range(1, 51) 27 | # C# Train Loss in 50 Epochs, ~60s per epoch 28 | cs_loss = [1.11898974609375, 0.504269653320312, 0.395708129882813, 0.335024261474609, 0.293223724365234, 0.261924194335938, 0.237655487060547, 0.218366134643555, 0.202596160888672, 0.189925918579102, 0.178370178222656, 0.168332656860352, 0.160474334716797, 0.153653518676758, 0.147071426391602, 0.141640625, 0.136792068481445, 0.132845932006836, 0.128911636352539, 0.124887222290039, 0.121858062744141, 0.118886299133301, 0.116081825256348, 0.11357048034668, 0.111425018310547, 0.109386741638184, 0.106987724304199, 0.105347694396973, 0.103544723510742, 0.101966697692871, 0.100522766113281, 0.0991875152587891, 0.0977923355102539, 0.0962695617675781, 0.0952405776977539, 0.0941733703613281, 0.0931077041625977, 0.0919208374023437, 0.0909454040527344, 0.0902716293334961, 0.0892609939575195, 0.0885543212890625, 0.0877673034667969, 0.0871119842529297, 0.0861257705688477, 0.0852307739257813, 0.0846356658935547, 0.0838711013793945, 0.0835793838500977, 0.083037956237793] 29 | 30 | # TF Train/Test Loss in 50 Epochs, ~28.75s per epoch 31 | tf_train_loss = [0.6, 0.462, 0.398, 0.35, 0.315, 0.286, 0.261, 0.244, 0.228, 0.211, 0.198, 0.187, 0.174, 0.162, 0.154, 0.145, 0.134, 0.127, 0.124, 0.117, 0.11, 0.105, 0.1, 0.096, 0.092, 0.086, 0.084, 0.08, 0.077, 0.074, 0.072, 0.07, 0.068, 0.065, 0.063, 0.062, 0.059, 0.058, 0.056, 0.055, 0.055, 0.052, 0.051, 0.05, 0.048, 0.047, 0.046, 0.045, 0.045, 0.043] 32 | tf_test_loss = [0.687, 0.595, 0.563, 0.543, 0.533, 0.525, 0.524, 0.525, 0.526, 0.531, 0.533, 0.534, 0.538, 0.544, 0.543, 0.552, 0.548, 0.552, 0.557, 0.56, 0.565, 0.565, 0.571, 0.579, 0.575, 0.583, 0.583, 0.589, 0.596, 0.596, 0.6, 0.606, 0.607, 0.613, 0.613, 0.616, 0.621, 0.621, 0.628, 0.624, 0.63, 0.633, 0.638, 0.641, 0.642, 0.648, 0.649, 0.657, 0.658, 0.658] 33 | 34 | # TF Train/Test Loss in 50 Epochs, 35 | # 18 times bigger training set, 36 | # 2 times bigger test set, 37 | # ~650s per epoch 38 | tf_train_loss_large20 = [0.304, 0.256, 0.231, 0.213, 0.199, 0.188, 0.179, 0.172, 0.167, 0.162, 0.157, 0.153, 0.149, 0.146, 0.142, 0.139, 0.137, 0.134, 0.132, 0.13, 0.128, 0.126, 0.124, 0.122, 0.121, 0.12, 0.118, 0.117, 0.115, 0.114, 0.113, 0.111, 0.11, 0.111, 0.108, 0.108, 0.107, 0.106, 0.105, 0.104, 0.103, 0.103, 0.102, 0.101, 0.102, 0.1, 0.099, 0.099, 0.098, 0.098] 39 | tf_test_loss_large20 = [0.347, 0.306, 0.285, 0.271, 0.261, 0.252, 0.246, 0.241, 0.237, 0.234, 0.231, 0.228, 0.225, 0.223, 0.221, 0.219, 0.217, 0.215, 0.214, 0.212, 0.211, 0.21, 0.209, 0.208, 0.207, 0.207, 0.205, 0.205, 0.203, 0.203, 0.202, 0.201, 0.2, 0.201, 0.2, 0.199, 0.198, 0.198, 0.197, 0.197, 0.196, 0.196, 0.196, 0.195, 0.196, 0.195, 0.195, 0.194, 0.194, 0.194] 40 | 41 | tf_train_loss_large20_dense = [0.3067, 0.2533, 0.2252, 0.2075, 0.1934, 0.1829, 0.1743, 0.1674, 0.1609, 0.1554, 0.1506,0.1462,0.1419,0.1386,0.1353,0.1328,0.1303,0.1281,0.1259,0.1238,0.1215,0.1198,0.1180,0.1166,0.1150,0.1136,0.1121,0.1108,0.1097,0.1088,0.1076,0.1066,0.1056,0.1046,0.1038,0.1030,0.1023,0.1016,0.1009,0.1002,0.09948,0.09874,0.09810,0.09745,0.09692,0.09639,0.09585,0.09539,0.09491,0.09435] 42 | tf_test_loss_large20_dense = [0.3518, 0.3054, 0.2827, 0.2692, 0.2586, 0.2507, 0.2444, 0.2395, 0.2349, 0.2312,0.2279,0.2245,0.2215,0.2194,0.2174,0.2157,0.2139,0.2122,0.2107, 0.2093, 0.2080,0.2071,0.2060,0.2052,0.2046,0.2039,0.2027,0.2020,0.2014,0.2007,0.2002,0.1997,0.1991,0.1987,0.1981,0.1978,0.1974,0.1972,0.1968,0.1964,0.1959, 0.1956, 0.1952, 0.1949, 0.1945, 0.1945, 0.1942, 0.1938, 0.1939, 0.1934] 43 | 44 | #print (len(tf_test_loss_large20_dense)) 45 | #print (len(tf_train_loss_large20_dense)) 46 | # TF_Dist Train/Test Loss in 50 Epoch (Local GPU, 1 remote CPU), ~31.25s per epoch 47 | # !TODO Distributed Version not working properly 48 | tf_train_loss_dist_lg1r = [0.179, 0.163, 0.15, 0.141, 0.131, 0.123, 0.117, 0.11, 0.106, 0.099, 0.094, 0.091, 0.086, 0.082, 0.08, 0.076, 0.074, 0.071, 0.069, 0.066, 0.064, 0.062, 0.061, 0.057, 0.057, 0.055, 0.053, 0.053, 0.05, 0.05, 0.048, 0.047, 0.045, 0.044, 0.044, 0.042, 0.043, 0.046, 0.041, 0.039, 0.039, 0.037, 0.037, 0.036, 0.036, 0.035, 0.035, 0.034, 0.033, 0.033] 49 | tf_test_loss_dist_lg1r = [0.538, 0.551, 0.561, 0.572, 0.576, 0.584, 0.591, 0.599, 0.602, 0.61, 0.61, 0.621, 0.62, 0.625, 0.631, 0.634, 0.635, 0.639, 0.643, 0.643, 0.647, 0.65, 0.653, 0.656, 0.659, 0.659, 0.66, 0.667, 0.666, 0.669, 0.67, 0.675, 0.675, 0.679, 0.681, 0.681, 0.685, 0.676, 0.686, 0.687, 0.69, 0.691, 0.696, 0.695, 0.701, 0.7, 0.705, 0.704, 0.709, 0.71] 50 | 51 | # TF_Dist Train/Test Loss in 50 Epoch (Local GPU, 2 remote CPUs), ~31.8s per epoch 52 | # !TODO Distributed Version not working properly 53 | tf_train_loss_dist_lg2r = [0.116, 0.086, 0.073, 0.067, 0.06, 0.055, 0.051, 0.048, 0.046, 0.044, 0.043, 0.041, 0.04, 0.038, 0.037, 0.036, 0.036, 0.034, 0.034, 0.033, 0.032, 0.032, 0.031, 0.031, 0.03, 0.029, 0.029, 0.028, 0.028, 0.027, 0.027, 0.027, 0.026, 0.026, 0.025, 0.025, 0.025, 0.024, 0.024, 0.024, 0.023, 0.029, 0.027, 0.023, 0.022, 0.022, 0.023, 0.022, 0.022, 0.021] 54 | tf_test_loss_dist_lg2r = [0.599, 0.61, 0.625, 0.638, 0.646, 0.658, 0.663, 0.67, 0.672, 0.679, 0.683, 0.687, 0.693, 0.697, 0.699, 0.704, 0.705, 0.713, 0.711, 0.719, 0.718, 0.722, 0.724, 0.729, 0.729, 0.732, 0.735, 0.736, 0.74, 0.741, 0.746, 0.742, 0.75, 0.749, 0.751, 0.753, 0.756, 0.756, 0.761, 0.758, 0.763, 0.736, 0.748, 0.759, 0.767, 0.769, 0.766, 0.768, 0.769, 0.776] 55 | 56 | plt.subplot(1, 2, 1) 57 | plt.grid() 58 | plt.plot(epochs, cs_loss, '-', color= (0, 0, 0, 1),linewidth=2, label='C#') 59 | plt.plot(epochs, tf_train_loss, 'b-', linewidth=2, label='TF-Small') 60 | plt.plot(epochs, tf_train_loss_large20, 'r-', linewidth=2, label='TF-Large') 61 | plt.plot(epochs, tf_train_loss_large20_dense, 'g-', linewidth=2, label='TF-Large(Reduced)') 62 | 63 | plt.plot(epochs, tf_test_loss, 'b--', linewidth=2) 64 | plt.plot(epochs, tf_test_loss_large20, 'r--', linewidth=2) 65 | plt.plot(epochs, tf_test_loss_large20_dense, 'g--', linewidth=2) 66 | #plt.plot(epochs, tf_train_loss_dist_lg1r, 'r-', linewidth=2, label='Local(g):1Remote') 67 | #plt.plot(epochs, tf_train_loss_dist_lg2r, 'c-', linewidth=2, label='Local(g):2Remote') 68 | plt.xlabel("Epoch") 69 | plt.ylabel("Loss") 70 | plt.title("Loss") 71 | plt.legend() 72 | 73 | 74 | # plt.subplot(2,2,2) 75 | # plt.plot(epochs, tf_test_loss, 'b--', linewidth=2, label='L(g):0R') 76 | # #plt.plot(epochs, tf_test_loss_dist_lg1r, 'r--', linewidth=2, label='L(g):1R') 77 | # #plt.plot(epochs, tf_test_loss_dist_lg2r, 'c--', linewidth=2, label='L(g):2R') 78 | # plt.xlabel("Epoch") 79 | # plt.ylabel("Loss") 80 | # plt.title("Test Loss") 81 | # plt.ylim(0, 1.2) 82 | 83 | plt.subplot(1, 2, 2) 84 | plt.grid(axis='y') 85 | barwidth = 0.3 86 | COUNT = 4 87 | x = np.arange(COUNT) + 1 - barwidth / 2 88 | time = [50, 28.75, 660/18.0, 676.49/18.0] 89 | 90 | barlist = plt.bar(x, time, barwidth) 91 | barlist[0].set_color((0, 0, 0, 0.8)) 92 | barlist[1].set_color('b') 93 | barlist[2].set_color('r') 94 | barlist[3].set_color('g') 95 | 96 | plt.xticks(x + barwidth / 2, ('C#', 'TF-Small', 'TF-Large', 'TF-Large(Reduced)')) 97 | plt.title('Train Time') 98 | plt.ylabel("Training Time Per 1M Queries (s)") 99 | plt.xlim(0.5, COUNT + 0.5) 100 | plt.ylim(0, 60) 101 | 102 | plt.tight_layout() 103 | plt.show() 104 | -------------------------------------------------------------------------------- /tf-datasize.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fwd4/dssm/18842289e2eb8447efca2405e1edea080d5f7bcb/tf-datasize.png --------------------------------------------------------------------------------