├── .gitignore ├── 00-MXnet Basics └── basic.ipynb ├── 01-Linear Regression └── linear_regression.py ├── 02-Logistic Regression └── logistic_regression.py ├── 03-Neural Network └── neural network.py ├── 04-Convolution Neural Network └── convolution_network.py ├── 05-Recurrnet Neural Network └── recurrent_network.py ├── 06-Natural Language Process ├── N-Gram.py ├── bag-of-word.py └── seq-lstm.py ├── 08-AutoEncoder ├── Variational_autoencoder.py ├── conv_autoencoder.py └── simple_autoencoder.py ├── 09-Generative Adversarial network ├── conv_gan.py └── simple_gan.py └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .ipynb_checkpoints 2 | save_model 3 | data 4 | *.params 5 | .idea 6 | -------------------------------------------------------------------------------- /00-MXnet Basics/basic.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 4, 6 | "metadata": { 7 | "collapsed": false 8 | }, 9 | "outputs": [], 10 | "source": [ 11 | "import warnings\n", 12 | "warnings.filterwarnings('ignore')" 13 | ] 14 | }, 15 | { 16 | "cell_type": "code", 17 | "execution_count": 5, 18 | "metadata": { 19 | "collapsed": false 20 | }, 21 | "outputs": [], 22 | "source": [ 23 | "import mxnet as mx\n", 24 | "import numpy as np" 25 | ] 26 | }, 27 | { 28 | "cell_type": "markdown", 29 | "metadata": {}, 30 | "source": [ 31 | "## mxnet取名为mxnet是因为mixture network，是采用混合编程的方式，符号编程和显示编程，里面有三种基本的处理类型，ndarray, module, symbol" 32 | ] 33 | }, 34 | { 35 | "cell_type": "markdown", 36 | "metadata": {}, 37 | "source": [ 38 | "### define a nd.array like np.array" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 26, 44 | "metadata": { 45 | "collapsed": false 46 | }, 47 | "outputs": [], 48 | "source": [ 49 | "# define a nd.array like np.array\n", 50 | "a = mx.nd.random_exponential(shape=(2, 3))\n", 51 | "b = mx.nd.ones((3, 4))\n", 52 | "c = mx.nd.full((2, 3), 3) # get a 2x3 matrix with every item 3\n", 53 | "d = mx.nd.array([[2, 3], [5, 9]], ctx=mx.gpu(0))" 54 | ] 55 | }, 56 | { 57 | "cell_type": "markdown", 58 | "metadata": {}, 59 | "source": [ 60 | "### if you print it, it won't show implicity, but it will give the information about the array" 61 | ] 62 | }, 63 | { 64 | "cell_type": "code", 65 | "execution_count": 27, 66 | "metadata": { 67 | "collapsed": false 68 | }, 69 | "outputs": [ 70 | { 71 | "name": "stdout", 72 | "output_type": "stream", 73 | "text": [ 74 | "\n" 75 | ] 76 | } 77 | ], 78 | "source": [ 79 | "print(a)" 80 | ] 81 | }, 82 | { 83 | "cell_type": "code", 84 | "execution_count": 28, 85 | "metadata": { 86 | "collapsed": false 87 | }, 88 | "outputs": [ 89 | { 90 | "name": "stdout", 91 | "output_type": "stream", 92 | "text": [ 93 | "\n" 94 | ] 95 | } 96 | ], 97 | "source": [ 98 | "print(b)" 99 | ] 100 | }, 101 | { 102 | "cell_type": "code", 103 | "execution_count": null, 104 | "metadata": { 105 | "collapsed": true 106 | }, 107 | "outputs": [], 108 | "source": [ 109 | "print(c)" 110 | ] 111 | }, 112 | { 113 | "cell_type": "markdown", 114 | "metadata": {}, 115 | "source": [ 116 | "### this means its shape and on which device(cpu or gpu)" 117 | ] 118 | }, 119 | { 120 | "cell_type": "markdown", 121 | "metadata": {}, 122 | "source": [ 123 | "### show it implicity" 124 | ] 125 | }, 126 | { 127 | "cell_type": "code", 128 | "execution_count": 29, 129 | "metadata": { 130 | "collapsed": false 131 | }, 132 | "outputs": [ 133 | { 134 | "data": { 135 | "text/plain": [ 136 | "array([[ 0.83943278, 0.49887207],\n", 137 | " [ 2.59825444, 1.80836928]], dtype=float32)" 138 | ] 139 | }, 140 | "execution_count": 29, 141 | "metadata": {}, 142 | "output_type": "execute_result" 143 | } 144 | ], 145 | "source": [ 146 | "# show it implicity\n", 147 | "a.asnumpy()" 148 | ] 149 | }, 150 | { 151 | "cell_type": "code", 152 | "execution_count": 14, 153 | "metadata": { 154 | "collapsed": false 155 | }, 156 | "outputs": [ 157 | { 158 | "data": { 159 | "text/plain": [ 160 | "array([[ 1., 1., 1., 1.],\n", 161 | " [ 1., 1., 1., 1.],\n", 162 | " [ 1., 1., 1., 1.]], dtype=float32)" 163 | ] 164 | }, 165 | "execution_count": 14, 166 | "metadata": {}, 167 | "output_type": "execute_result" 168 | } 169 | ], 170 | "source": [ 171 | "b.asnumpy()" 172 | ] 173 | }, 174 | { 175 | "cell_type": "code", 176 | "execution_count": null, 177 | "metadata": { 178 | "collapsed": true 179 | }, 180 | "outputs": [], 181 | "source": [ 182 | "c.asnumpy()" 183 | ] 184 | }, 185 | { 186 | "cell_type": "markdown", 187 | "metadata": {}, 188 | "source": [ 189 | "### get shape and size" 190 | ] 191 | }, 192 | { 193 | "cell_type": "code", 194 | "execution_count": 15, 195 | "metadata": { 196 | "collapsed": false 197 | }, 198 | "outputs": [ 199 | { 200 | "data": { 201 | "text/plain": [ 202 | "(3, 4)" 203 | ] 204 | }, 205 | "execution_count": 15, 206 | "metadata": {}, 207 | "output_type": "execute_result" 208 | } 209 | ], 210 | "source": [ 211 | "# get shape\n", 212 | "b.shape" 213 | ] 214 | }, 215 | { 216 | "cell_type": "code", 217 | "execution_count": 22, 218 | "metadata": { 219 | "collapsed": false 220 | }, 221 | "outputs": [ 222 | { 223 | "data": { 224 | "text/plain": [ 225 | "12" 226 | ] 227 | }, 228 | "execution_count": 22, 229 | "metadata": {}, 230 | "output_type": "execute_result" 231 | } 232 | ], 233 | "source": [ 234 | "# get size(total number of items)\n", 235 | "b.size" 236 | ] 237 | }, 238 | { 239 | "cell_type": "markdown", 240 | "metadata": {}, 241 | "source": [ 242 | "## operation" 243 | ] 244 | }, 245 | { 246 | "cell_type": "code", 247 | "execution_count": 38, 248 | "metadata": { 249 | "collapsed": false 250 | }, 251 | "outputs": [ 252 | { 253 | "data": { 254 | "text/plain": [ 255 | "array([[2, 2, 2],\n", 256 | " [2, 2, 2]], dtype=int32)" 257 | ] 258 | }, 259 | "execution_count": 38, 260 | "metadata": {}, 261 | "output_type": "execute_result" 262 | } 263 | ], 264 | "source": [ 265 | "a = mx.nd.ones((2, 3), dtype=np.int32)\n", 266 | "b = mx.nd.ones((2, 3), dtype=np.int32)\n", 267 | "c = a + b\n", 268 | "c.asnumpy()" 269 | ] 270 | }, 271 | { 272 | "cell_type": "code", 273 | "execution_count": 7, 274 | "metadata": { 275 | "collapsed": false 276 | }, 277 | "outputs": [], 278 | "source": [ 279 | "a = mx.nd.array([[2, 3], [4, 5]])\n", 280 | "b = mx.nd.array([[6, 7], [8, 9]])\n", 281 | "c = mx.nd.dot(a, b) # matrix multiply\n", 282 | "d = a * b # multiply element-wise" 283 | ] 284 | }, 285 | { 286 | "cell_type": "code", 287 | "execution_count": 9, 288 | "metadata": { 289 | "collapsed": false 290 | }, 291 | "outputs": [ 292 | { 293 | "name": "stdout", 294 | "output_type": "stream", 295 | "text": [ 296 | "[[ 36. 41.]\n", 297 | " [ 64. 73.]]\n" 298 | ] 299 | } 300 | ], 301 | "source": [ 302 | "print(c.asnumpy())" 303 | ] 304 | }, 305 | { 306 | "cell_type": "code", 307 | "execution_count": 10, 308 | "metadata": { 309 | "collapsed": false 310 | }, 311 | "outputs": [ 312 | { 313 | "name": "stdout", 314 | "output_type": "stream", 315 | "text": [ 316 | "[[ 12. 21.]\n", 317 | " [ 32. 45.]]\n" 318 | ] 319 | } 320 | ], 321 | "source": [ 322 | "print(d.asnumpy())" 323 | ] 324 | }, 325 | { 326 | "cell_type": "markdown", 327 | "metadata": {}, 328 | "source": [ 329 | "### advanced operation" 330 | ] 331 | }, 332 | { 333 | "cell_type": "code", 334 | "execution_count": null, 335 | "metadata": { 336 | "collapsed": true 337 | }, 338 | "outputs": [], 339 | "source": [ 340 | "a = mx.nd.ones((100,100), mx.cpu())\n", 341 | "b = mx.nd.ones((100,100), mx.gpu())\n", 342 | "c = mx.nd.ones((100,100), mx.gpu())\n", 343 | "a.copyto(c) # copy from CPU to GPU\n", 344 | "d = a.as_in_context(c.context) # same to above" 345 | ] 346 | } 347 | ], 348 | "metadata": { 349 | "kernelspec": { 350 | "display_name": "mxnet", 351 | "language": "python", 352 | "name": "mxnet" 353 | }, 354 | "language_info": { 355 | "codemirror_mode": { 356 | "name": "ipython", 357 | "version": 3 358 | }, 359 | "file_extension": ".py", 360 | "mimetype": "text/x-python", 361 | "name": "python", 362 | "nbconvert_exporter": "python", 363 | "pygments_lexer": "ipython3", 364 | "version": "3.6.0" 365 | } 366 | }, 367 | "nbformat": 4, 368 | "nbformat_minor": 2 369 | } 370 | -------------------------------------------------------------------------------- /01-Linear Regression/linear_regression.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | import matplotlib.pyplot as plt 5 | import mxnet as mx 6 | import numpy as np 7 | 8 | logging.basicConfig(level=logging.INFO, 9 | format='%(asctime)-15s %(message)s', 10 | stream=sys.stdout) 11 | 12 | ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu() 13 | batch_size = 4 14 | total_epochs = 200 15 | 16 | x_train = np.array( 17 | [[3.3], [4.4], [5.5], [6.71], [6.93], [4.168], [9.779], [6.182], [7.59], 18 | [2.167], [7.042], [10.791], [5.313], [7.997], [3.1]], 19 | dtype=np.float32) 20 | 21 | y_train = np.array( 22 | [[1.7], [2.76], [2.09], [3.19], [1.694], [1.573], [3.366], [2.596], [2.53], 23 | [1.221], [2.827], [3.465], [1.65], [2.904], [1.3]], 24 | dtype=np.float32) 25 | 26 | train_iter = mx.io.NDArrayIter(x_train, y_train, batch_size=batch_size, shuffle=True, label_name='reg_label') 27 | 28 | # define network structure 29 | data = mx.sym.var('data') 30 | label = mx.sym.var('reg_label') 31 | reg = mx.sym.FullyConnected(data, num_hidden=1, name='fc') 32 | reg = mx.sym.LinearRegressionOutput(data=reg, label=label) 33 | 34 | mod = mx.mod.Module(reg, data_names=['data'], label_names=['reg_label'], context=ctx) 35 | mod.fit(train_iter, 36 | optimizer='sgd', 37 | optimizer_params={'learning_rate': 0.01, 'momentum': 0.9}, 38 | num_epoch=total_epochs, 39 | eval_metric='mse', 40 | batch_end_callback=mx.callback.Speedometer(batch_size, frequent=1), 41 | epoch_end_callback=mx.callback.do_checkpoint('linear', period=10)) 42 | 43 | # predict result 44 | mod.forward(mx.io.DataBatch(data=[mx.nd.array(x_train)], label=[mx.nd.array(y_train)]), is_train=False) 45 | predict = mod.get_outputs()[0].asnumpy() 46 | 47 | plt.plot(x_train, y_train, 'ro', label='actual') 48 | plt.plot(x_train, predict, label='predicted line') 49 | plt.show() 50 | -------------------------------------------------------------------------------- /02-Logistic Regression/logistic_regression.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | import mxnet as mx 5 | 6 | logging.basicConfig(level=logging.DEBUG, 7 | format='%(asctime)-15s %(message)s', 8 | stream=sys.stdout) 9 | 10 | # define hyperparameters 11 | batch_size = 32 12 | learning_rate = 1e-3 13 | epochs = 100 14 | step = 300 15 | ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu() 16 | 17 | # define dataset and dataloader 18 | mnist = mx.test_utils.get_mnist() 19 | 20 | train_iter = mx.io.NDArrayIter(mnist['train_data'], mnist['train_label'], batch_size, shuffle=True, 21 | data_name='data', label_name='softmax_label') 22 | test_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], batch_size * 2, shuffle=False, 23 | data_name='data', label_name='softmax_label') 24 | 25 | # define symbol 26 | data = mx.sym.var('data') # (bs, 1, 28, 28) 27 | label = mx.sym.var('softmax_label') 28 | data = mx.sym.Flatten(data) # (bs, 28*28) 29 | fc = mx.sym.FullyConnected(data, num_hidden=10, name='fc') 30 | logist = mx.sym.SoftmaxOutput(fc, label=label, name='softmax') 31 | 32 | metric_list = [mx.metric.Accuracy(output_names=['softmax_output'], label_names=['softmax_label']), 33 | mx.metric.CrossEntropy(output_names=['softmax_output'], label_names=['softmax_label'])] 34 | eval_metrics = mx.metric.CompositeEvalMetric(metric_list) 35 | 36 | mod = mx.mod.Module(logist, data_names=['data'], label_names=['softmax_label'], context=ctx) 37 | mod.fit( 38 | train_data=train_iter, 39 | eval_data=test_iter, 40 | optimizer='sgd', 41 | optimizer_params={'learning_rate': 0.01}, 42 | batch_end_callback=mx.callback.Speedometer(batch_size, 500), 43 | epoch_end_callback=mx.callback.do_checkpoint('logistic', 10), 44 | eval_metric=eval_metrics, 45 | num_epoch=epochs 46 | ) 47 | -------------------------------------------------------------------------------- /03-Neural Network/neural network.py: -------------------------------------------------------------------------------- 1 | import logging 2 | import sys 3 | 4 | import mxnet as mx 5 | 6 | logging.basicConfig(level=logging.DEBUG, 7 | format="%(asctime)-15s %(message)s", 8 | stream=sys.stdout) 9 | 10 | # define hyperparameters 11 | batch_size = 32 12 | learning_rate = 1e-2 13 | epochs = 100 14 | ctx = mx.gpu() if mx.test_utils.list_gpus() else mx.cpu() 15 | 16 | mnist = mx.test_utils.get_mnist() 17 | train_iter = mx.io.NDArrayIter(mnist['train_data'], mnist['train_label'], batch_size, shuffle=True, 18 | label_name='softmax_label') 19 | val_iter = mx.io.NDArrayIter(mnist['test_data'], mnist['test_label'], 2 * batch_size, shuffle=False, 20 | label_name='softmax_label') 21 | 22 | # define symbol 23 | data = mx.sym.var('data') 24 | label = mx.sym.var('softmax_label') 25 | 26 | data = mx.sym.Flatten(data) # (bs, 28*28) 27 | fc1 = mx.sym.FullyConnected(data, num_hidden=300, name='fc1') 28 | act1 = mx.sym.Activation(fc1, act_type='relu', name='relu1') 29 | fc2 = mx.sym.FullyConnected(act1, num_hidden=100, name='fc2') 30 | act2 = mx.sym.Activation(fc2, act_type='relu', name='relu2') 31 | fc3 = mx.sym.FullyConnected(act2, num_hidden=10, name='fc3') 32 | net = mx.sym.SoftmaxOutput(fc3, label=label, name='softmax') 33 | 34 | mod = mx.mod.Module(net, data_names=['data'], label_names=['softmax_label'], context=ctx) 35 | 36 | metric_list = [mx.metric.CrossEntropy(output_names=['softmax_output'], label_names=['softmax_label']), 37 | mx.metric.Accuracy(output_names=['softmax_output'], label_names=['softmax_label'])] 38 | eval_metrics = mx.metric.CompositeEvalMetric(metric_list) 39 | 40 | mod.fit( 41 | train_data=train_iter, 42 | eval_data=val_iter, 43 | initializer=mx.init.MSRAPrelu(), 44 | optimizer='sgd', 45 | optimizer_params={'learning_rate': 0.1}, 46 | eval_metric=eval_metrics, 47 | batch_end_callback=mx.callback.Speedometer(batch_size, 500), 48 | epoch_end_callback=mx.callback.do_checkpoint('nn', 10), 49 | num_epoch=epochs 50 | ) 51 | -------------------------------------------------------------------------------- /04-Convolution Neural Network/convolution_network.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import time 4 | 5 | import mxnet as mx 6 | import mxnet.gluon as g 7 | import numpy as np 8 | 9 | # define hyperparameters 10 | batch_size = 32 11 | learning_rate = 1e-2 12 | epochs = 100 13 | step = 300 14 | ctx = mx.gpu() 15 | 16 | 17 | # define data transform 18 | def data_transform(data, label): 19 | return mx.nd.transpose(data.astype(np.float32) / 255, 20 | (2, 0, 1)), label.astype(np.float32) 21 | 22 | 23 | # define dataset and dataloader 24 | train_dataset = g.data.vision.MNIST(transform=data_transform) 25 | test_dataset = g.data.vision.MNIST(train=False, transform=data_transform) 26 | 27 | train_loader = g.data.DataLoader( 28 | train_dataset, batch_size=batch_size, shuffle=True) 29 | test_loader = g.data.DataLoader( 30 | test_dataset, batch_size=batch_size, shuffle=False) 31 | 32 | # define model 33 | cnn = g.nn.Sequential(prefix='cnn_') 34 | with cnn.name_scope(): 35 | cnn.add(g.nn.Conv2D(6, 3, strides=1, padding=1, activation='relu')) 36 | cnn.add(g.nn.MaxPool2D(2, 2)) 37 | cnn.add(g.nn.Conv2D(16, 5, strides=1, padding=0, activation='relu')) 38 | cnn.add(g.nn.MaxPool2D(2, 2)) 39 | cnn.add(g.nn.Flatten()) 40 | cnn.add(g.nn.Dense(120, activation='relu')) 41 | cnn.add(g.nn.Dense(84, activation='relu')) 42 | cnn.add(g.nn.Dense(10)) 43 | 44 | cnn.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 45 | 46 | criterion = g.loss.SoftmaxCrossEntropyLoss() 47 | optimizer = g.Trainer(cnn.collect_params(), 'sgd', 48 | {'learning_rate': learning_rate}) 49 | 50 | # start train 51 | for e in range(epochs): 52 | print('*' * 10) 53 | print('epoch {}'.format(e + 1)) 54 | since = time.time() 55 | moving_loss = 0.0 56 | moving_acc = 0.0 57 | for i, (img, label) in enumerate(train_loader, 1): 58 | img = img.as_in_context(ctx) 59 | label = label.as_in_context(ctx) 60 | with mx.autograd.record(): 61 | output = cnn(img) 62 | loss = criterion(output, label) 63 | loss.backward() 64 | optimizer.step(img.shape[0]) 65 | # =========== keep average loss and accuracy ============== 66 | moving_loss += mx.nd.mean(loss).asscalar() 67 | predict = mx.nd.argmax(output, axis=1) 68 | acc = mx.nd.mean(predict == label).asscalar() 69 | moving_acc += acc 70 | 71 | if i % step == 0: 72 | print('[{}/{}] Loss: {:.6f}, Acc: {:.6f}'.format( 73 | i, len(train_loader), moving_loss / step, moving_acc / step)) 74 | moving_loss = 0.0 75 | moving_acc = 0.0 76 | test_loss = 0.0 77 | test_acc = 0.0 78 | total = 0.0 79 | for img, label in test_loader: 80 | img = img.as_in_context(ctx) 81 | label = label.as_in_context(ctx) 82 | output = cnn(img) 83 | loss = criterion(output, label) 84 | test_loss += mx.nd.sum(loss).asscalar() 85 | predict = mx.nd.argmax(output, axis=1) 86 | test_acc += mx.nd.sum(predict == label).asscalar() 87 | total += img.shape[0] 88 | print('Test Loss: {:.6f}, Test Acc: {:.6f}'.format(test_loss / total, 89 | test_acc / total)) 90 | print('Time: {:.1f} s'.format(time.time() - since)) 91 | 92 | cnn.save_params('./cnn.params') -------------------------------------------------------------------------------- /05-Recurrnet Neural Network/recurrent_network.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import time 4 | 5 | import mxnet as mx 6 | import mxnet.gluon as g 7 | import numpy as np 8 | 9 | # define hyperparameters 10 | batch_size = 100 11 | learning_rate = 1e-3 12 | epochs = 20 13 | step = 300 14 | ctx = mx.gpu() 15 | 16 | 17 | # define data transform 18 | def data_transform(data, label): 19 | return mx.nd.transpose(data.astype(np.float32) / 255, 20 | (2, 0, 1)), label.astype(np.float32) 21 | 22 | 23 | # define dataset and dataloader 24 | train_dataset = g.data.vision.MNIST(transform=data_transform) 25 | test_dataset = g.data.vision.MNIST(train=False, transform=data_transform) 26 | 27 | train_loader = g.data.DataLoader( 28 | train_dataset, batch_size=batch_size, shuffle=True) 29 | test_loader = g.data.DataLoader( 30 | test_dataset, batch_size=batch_size, shuffle=False) 31 | 32 | 33 | # define model 34 | class rnn(g.Block): 35 | def __init__(self, hidden, n_layer, n_class): 36 | super(rnn, self).__init__() 37 | with self.name_scope(): 38 | self.lstm = g.rnn.LSTM(hidden, n_layer) 39 | self.classifier = g.nn.Dense(n_class) 40 | 41 | def forward(self, x, hidden): 42 | out, hidden = self.lstm(x, hidden) 43 | out = out[out.shape[0] - 1, :, :] 44 | out = self.classifier(out) 45 | return out 46 | 47 | 48 | model = rnn(128, 2, 10) 49 | 50 | model.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 51 | 52 | criterion = g.loss.SoftmaxCrossEntropyLoss() 53 | optimizer = g.Trainer(model.collect_params(), 'adam', 54 | {'learning_rate': learning_rate}) 55 | 56 | # start train 57 | for e in range(epochs): 58 | print('*' * 10) 59 | print('epoch {}'.format(e + 1)) 60 | since = time.time() 61 | moving_loss = 0.0 62 | moving_acc = 0.0 63 | for i, (img, label) in enumerate(train_loader, 1): 64 | b, c, h, w = img.shape 65 | img = img.reshape((b, h, w)) 66 | img = mx.nd.transpose(img, (2, 0, 1)) 67 | img = img.as_in_context(ctx) 68 | label = label.as_in_context(ctx) 69 | h0 = mx.nd.zeros(shape=(2, b, 128), ctx=ctx) 70 | c0 = mx.nd.zeros(shape=(2, b, 128), ctx=ctx) 71 | with mx.autograd.record(): 72 | output = model(img, [h0, c0]) 73 | loss = criterion(output, label) 74 | loss.backward() 75 | optimizer.step(b) 76 | # =========== keep average loss and accuracy ============== 77 | moving_loss += mx.nd.mean(loss).asscalar() 78 | predict = mx.nd.argmax(output, axis=1) 79 | acc = mx.nd.mean(predict == label).asscalar() 80 | moving_acc += acc 81 | 82 | if i % step == 0: 83 | print('[{}/{}] Loss: {:.6f}, Acc: {:.6f}'.format( 84 | i, len(train_loader), moving_loss / step, moving_acc / step)) 85 | moving_loss = 0.0 86 | moving_acc = 0.0 87 | test_loss = 0.0 88 | test_acc = 0.0 89 | total = 0.0 90 | for img, label in test_loader: 91 | b, c, h, w = img.shape 92 | img = img.reshape((b, h, w)) 93 | img = mx.nd.transpose(img, (2, 0, 1)) 94 | img = img.as_in_context(ctx) 95 | label = label.as_in_context(ctx) 96 | h0 = mx.nd.zeros(shape=(2, b, 128), ctx=ctx) 97 | c0 = mx.nd.zeros(shape=(2, b, 128), ctx=ctx) 98 | output = model(img, [h0, c0]) 99 | loss = criterion(output, label) 100 | test_loss += mx.nd.sum(loss).asscalar() 101 | predict = mx.nd.argmax(output, axis=1) 102 | test_acc += mx.nd.sum(predict == label).asscalar() 103 | total += b 104 | print('Test Loss: {:.6f}, Test Acc: {:.6f}'.format(test_loss / total, 105 | test_acc / total)) 106 | print('Time: {:.1f} s'.format(time.time() - since)) 107 | 108 | model.save_params('./lstm.params') -------------------------------------------------------------------------------- /06-Natural Language Process/N-Gram.py: -------------------------------------------------------------------------------- 1 | import mxnet as mx 2 | import mxnet.gluon as g 3 | import numpy as np 4 | 5 | ctx = mx.gpu() 6 | CONTEXT_SIZE = 2 7 | EMBEDDING_DIM = 10 8 | # We will use Shakespeare Sonnet 2 9 | test_sentence = """When forty winters shall besiege thy brow, 10 | And dig deep trenches in thy beauty's field, 11 | Thy youth's proud livery so gazed on now, 12 | Will be a totter'd weed of small worth held: 13 | Then being asked, where all thy beauty lies, 14 | Where all the treasure of thy lusty days; 15 | To say, within thine own deep sunken eyes, 16 | Were an all-eating shame, and thriftless praise. 17 | How much more praise deserv'd thy beauty's use, 18 | If thou couldst answer 'This fair child of mine 19 | Shall sum my count, and make my old excuse,' 20 | Proving his beauty by succession thine! 21 | This were to be new made when thou art old, 22 | And see thy blood warm when thou feel'st it cold.""".split() 23 | 24 | trigram = [((test_sentence[i], test_sentence[i + 1]), test_sentence[i + 2]) 25 | for i in range(len(test_sentence) - 2)] 26 | 27 | vocb = set(test_sentence) 28 | word_to_idx = {word: i for i, word in enumerate(vocb)} 29 | idx_to_word = {word_to_idx[word]: word for word in word_to_idx} 30 | 31 | 32 | class NgramModel(g.Block): 33 | def __init__(self, vocb_size, context_size, n_dim): 34 | super(NgramModel, self).__init__() 35 | with self.name_scope(): 36 | self.n_word = vocb_size 37 | self.embedding = g.nn.Embedding(self.n_word, n_dim) 38 | self.linear1 = g.nn.Dense(128, activation='relu') 39 | self.linear2 = g.nn.Dense(self.n_word) 40 | 41 | def forward(self, x): 42 | emb = self.embedding(x) 43 | emb = emb.reshape((1, -1)) 44 | out = self.linear1(emb) 45 | out = self.linear2(out) 46 | log_prob = mx.nd.log_softmax(out) 47 | return log_prob 48 | 49 | 50 | ngrammodel = NgramModel(len(word_to_idx), CONTEXT_SIZE, 100) 51 | ngrammodel.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 52 | criterion = g.loss.SoftmaxCrossEntropyLoss() 53 | optimizer = g.Trainer(ngrammodel.collect_params(), 'adam', 54 | {'learning_rate': 1e-3}) 55 | 56 | for epoch in range(100): 57 | print('epoch: {}'.format(epoch + 1)) 58 | print('*' * 10) 59 | running_loss = 0 60 | for data in trigram: 61 | word, label = data 62 | word = mx.nd.array( 63 | [word_to_idx[i] for i in word], ctx=ctx, dtype=np.int32) 64 | label = mx.nd.array([word_to_idx[label]], ctx=ctx, dtype=np.int32) 65 | # forward 66 | with mx.autograd.record(): 67 | out = ngrammodel(word) 68 | loss = criterion(out, label) 69 | running_loss += mx.nd.mean(loss).asscalar() 70 | # backward 71 | loss.backward() 72 | optimizer.step(1) 73 | print('Loss: {:.6f}'.format(running_loss / len(word_to_idx))) 74 | 75 | word, label = trigram[3] 76 | word = mx.nd.array([word_to_idx[i] for i in word], ctx=ctx, dtype=np.int32) 77 | out = ngrammodel(word) 78 | predict_label = mx.nd.argmax(out, axis=1) 79 | predict_word = idx_to_word[predict_label.asscalar().astype(np.int32)] 80 | print('real word is {}, predict word is {}'.format(label, predict_word)) 81 | -------------------------------------------------------------------------------- /06-Natural Language Process/bag-of-word.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import mxnet as mx 4 | import mxnet.gluon as g 5 | import numpy as np 6 | 7 | ctx = mx.gpu() 8 | # The Continuous Bag-of-Words model (CBOW) is frequently used in NLP deep learning. It is a model that tries to predict words given the context of a few words before and a few words after the target word. This is distinct from language modeling, since CBOW is not sequential and does not have to be probabilistic. Typcially, CBOW is used to quickly train word embeddings, and these embeddings are used to initialize the embeddings of some more complicated model. Usually, this is referred to as pretraining embeddings. It almost always helps performance a couple of percent. 9 | 10 | CONTEXT_SIZE = 2 # 2 words to the left, 2 to the right 11 | raw_text = """We are about to study the idea of a computational process. 12 | Computational processes are abstract beings that inhabit computers. 13 | As they evolve, processes manipulate other abstract things called data. 14 | The evolution of a process is directed by a pattern of rules 15 | called a program. People create programs to direct processes. In effect, 16 | we conjure the spirits of the computer with our spells.""".split() 17 | 18 | vocab = set(raw_text) 19 | word_to_idx = {word: i for i, word in enumerate(vocab)} 20 | 21 | data = [] 22 | for i in range(CONTEXT_SIZE, len(raw_text) - CONTEXT_SIZE): 23 | context = [ 24 | raw_text[i - 2], raw_text[i - 1], raw_text[i + 1], raw_text[i + 2] 25 | ] 26 | target = raw_text[i] 27 | data.append((context, target)) 28 | 29 | 30 | class CBOW(g.Block): 31 | def __init__(self, n_word, n_dim, context_size): 32 | super(CBOW, self).__init__() 33 | with self.name_scope(): 34 | self.embedding = g.nn.Embedding(n_word, n_dim) 35 | self.linear1 = g.nn.Dense(128, activation='relu') 36 | self.linear2 = g.nn.Dense(n_word) 37 | 38 | def forward(self, x): 39 | x = self.embedding(x) 40 | x = x.reshape((1, -1)) 41 | x = self.linear1(x) 42 | x = self.linear2(x) 43 | x = mx.nd.log_softmax(x) 44 | return x 45 | 46 | 47 | model = CBOW(len(word_to_idx), 100, CONTEXT_SIZE) 48 | model.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 49 | 50 | criterion = g.loss.SoftmaxCrossEntropyLoss() 51 | optimizer = g.Trainer(model.collect_params(), 'sgd', {'learning_rate': 1e-2}) 52 | 53 | for epoch in range(100): 54 | print('epoch {}'.format(epoch)) 55 | print('*' * 10) 56 | running_loss = 0 57 | for word in data: 58 | context, target = word 59 | context = mx.nd.array( 60 | [word_to_idx[i] for i in context], ctx=ctx).astype(np.int32) 61 | # context = context.astype(np.int32) 62 | target = mx.nd.array([word_to_idx[target]], ctx=ctx).astype(np.int32) 63 | # target = target.astype(np.int32) 64 | # forward 65 | with mx.autograd.record(): 66 | out = model(context) 67 | loss = criterion(out, target) 68 | running_loss += mx.nd.mean(loss).asscalar() 69 | # backward 70 | loss.backward() 71 | optimizer.step(1) 72 | print('loss: {:.6f}'.format(running_loss / len(data))) 73 | -------------------------------------------------------------------------------- /06-Natural Language Process/seq-lstm.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import mxnet as mx 4 | import mxnet.gluon as g 5 | import numpy as np 6 | 7 | ctx = mx.gpu() 8 | 9 | training_data = [("The dog ate the apple".split(), 10 | ["DET", "NN", "V", "DET", "NN"]), 11 | ("Everybody read that book".split(), ["NN", "V", "DET", 12 | "NN"])] 13 | 14 | word_to_idx = {} 15 | tag_to_idx = {} 16 | for context, tag in training_data: 17 | for word in context: 18 | if word not in word_to_idx: 19 | word_to_idx[word] = len(word_to_idx) 20 | for label in tag: 21 | if label not in tag_to_idx: 22 | tag_to_idx[label] = len(tag_to_idx) 23 | alphabet = 'abcdefghijklmnopqrstuvwxyz' 24 | character_to_idx = {} 25 | for i in range(len(alphabet)): 26 | character_to_idx[alphabet[i]] = i 27 | 28 | 29 | class CharLSTM(g.Block): 30 | def __init__(self, n_char, char_dim, char_hidden): 31 | super(CharLSTM, self).__init__() 32 | with self.name_scope(): 33 | self.char_embedding = g.nn.Embedding(n_char, char_dim) 34 | self.char_lstm = g.rnn.LSTM(char_hidden) 35 | 36 | def forward(self, x, hidden): 37 | x = self.char_embedding(x) 38 | x = mx.nd.transpose(x, (1, 0, 2)) 39 | _, h = self.char_lstm(x, hidden) 40 | return h[0] 41 | 42 | 43 | class LSTMTagger(g.Block): 44 | def __init__(self, n_word, n_char, char_dim, n_dim, char_hidden, n_hidden, 45 | n_tag): 46 | super(LSTMTagger, self).__init__() 47 | with self.name_scope(): 48 | self.word_embedding = g.nn.Embedding(n_word, n_dim) 49 | self.char_lstm = CharLSTM(n_char, char_dim, char_hidden) 50 | self.lstm = g.rnn.LSTM(n_hidden) 51 | self.linear = g.nn.Dense(n_tag) 52 | 53 | def forward(self, x, char_hidden, hidden, word): 54 | for i, each in enumerate(word): 55 | char_list = [] 56 | for letter in each: 57 | char_list.append(character_to_idx[letter.lower()]) 58 | char_list = mx.nd.array(char_list, dtype=np.int32, ctx=ctx) 59 | char_list = mx.nd.expand_dims(char_list, axis=0) 60 | tempchar = self.char_lstm(char_list, char_hidden) 61 | t_shape = tempchar.shape 62 | tempchar = tempchar.reshape((t_shape[0], t_shape[2])) 63 | if i == 0: 64 | char = tempchar 65 | else: 66 | char = mx.nd.concat(char, tempchar, dim=0) 67 | x = self.word_embedding(x) 68 | x = mx.nd.concat(x, char, dim=1) 69 | x = mx.nd.expand_dims(x, axis=1) 70 | x, _ = self.lstm(x, hidden) 71 | x = x.reshape((x.shape[0], x.shape[2])) 72 | x = self.linear(x) 73 | y = mx.nd.log_softmax(x) 74 | return y 75 | 76 | 77 | model = LSTMTagger( 78 | len(word_to_idx), len(character_to_idx), 10, 100, 50, 128, len(tag_to_idx)) 79 | 80 | model.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 81 | 82 | criterion = g.loss.SoftmaxCrossEntropyLoss() 83 | optimizer = g.Trainer(model.collect_params(), 'sgd', {'learning_rate': 1e-2}) 84 | 85 | 86 | def make_sequence(x, dic): 87 | idx = mx.nd.array([dic[i] for i in x], ctx=ctx, dtype=np.int32) 88 | return idx 89 | 90 | 91 | for epoch in range(300): 92 | print('*' * 10) 93 | print('epoch {}'.format(epoch + 1)) 94 | running_loss = 0 95 | for data in training_data: 96 | word, tag = data 97 | word_list = make_sequence(word, word_to_idx) 98 | tag = make_sequence(tag, tag_to_idx) 99 | # forward 100 | h0 = mx.nd.zeros((1, 1, 128), ctx=ctx) 101 | c0 = mx.nd.zeros((1, 1, 128), ctx=ctx) 102 | char_h0 = mx.nd.zeros((1, 1, 50), ctx=ctx) 103 | char_c0 = mx.nd.zeros((1, 1, 50), ctx=ctx) 104 | with mx.autograd.record(): 105 | out = model(word_list, [char_h0, char_c0], [h0, c0], word) 106 | loss = criterion(out, tag) 107 | running_loss += mx.nd.mean(loss).asscalar() 108 | # backward 109 | loss.backward() 110 | optimizer.step(1) 111 | print('Loss: {}'.format(running_loss / len(data))) 112 | print() 113 | input = make_sequence("Everybody ate the apple".split(), word_to_idx) 114 | 115 | h0 = mx.nd.zeros((1, 1, 128), ctx=ctx) 116 | c0 = mx.nd.zeros((1, 1, 128), ctx=ctx) 117 | char_h0 = mx.nd.zeros((1, 1, 50), ctx=ctx) 118 | char_c0 = mx.nd.zeros((1, 1, 50), ctx=ctx) 119 | out = model(input, [char_h0, char_c0], [h0, c0], 120 | "Everybody ate the apple".split()) 121 | print(out) 122 | -------------------------------------------------------------------------------- /08-AutoEncoder/Variational_autoencoder.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import os 4 | 5 | import mxnet as mx 6 | import numpy as np 7 | import torch 8 | from mxnet import gluon as g 9 | from mxnet import ndarray as nd 10 | from torchvision.utils import save_image 11 | 12 | if not os.path.exists('./vae_img'): 13 | os.mkdir('./vae_img') 14 | 15 | 16 | def to_img(x): 17 | x = x.clamp(0, 1) 18 | x = x.view(x.size(0), 1, 28, 28) 19 | return x 20 | 21 | 22 | num_epochs = 100 23 | batch_size = 128 24 | learning_rate = 1e-3 25 | ctx = mx.gpu() 26 | 27 | 28 | def img_transform(data, label): 29 | return data.astype('float32') / 255, label.astype('float32') 30 | 31 | 32 | dataset = g.data.vision.MNIST(transform=img_transform) 33 | dataloader = g.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) 34 | 35 | 36 | class VAE(g.HybridBlock): 37 | def __init__(self): 38 | super(VAE, self).__init__() 39 | with self.name_scope(): 40 | self.fc1 = g.nn.Dense(400) 41 | self.fc21 = g.nn.Dense(20) 42 | self.fc22 = g.nn.Dense(20) 43 | self.fc3 = g.nn.Dense(400) 44 | self.fc4 = g.nn.Dense(784) 45 | # self.fc1 = g.nn.Dense( 46 | # 400, 47 | # in_units=784, 48 | # weight_initializer=mx.init.Uniform(1. / np.sqrt(784)), 49 | # bias_initializer=mx.init.Uniform(1. / np.sqrt(784))) 50 | # self.fc21 = g.nn.Dense( 51 | # 20, 52 | # in_units=400, 53 | # weight_initializer=mx.init.Uniform(1. / np.sqrt(400)), 54 | # bias_initializer=mx.init.Uniform(1. / np.sqrt(400))) 55 | # self.fc22 = g.nn.Dense( 56 | # 20, 57 | # in_units=400, 58 | # weight_initializer=mx.init.Uniform(1. / np.sqrt(400)), 59 | # bias_initializer=mx.init.Uniform(1. / np.sqrt(400))) 60 | # self.fc3 = g.nn.Dense( 61 | # 400, 62 | # in_units=20, 63 | # weight_initializer=mx.init.Uniform(1. / np.sqrt(20)), 64 | # bias_initializer=mx.init.Uniform(1. / np.sqrt(20))) 65 | # self.fc4 = g.nn.Dense( 66 | # 784, 67 | # in_units=400, 68 | # weight_initializer=mx.init.Uniform(1. / np.sqrt(784)), 69 | # bias_initializer=mx.init.Uniform(1. / np.sqrt(784))) 70 | 71 | def encode(self, x): 72 | h1 = nd.Activation(self.fc1(x), 'relu') 73 | return self.fc21(h1), self.fc22(h1) 74 | 75 | def reparametrize(self, mu, logvar): 76 | ''' 77 | mu is a number and logvar is a ndarray 78 | ''' 79 | std = nd.exp(0.5 * logvar) 80 | eps = nd.random_normal( 81 | loc=0, scale=1, shape=std.shape).as_in_context(ctx) 82 | return mu + eps * std 83 | 84 | def decode(self, z): 85 | h3 = nd.Activation(self.fc3(z), 'relu') 86 | return nd.Activation(self.fc4(h3), 'sigmoid') 87 | 88 | def forward(self, x): 89 | mu, logvar = self.encode(x) 90 | z = self.reparametrize(mu, logvar) 91 | return self.decode(z), mu, logvar 92 | 93 | 94 | model = VAE() 95 | model.collect_params().initialize(mx.init.Uniform(1 / np.sqrt(400)), ctx=ctx) 96 | 97 | reconstruction_function = g.loss.L2Loss() 98 | 99 | 100 | def loss_function(recon_x, x, mu, logvar): 101 | """ 102 | recon_x: generating images 103 | x: origin images 104 | mu: latent mean 105 | logvar: latent log variance 106 | """ 107 | BCE = reconstruction_function(recon_x, x) # mse loss 108 | BCE = nd.sum(BCE) 109 | # loss = 0.5 * sum(1 - log(sigma^2) + mu^2 + sigma^2) 110 | KLD_element = (nd.power(mu, 2) + nd.exp(logvar)) * (-1) + 1 + logvar 111 | KLD = nd.sum(KLD_element) * (-0.5) 112 | # KLD_element = nd.exp(logvar) + nd.power(mu, 2) - logvar - 1 113 | # KLD = nd.sum(KLD_element) * 0.5 114 | # KL divergence 115 | return BCE + KLD 116 | 117 | 118 | optimizer = g.Trainer(model.collect_params(), 'adam', 119 | {'learning_rate': learning_rate}) 120 | 121 | for epoch in range(num_epochs): 122 | train_loss = 0 123 | for batch_idx, data in enumerate(dataloader): 124 | img, _ = data 125 | batch = img.shape[0] 126 | img = img.reshape((batch, -1)) 127 | img = img.as_in_context(ctx) 128 | with mx.autograd.record(): 129 | recon_batch, mu, logvar = model(img) 130 | loss = loss_function(recon_batch, img, mu, logvar) 131 | loss.backward() 132 | train_loss += loss.asscalar() 133 | optimizer.step(batch) 134 | if batch_idx % 100 == 0: 135 | print('Train Epoch: {} [{}/{} ({:.0f}%)]\tLoss: {:.6f}'.format( 136 | epoch, batch_idx * batch, 137 | len(dataset), 100. * batch_idx / int( 138 | len(dataset) / batch_size), loss.asscalar() / batch)) 139 | 140 | print('====> Epoch: {} Average loss: {:.4f}'.format( 141 | epoch, train_loss / len(dataset))) 142 | if epoch % 10 == 0: 143 | save = to_img(torch.FloatTensor(recon_batch.asnumpy())) 144 | save_image(save, './vae_img/image_{}.png'.format(epoch)) 145 | 146 | model.save_params('./vae.params') 147 | -------------------------------------------------------------------------------- /08-AutoEncoder/conv_autoencoder.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import mxnet as mx 4 | from mxnet import gluon as g 5 | from mxnet import nd 6 | import torch 7 | from torchvision.utils import save_image 8 | import os 9 | 10 | ctx = mx.gpu() 11 | 12 | if not os.path.exists('./dc_img'): 13 | os.mkdir('./dc_img') 14 | 15 | 16 | def to_img(x): 17 | x = 0.5 * (x + 1) 18 | x = x.clamp(0, 1) 19 | x = x.view(x.size(0), 1, 28, 28) 20 | return x 21 | 22 | 23 | num_epochs = 100 24 | batch_size = 128 25 | learning_rate = 1e-3 26 | 27 | 28 | def img_transform(data, label): 29 | data = data.transpose((2, 0, 1)) 30 | return (data.astype('float32') / 255 - 0.5) / 0.5, label.astype('float32') 31 | 32 | 33 | dataset = g.data.vision.MNIST(transform=img_transform) 34 | dataloader = g.data.DataLoader(dataset, batch_size=batch_size, shuffle=True) 35 | 36 | 37 | class autoencoder(g.nn.HybridBlock): 38 | def __init__(self): 39 | super(autoencoder, self).__init__() 40 | with self.name_scope(): 41 | self.encoder = g.nn.HybridSequential('encoder_') 42 | with self.encoder.name_scope(): 43 | # b, 16, 10, 10 44 | self.encoder.add( 45 | g.nn.Conv2D( 46 | 16, 3, strides=3, padding=1, activation='relu')) 47 | self.encoder.add(g.nn.MaxPool2D(2, 2)) # b, 16, 5, 5 48 | self.encoder.add( 49 | g.nn.Conv2D(8, 3, strides=2, padding=1, 50 | activation='relu')) # b, 8, 3, 3 51 | self.encoder.add(g.nn.MaxPool2D(2, 1)) # b, 8, 2, 2 52 | 53 | self.decoder = g.nn.HybridSequential('decoder_') 54 | with self.decoder.name_scope(): 55 | self.decoder.add( 56 | g.nn.Conv2DTranspose(16, 3, strides=2, activation='relu')) 57 | self.decoder.add( 58 | g.nn.Conv2DTranspose( 59 | 8, 5, strides=3, padding=1, activation='relu')) 60 | self.decoder.add( 61 | g.nn.Conv2DTranspose( 62 | 1, 2, strides=2, padding=1, activation='tanh')) 63 | 64 | def forward(self, x): 65 | x = self.encoder(x) 66 | x = self.decoder(x) 67 | return x 68 | 69 | 70 | model = autoencoder() 71 | model.hybridize() 72 | model.collect_params().initialize(mx.init.Xavier('gaussian'), ctx=ctx) 73 | 74 | criterion = g.loss.L2Loss() 75 | optimizer = g.Trainer(model.collect_params(), 'adam', 76 | {'learning_rate': learning_rate, 77 | 'wd': 1e-5}) 78 | 79 | for epoch in range(num_epochs): 80 | for data in dataloader: 81 | img, _ = data 82 | img = img.as_in_context(ctx) 83 | batch = img.shape[0] 84 | # ===================forward===================== 85 | with mx.autograd.record(): 86 | output = model(img) 87 | loss = criterion(output, img) 88 | # ===================backward==================== 89 | loss.backward() 90 | optimizer.step(batch) 91 | # ===================log======================== 92 | print('epoch [{}/{}], loss:{:.4f}' 93 | .format(epoch + 1, num_epochs, nd.mean(loss).asscalar())) 94 | if epoch % 10 == 0: 95 | pic = to_img(torch.FloatTensor(output.asnumpy())) 96 | save_image(pic, './dc_img/image_{}.png'.format(epoch)) 97 | 98 | model.save_params('./con_autoencoder.params') -------------------------------------------------------------------------------- /08-AutoEncoder/simple_autoencoder.py: -------------------------------------------------------------------------------- 1 | __author__ = 'SherlockLiao' 2 | 3 | import os 4 | 5 | import mxnet as mx 6 | import numpy as np 7 | import torch 8 | from mxnet import gluon 9 | from torchvision.utils import save_image 10 | 11 | if not os.path.exists('./mlp_img'): 12 | os.mkdir('./mlp_img') 13 | 14 | 15 | def to_img(x): 16 | x = 0.5 * (x + 1) 17 | x = x.clamp(0, 1) 18 | x = x.view(x.size(0), 1, 28, 28) 19 | return x 20 | 21 | 22 | num_epochs = 100 23 | batch_size = 128 24 | learning_rate = 1e-3 25 | ctx = mx.gpu() 26 | 27 | 28 | def transform(data, label): 29 | return (data.astype('float32') / 255 - 0.5) / 0.5, label.astype('float32') 30 | 31 | 32 | mnist_train = gluon.data.vision.FashionMNIST(train=True, transform=transform) 33 | 34 | dataloader = gluon.data.DataLoader( 35 | mnist_train, batch_size=batch_size, shuffle=True) 36 | 37 | 38 | class autoencoder(gluon.Block): 39 | def __init__(self): 40 | super(autoencoder, self).__init__() 41 | with self.name_scope(): 42 | self.encoder = gluon.nn.Sequential('encoder_') 43 | with self.encoder.name_scope(): 44 | self.encoder.add(gluon.nn.Dense(128, activation='relu')) 45 | self.encoder.add(gluon.nn.Dense(64, activation='relu')) 46 | self.encoder.add(gluon.nn.Dense(12, activation='relu')) 47 | self.encoder.add(gluon.nn.Dense(3)) 48 | 49 | self.decoder = gluon.nn.Sequential('decoder_') 50 | with self.decoder.name_scope(): 51 | self.decoder.add(gluon.nn.Dense(12, activation='relu')) 52 | self.decoder.add(gluon.nn.Dense(64, activation='relu')) 53 | self.decoder.add(gluon.nn.Dense(128, activation='relu')) 54 | self.decoder.add(gluon.nn.Dense(28 * 28, activation='tanh')) 55 | 56 | def forward(self, x): 57 | x = self.encoder(x) 58 | x = self.decoder(x) 59 | return x 60 | 61 | 62 | model = autoencoder() 63 | model.collect_params().initialize(mx.init.Xavier(), ctx=ctx) 64 | criterion = gluon.loss.L2Loss() 65 | optimizer = gluon.Trainer(model.collect_params(), 'adam', 66 | {'learning_rate': learning_rate, 67 | 'wd': 1e-5}) 68 | 69 | for epoch in range(num_epochs): 70 | running_loss = 0.0 71 | n_total = 0.0 72 | for data in dataloader: 73 | img, _ = data 74 | img = img.reshape((img.shape[0], -1)).as_in_context(ctx) 75 | 76 | with mx.autograd.record(): 77 | output = model(img) 78 | loss = criterion(output, img) 79 | loss.backward() 80 | optimizer.step(img.shape[0]) 81 | running_loss += mx.nd.sum(loss).asscalar() 82 | n_total += img.shape[0] 83 | # ===================log======================== 84 | print('epoch [{}/{}], loss:{:.4f}' 85 | .format(epoch + 1, num_epochs, running_loss / n_total)) 86 | if epoch % 10 == 0: 87 | pic = to_img(torch.FloatTensor(output.asnumpy())) 88 | save_image(pic, './mlp_img/{}_autoencoder.png'.format(epoch)) 89 | model.save_params('./simple_autoencoder.params') 90 | -------------------------------------------------------------------------------- /09-Generative Adversarial network/conv_gan.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import mxnet as mx 4 | import torch 5 | from mxnet import gluon as g 6 | from mxnet import ndarray as nd 7 | from torchvision.utils import save_image 8 | 9 | if not os.path.exists('./img'): 10 | os.mkdir('./img') 11 | 12 | 13 | def to_img(x): 14 | out = 0.5 * (x + 1) 15 | out = out.clamp(0, 1) 16 | out = out.view(-1, 1, 28, 28) 17 | return out 18 | 19 | 20 | batch_size = 128 21 | num_epoch = 5 22 | z_dimension = 100 23 | ctx = mx.gpu() 24 | 25 | 26 | # Image processing 27 | def img_transform(data, label): 28 | data = (data.astype('float32') / 255 - 0.5) / 0.5 29 | return data.transpose((2, 0, 1)), label.astype('float32') 30 | 31 | 32 | # MNIST dataset 33 | mnist = g.data.vision.MNIST(transform=img_transform) 34 | # Data loader 35 | dataloader = g.data.DataLoader( 36 | dataset=mnist, batch_size=batch_size, shuffle=True) 37 | 38 | 39 | # Discriminator 40 | class discriminator(g.HybridBlock): 41 | def __init__(self): 42 | super(discriminator, self).__init__() 43 | with self.name_scope(): 44 | self.conv1 = g.nn.HybridSequential(prefix='conv1_') 45 | with self.conv1.name_scope(): 46 | self.conv1.add(g.nn.Conv2D(32, 5, padding=2)) 47 | self.conv1.add(g.nn.LeakyReLU(0.2)) 48 | self.conv1.add(g.nn.AvgPool2D(2, 2)) 49 | 50 | self.conv2 = g.nn.HybridSequential(prefix='conv2_') 51 | with self.conv2.name_scope(): 52 | self.conv2.add(g.nn.Conv2D(64, 5, padding=2)) 53 | self.conv2.add(g.nn.LeakyReLU(0.2)) 54 | self.conv2.add(g.nn.AvgPool2D(2, 2)) 55 | 56 | self.fc = g.nn.HybridSequential(prefix='fc_') 57 | with self.fc.name_scope(): 58 | self.fc.add(g.nn.Flatten()) 59 | self.fc.add(g.nn.Dense(1024)) 60 | self.fc.add(g.nn.LeakyReLU(0.2)) 61 | self.fc.add(g.nn.Dense(1)) 62 | 63 | def forward(self, x): 64 | x = self.conv1(x) 65 | x = self.conv2(x) 66 | x = self.fc(x) 67 | return x 68 | 69 | 70 | d = discriminator() 71 | d.hybridize() 72 | 73 | 74 | # Generator 75 | class generator(g.HybridBlock): 76 | def __init__(self, num_feature): 77 | super(generator, self).__init__() 78 | with self.name_scope(): 79 | self.fc = g.nn.Dense(num_feature) 80 | 81 | self.br = g.nn.HybridSequential(prefix='batch_relu_') 82 | with self.br.name_scope(): 83 | self.br.add(g.nn.BatchNorm()) 84 | self.br.add(g.nn.Activation('relu')) 85 | 86 | self.downsample = g.nn.HybridSequential(prefix='ds_') 87 | with self.downsample.name_scope(): 88 | self.downsample.add(g.nn.Conv2D(50, 3, strides=1, padding=1)) 89 | self.downsample.add(g.nn.BatchNorm()) 90 | self.downsample.add(g.nn.Activation('relu')) 91 | self.downsample.add(g.nn.Conv2D(25, 3, strides=1, padding=1)) 92 | self.downsample.add(g.nn.BatchNorm()) 93 | self.downsample.add(g.nn.Activation('relu')) 94 | self.downsample.add( 95 | g.nn.Conv2D(1, 2, strides=2, activation='tanh')) 96 | 97 | def forward(self, x): 98 | x = self.fc(x) 99 | x = x.reshape((x.shape[0], 1, 56, 56)) 100 | x = self.br(x) 101 | x = self.downsample(x) 102 | return x 103 | 104 | 105 | ge = generator(3136) 106 | ge.hybridize() 107 | 108 | d.collect_params().initialize(mx.init.Xavier(), ctx) 109 | ge.collect_params().initialize(mx.init.Xavier(), ctx) 110 | # Binary cross entropy loss and optimizer 111 | bce = g.loss.SigmoidBinaryCrossEntropyLoss() 112 | 113 | d_optimizer = g.Trainer(d.collect_params(), 'adam', {'learning_rate': 0.0003}) 114 | g_optimizer = g.Trainer(ge.collect_params(), 'adam', {'learning_rate': 0.0003}) 115 | 116 | # Start training 117 | for epoch in range(num_epoch): 118 | for i, (img, _) in enumerate(dataloader): 119 | num_img = img.shape[0] 120 | # =================train discriminator 121 | real_img = img.as_in_context(ctx) 122 | real_label = nd.ones(shape=[num_img], ctx=ctx) 123 | fake_label = nd.zeros(shape=[num_img], ctx=ctx) 124 | 125 | # compute loss of real_img 126 | with mx.autograd.record(): 127 | real_out = d(real_img) 128 | d_loss_real = bce(real_out, real_label) 129 | real_scores = real_out # closer to 1 means better 130 | 131 | # compute loss of fake_img 132 | z = nd.random_normal( 133 | loc=0, scale=1, shape=[num_img, z_dimension], ctx=ctx) 134 | with g.autograd.record(): 135 | fake_img = ge(z) 136 | fake_out = d(fake_img) 137 | d_loss_fake = bce(fake_out, fake_label) 138 | fake_scores = fake_out # closer to 0 means better 139 | 140 | # bp and optimize 141 | with g.autograd.record(): 142 | d_loss = d_loss_real + d_loss_fake 143 | d_loss.backward() 144 | d_optimizer.step(num_img) 145 | 146 | # ===============train generator 147 | # compute loss of fake_img 148 | with g.autograd.record(): 149 | fake_img = ge(z) 150 | output = d(fake_img) 151 | g_loss = bce(output, real_label) 152 | 153 | # bp and optimize 154 | g_loss.backward() 155 | g_optimizer.step(num_img) 156 | 157 | if (i + 1) % 100 == 0: 158 | print('Epoch [{}/{}], d_loss: {:.6f}, g_loss: {:.6f} ' 159 | 'D real: {:.6f}, D fake: {:.6f}'.format( 160 | epoch, num_epoch, 161 | nd.mean(d_loss).asscalar(), 162 | nd.mean(g_loss).asscalar(), 163 | nd.mean(real_scores).asscalar(), 164 | nd.mean(fake_scores).asscalar())) 165 | if epoch == 0: 166 | real_images = to_img(torch.FloatTensor(real_img.asnumpy())) 167 | save_image(real_images, './img/real_images.png') 168 | 169 | fake_images = to_img(torch.FloatTensor(fake_img.asnumpy())) 170 | save_image(fake_images, './img/fake_images-{}.png'.format(epoch + 1)) 171 | 172 | d.save_params('./dis.params') 173 | ge.save_params('./gen.params') 174 | -------------------------------------------------------------------------------- /09-Generative Adversarial network/simple_gan.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import mxnet as mx 4 | import torch 5 | from mxnet import gluon as g 6 | from mxnet import ndarray as nd 7 | from torchvision.utils import save_image 8 | 9 | if not os.path.exists('./img'): 10 | os.mkdir('./img') 11 | 12 | 13 | def to_img(x): 14 | out = 0.5 * (x + 1) 15 | out = out.clamp(0, 1) 16 | out = out.view(-1, 1, 28, 28) 17 | return out 18 | 19 | 20 | batch_size = 128 21 | num_epoch = 100 22 | z_dimension = 100 23 | ctx = mx.gpu() 24 | 25 | 26 | # Image processing 27 | def img_transform(data, label): 28 | data = (data.astype('float32') / 255 - 0.5) / 0.5 29 | return data.transpose((2, 0, 1)), label.astype('float32') 30 | 31 | 32 | # MNIST dataset 33 | mnist = g.data.vision.MNIST(transform=img_transform) 34 | # Data loader 35 | dataloader = g.data.DataLoader( 36 | dataset=mnist, batch_size=batch_size, shuffle=True) 37 | 38 | # Discriminator 39 | discriminator = g.nn.HybridSequential(prefix='dis_') 40 | with discriminator.name_scope(): 41 | discriminator.add(g.nn.Dense(256)) 42 | discriminator.add(g.nn.LeakyReLU(0.2)) 43 | discriminator.add(g.nn.Dense(256)) 44 | discriminator.add(g.nn.LeakyReLU(0.2)) 45 | discriminator.add(g.nn.Dense(1, activation='sigmoid')) 46 | 47 | discriminator.hybridize() 48 | # Generator 49 | generator = g.nn.HybridSequential(prefix='gen_') 50 | with generator.name_scope(): 51 | generator.add(g.nn.Dense(256, activation='relu')) 52 | generator.add(g.nn.Dense(256, activation='relu')) 53 | generator.add(g.nn.Dense(784, activation='tanh')) 54 | 55 | generator.hybridize() 56 | 57 | discriminator.collect_params().initialize(mx.init.Xavier(), ctx) 58 | generator.collect_params().initialize(mx.init.Xavier(), ctx) 59 | # Binary cross entropy loss and optimizer 60 | bce = g.loss.SigmoidBinaryCrossEntropyLoss(from_sigmoid=True) 61 | 62 | d_optimizer = g.Trainer(discriminator.collect_params(), 'adam', 63 | {'learning_rate': 0.0003}) 64 | g_optimizer = g.Trainer(generator.collect_params(), 'adam', 65 | {'learning_rate': 0.0003}) 66 | 67 | # Start training 68 | for epoch in range(num_epoch): 69 | for i, (img, _) in enumerate(dataloader): 70 | num_img = img.shape[0] 71 | # =================train discriminator 72 | img = img.reshape((num_img, -1)) 73 | real_img = img.as_in_context(ctx) 74 | real_label = nd.ones(shape=[num_img], ctx=ctx) 75 | fake_label = nd.zeros(shape=[num_img], ctx=ctx) 76 | 77 | # compute loss of real_img 78 | with mx.autograd.record(): 79 | real_out = discriminator(real_img) 80 | d_loss_real = bce(real_out, real_label) 81 | real_scores = real_out # closer to 1 means better 82 | 83 | # compute loss of fake_img 84 | z = nd.random_normal( 85 | loc=0, scale=1, shape=[num_img, z_dimension], ctx=ctx) 86 | with g.autograd.record(): 87 | fake_img = generator(z) 88 | fake_out = discriminator(fake_img) 89 | d_loss_fake = bce(fake_out, fake_label) 90 | fake_scores = fake_out # closer to 0 means better 91 | 92 | # bp and optimize 93 | with g.autograd.record(): 94 | d_loss = d_loss_real + d_loss_fake 95 | d_loss.backward() 96 | d_optimizer.step(num_img) 97 | 98 | # ===============train generator 99 | # compute loss of fake_img 100 | with g.autograd.record(): 101 | fake_img = generator(z) 102 | output = discriminator(fake_img) 103 | g_loss = bce(output, real_label) 104 | 105 | # bp and optimize 106 | g_loss.backward() 107 | g_optimizer.step(num_img) 108 | 109 | if (i + 1) % 100 == 0: 110 | print('Epoch [{}/{}], d_loss: {:.6f}, g_loss: {:.6f} ' 111 | 'D real: {:.6f}, D fake: {:.6f}'.format( 112 | epoch, num_epoch, 113 | nd.mean(d_loss).asscalar(), 114 | nd.mean(g_loss).asscalar(), 115 | nd.mean(real_scores).asscalar(), 116 | nd.mean(fake_scores).asscalar())) 117 | if epoch == 0: 118 | real_images = to_img(torch.FloatTensor(real_img.asnumpy())) 119 | save_image(real_images, './img/real_images.png') 120 | 121 | fake_images = to_img(torch.FloatTensor(fake_img.asnumpy())) 122 | save_image(fake_images, './img/fake_images-{}.png'.format(epoch + 1)) 123 | 124 | discriminator.save_params('./dis.params') 125 | generator.save_params('./gen.params') 126 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mxnet-tutorial 2 | my simple tutorial for mxnet, a fast deep learning framework --------------------------------------------------------------------------------