├── environment.yml ├── LICENSE ├── .gitignore ├── README.md ├── GAN ├── boundary_seeking_gan │ ├── bgan_pytorch.py │ └── bgan_tensorflow.py ├── least_squares_gan │ ├── lsgan_pytorch.py │ └── lsgan_tensorflow.py ├── softmax_gan │ ├── softmax_gan_pytorch.py │ └── softmax_gan_tensorflow.py ├── ebgan │ ├── ebgan_pytorch.py │ └── ebgan_tensorflow.py ├── boundary_equilibrium_gan │ ├── began_pytorch.py │ └── began_tensorflow.py ├── wasserstein_gan │ ├── wgan_pytorch.py │ └── wgan_tensorflow.py ├── ali_bigan │ ├── ali_bigan_pytorch.py │ └── ali_bigan_tensorflow.py ├── f_gan │ ├── f_gan_pytorch.py │ └── f_gan_tensorflow.py ├── improved_wasserstein_gan │ └── wgan_gp_tensorflow.py ├── vanilla_gan │ ├── gan_tensorflow.py │ └── gan_pytorch.py ├── generative_adversarial_parallelization │ └── gap_pytorch.py ├── magan │ ├── magan_pytorch.py │ └── magan_tensorflow.py ├── mode_regularized_gan │ ├── mode_reg_gan_pytorch.py │ └── mode_reg_gan_tensorflow.py ├── conditional_gan │ ├── cgan_tensorflow.py │ └── cgan_pytorch.py ├── auxiliary_classifier_gan │ ├── ac_gan_pytorch.py │ └── ac_gan_tensorflow.py ├── infogan │ ├── infogan_tensorflow.py │ └── infogan_pytorch.py ├── disco_gan │ ├── discogan_pytorch.py │ └── discogan_tensorflow.py ├── dual_gan │ ├── dualgan_pytorch.py │ └── dualgan_tensorflow.py └── coupled_gan │ ├── cogan_pytorch.py │ └── cogan_tensorflow.py └── VAE ├── adversarial_autoencoder ├── aae_pytorch.py └── aae_tensorflow.py ├── adversarial_vb ├── avb_pytorch.py └── avb_tensorflow.py ├── vanilla_vae ├── vae_pytorch.py └── vae_tensorflow.py ├── denoising_vae ├── dvae_pytorch.py └── dvae_tensorflow.py └── conditional_vae ├── cvae_pytorch.py └── cvae_tensorflow.py /environment.yml: -------------------------------------------------------------------------------- 1 | name: generative-models 2 | dependencies: 3 | - python=3.5.1 4 | - numpy=1.11.0 5 | - scikit-learn=0.17.1 6 | - scipy=0.17.1 7 | - matplotlib=1.5.3 8 | - pip: 9 | - keras==1.1.1 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This is free and unencumbered software released into the public domain. 2 | 3 | Anyone is free to copy, modify, publish, use, compile, sell, or 4 | distribute this software, either in source code form or as a compiled 5 | binary, for any purpose, commercial or non-commercial, and by any 6 | means. 7 | 8 | In jurisdictions that recognize copyright laws, the author or authors 9 | of this software dedicate any and all copyright interest in the 10 | software to the public domain. We make this dedication for the benefit 11 | of the public at large and to the detriment of our heirs and 12 | successors. We intend this dedication to be an overt act of 13 | relinquishment in perpetuity of all present and future rights to this 14 | software under copyright law. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR 20 | OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 21 | ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | 24 | For more information, please refer to 25 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | local_settings.py 55 | 56 | # Flask stuff: 57 | instance/ 58 | .webassets-cache 59 | 60 | # Scrapy stuff: 61 | .scrapy 62 | 63 | # Sphinx documentation 64 | docs/_build/ 65 | 66 | # PyBuilder 67 | target/ 68 | 69 | # IPython Notebook 70 | .ipynb_checkpoints 71 | 72 | # pyenv 73 | .python-version 74 | 75 | # celery beat schedule file 76 | celerybeat-schedule 77 | 78 | # dotenv 79 | .env 80 | 81 | # virtualenv 82 | venv/ 83 | ENV/ 84 | 85 | # Spyder project settings 86 | .spyderproject 87 | 88 | # Rope project settings 89 | .ropeproject 90 | 91 | *.sublime* 92 | MNIST_data/ 93 | GAN/out/ 94 | VAE/out/ 95 | out/ 96 | 97 | # Unreleased 98 | GAN/unrolled_gan/ 99 | GAN/loss_sensitive_gan/ 100 | GAN/generative_adversarial_parallelization/gap_tensorflow.py -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Generative Models 2 | Collection of generative models, e.g. GAN, VAE in Pytorch and Tensorflow. 3 | 4 | ## Note: 5 | Generated samples will be stored in `GAN/{gan_model}/out` or `VAE/{vae_model}/out` directory during training. 6 | 7 | ## What's in it? 8 | 9 | #### Generative Adversarial Nets (GAN) 10 | 1. [Vanilla GAN](https://arxiv.org/abs/1406.2661) 11 | 2. [Conditional GAN](https://arxiv.org/abs/1411.1784) 12 | 3. [InfoGAN](https://arxiv.org/abs/1606.03657) 13 | 4. [Wasserstein GAN](https://arxiv.org/abs/1701.07875) 14 | 5. [Mode Regularized GAN](https://arxiv.org/abs/1612.02136) 15 | 6. [Coupled GAN](https://arxiv.org/abs/1606.07536) 16 | 7. [Auxiliary Classifier GAN](https://arxiv.org/abs/1610.09585) 17 | 8. [Least Squares GAN](https://arxiv.org/abs/1611.04076v2) 18 | 9. [Boundary Seeking GAN](https://arxiv.org/abs/1702.08431) 19 | 10. [Energy Based GAN](https://arxiv.org/abs/1609.03126) 20 | 11. [f-GAN](https://arxiv.org/abs/1606.00709) 21 | 12. [Generative Adversarial Parallelization](https://arxiv.org/abs/1612.04021) 22 | 13. [DiscoGAN](https://arxiv.org/abs/1703.05192) 23 | 14. [Adversarial Feature Learning](https://arxiv.org/abs/1605.09782) & [Adversarially Learned Inference](https://arxiv.org/abs/1606.00704) 24 | 15. [Boundary Equilibrium GAN](https://arxiv.org/abs/1703.10717) 25 | 16. [Improved Training for Wasserstein GAN](https://arxiv.org/abs/1704.00028) 26 | 17. [DualGAN](https://arxiv.org/abs/1704.02510) 27 | 18. [MAGAN: Margin Adaptation for GAN](https://arxiv.org/abs/1704.03817) 28 | 19. [Softmax GAN](https://arxiv.org/abs/1704.06191) 29 | #### Variational Autoencoder (VAE) 30 | 1. [Vanilla VAE](https://arxiv.org/abs/1312.6114) 31 | 2. [Conditional VAE](https://arxiv.org/abs/1406.5298) 32 | 3. [Denoising VAE](https://arxiv.org/abs/1511.06406) 33 | 4. [Adversarial Autoencoder](https://arxiv.org/abs/1511.05644) 34 | 5. [Adversarial Variational Bayes](https://arxiv.org/abs/1701.04722) 35 | 36 | ## Dependencies 37 | 38 | 1. Install miniconda 39 | 2. Do `conda env create` 40 | 3. Enter the env `source activate generative-models` 41 | 4. Install [Tensorflow](https://www.tensorflow.org/get_started/os_setup) 42 | 5. Install [Pytorch](https://github.com/pytorch/pytorch#installation) 43 | -------------------------------------------------------------------------------- /GAN/boundary_seeking_gan/bgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-3 22 | 23 | 24 | def log(x): 25 | return torch.log(x + 1e-8) 26 | 27 | 28 | G = torch.nn.Sequential( 29 | torch.nn.Linear(z_dim, h_dim), 30 | torch.nn.ReLU(), 31 | torch.nn.Linear(h_dim, X_dim), 32 | torch.nn.Sigmoid() 33 | ) 34 | 35 | 36 | D = torch.nn.Sequential( 37 | torch.nn.Linear(X_dim, h_dim), 38 | torch.nn.ReLU(), 39 | torch.nn.Linear(h_dim, 1), 40 | torch.nn.Sigmoid() 41 | ) 42 | 43 | 44 | def reset_grad(): 45 | G.zero_grad() 46 | D.zero_grad() 47 | 48 | 49 | G_solver = optim.Adam(G.parameters(), lr=lr) 50 | D_solver = optim.Adam(D.parameters(), lr=lr) 51 | 52 | 53 | for it in range(1000000): 54 | # Sample data 55 | z = Variable(torch.randn(mb_size, z_dim)) 56 | X, _ = mnist.train.next_batch(mb_size) 57 | X = Variable(torch.from_numpy(X)) 58 | 59 | # Dicriminator 60 | G_sample = G(z) 61 | D_real = D(X) 62 | D_fake = D(G_sample) 63 | 64 | D_loss = -torch.mean(log(D_real) + log(1 - D_fake)) 65 | 66 | D_loss.backward() 67 | D_solver.step() 68 | reset_grad() 69 | 70 | # Generator 71 | G_sample = G(z) 72 | D_fake = D(G_sample) 73 | 74 | G_loss = 0.5 * torch.mean((log(D_fake) - log(1 - D_fake))**2) 75 | 76 | G_loss.backward() 77 | G_solver.step() 78 | reset_grad() 79 | 80 | # Print and plot every now and then 81 | if it % 1000 == 0: 82 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 83 | .format(it, D_loss.data[0], G_loss.data[0])) 84 | 85 | samples = G(z).data.numpy()[:16] 86 | 87 | fig = plt.figure(figsize=(4, 4)) 88 | gs = gridspec.GridSpec(4, 4) 89 | gs.update(wspace=0.05, hspace=0.05) 90 | 91 | for i, sample in enumerate(samples): 92 | ax = plt.subplot(gs[i]) 93 | plt.axis('off') 94 | ax.set_xticklabels([]) 95 | ax.set_yticklabels([]) 96 | ax.set_aspect('equal') 97 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 98 | 99 | if not os.path.exists('out/'): 100 | os.makedirs('out/') 101 | 102 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 103 | cnt += 1 104 | plt.close(fig) 105 | -------------------------------------------------------------------------------- /GAN/least_squares_gan/lsgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | d_step = 3 22 | lr = 1e-3 23 | 24 | 25 | G = torch.nn.Sequential( 26 | torch.nn.Linear(z_dim, h_dim), 27 | torch.nn.ReLU(), 28 | torch.nn.Linear(h_dim, X_dim), 29 | torch.nn.Sigmoid() 30 | ) 31 | 32 | 33 | D = torch.nn.Sequential( 34 | torch.nn.Linear(X_dim, h_dim), 35 | torch.nn.ReLU(), 36 | torch.nn.Linear(h_dim, 1), 37 | ) 38 | 39 | 40 | def reset_grad(): 41 | G.zero_grad() 42 | D.zero_grad() 43 | 44 | 45 | G_solver = optim.Adam(G.parameters(), lr=lr) 46 | D_solver = optim.Adam(D.parameters(), lr=lr) 47 | 48 | 49 | for it in range(1000000): 50 | for _ in range(d_step): 51 | # Sample data 52 | z = Variable(torch.randn(mb_size, z_dim)) 53 | X, _ = mnist.train.next_batch(mb_size) 54 | X = Variable(torch.from_numpy(X)) 55 | 56 | # Dicriminator 57 | G_sample = G(z) 58 | D_real = D(X) 59 | D_fake = D(G_sample) 60 | 61 | D_loss = 0.5 * (torch.mean((D_real - 1)**2) + torch.mean(D_fake**2)) 62 | 63 | D_loss.backward() 64 | D_solver.step() 65 | reset_grad() 66 | 67 | # Generator 68 | z = Variable(torch.randn(mb_size, z_dim)) 69 | 70 | G_sample = G(z) 71 | D_fake = D(G_sample) 72 | 73 | G_loss = 0.5 * torch.mean((D_fake - 1)**2) 74 | 75 | G_loss.backward() 76 | G_solver.step() 77 | reset_grad() 78 | 79 | # Print and plot every now and then 80 | if it % 1000 == 0: 81 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 82 | .format(it, D_loss.data[0], G_loss.data[0])) 83 | 84 | samples = G(z).data.numpy()[:16] 85 | 86 | fig = plt.figure(figsize=(4, 4)) 87 | gs = gridspec.GridSpec(4, 4) 88 | gs.update(wspace=0.05, hspace=0.05) 89 | 90 | for i, sample in enumerate(samples): 91 | ax = plt.subplot(gs[i]) 92 | plt.axis('off') 93 | ax.set_xticklabels([]) 94 | ax.set_yticklabels([]) 95 | ax.set_aspect('equal') 96 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 97 | 98 | if not os.path.exists('out/'): 99 | os.makedirs('out/') 100 | 101 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 102 | cnt += 1 103 | plt.close(fig) 104 | -------------------------------------------------------------------------------- /GAN/softmax_gan/softmax_gan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 16 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-3 22 | 23 | 24 | def log(x): 25 | return torch.log(x + 1e-8) 26 | 27 | 28 | G = torch.nn.Sequential( 29 | torch.nn.Linear(z_dim, h_dim), 30 | torch.nn.ReLU(), 31 | torch.nn.Linear(h_dim, X_dim), 32 | torch.nn.Sigmoid() 33 | ) 34 | 35 | 36 | D = torch.nn.Sequential( 37 | torch.nn.Linear(X_dim, h_dim), 38 | torch.nn.ReLU(), 39 | torch.nn.Linear(h_dim, 1) 40 | ) 41 | 42 | 43 | def reset_grad(): 44 | G.zero_grad() 45 | D.zero_grad() 46 | 47 | 48 | G_solver = optim.Adam(G.parameters(), lr=lr) 49 | D_solver = optim.Adam(D.parameters(), lr=lr) 50 | 51 | 52 | D_target = 1./mb_size 53 | G_target = 1./(mb_size*2) 54 | 55 | for it in range(1000000): 56 | # Sample data 57 | z = Variable(torch.randn(mb_size, z_dim)) 58 | X, _ = mnist.train.next_batch(mb_size) 59 | X = Variable(torch.from_numpy(X)) 60 | 61 | G_sample = G(z) 62 | D_real = D(X) 63 | D_fake = D(G_sample) 64 | 65 | # Partition func. 66 | Z = torch.sum(torch.exp(-D_real)) + torch.sum(torch.exp(-D_fake)) 67 | 68 | # Dicriminator 69 | D_loss = torch.sum(D_target * D_real) + log(Z) 70 | 71 | D_loss.backward(retain_variables=True) 72 | D_solver.step() 73 | reset_grad() 74 | 75 | # Generator 76 | G_loss = torch.sum(G_target * D_real) + torch.sum(G_target * D_fake) + log(Z) 77 | 78 | G_loss.backward() 79 | G_solver.step() 80 | reset_grad() 81 | 82 | # Print and plot every now and then 83 | if it % 1000 == 0: 84 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 85 | .format(it, D_loss.data[0], G_loss.data[0])) 86 | 87 | samples = G(z).data.numpy()[:16] 88 | 89 | fig = plt.figure(figsize=(4, 4)) 90 | gs = gridspec.GridSpec(4, 4) 91 | gs.update(wspace=0.05, hspace=0.05) 92 | 93 | for i, sample in enumerate(samples): 94 | ax = plt.subplot(gs[i]) 95 | plt.axis('off') 96 | ax.set_xticklabels([]) 97 | ax.set_yticklabels([]) 98 | ax.set_aspect('equal') 99 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 100 | 101 | if not os.path.exists('out/'): 102 | os.makedirs('out/') 103 | 104 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 105 | cnt += 1 106 | plt.close(fig) 107 | -------------------------------------------------------------------------------- /GAN/ebgan/ebgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | d_step = 3 22 | lr = 1e-3 23 | m = 5 24 | 25 | 26 | G = torch.nn.Sequential( 27 | torch.nn.Linear(z_dim, h_dim), 28 | torch.nn.ReLU(), 29 | torch.nn.Linear(h_dim, X_dim), 30 | torch.nn.Sigmoid() 31 | ) 32 | 33 | # D is an autoencoder 34 | D_ = torch.nn.Sequential( 35 | torch.nn.Linear(X_dim, h_dim), 36 | torch.nn.ReLU(), 37 | torch.nn.Linear(h_dim, X_dim), 38 | ) 39 | 40 | 41 | # Energy is the MSE of autoencoder 42 | def D(X): 43 | X_recon = D_(X) 44 | return torch.mean(torch.sum((X - X_recon)**2, 1)) 45 | 46 | 47 | def reset_grad(): 48 | G.zero_grad() 49 | D_.zero_grad() 50 | 51 | 52 | G_solver = optim.Adam(G.parameters(), lr=lr) 53 | D_solver = optim.Adam(D_.parameters(), lr=lr) 54 | 55 | 56 | for it in range(1000000): 57 | # Sample data 58 | z = Variable(torch.randn(mb_size, z_dim)) 59 | X, _ = mnist.train.next_batch(mb_size) 60 | X = Variable(torch.from_numpy(X)) 61 | 62 | # Dicriminator 63 | G_sample = G(z) 64 | D_real = D(X) 65 | D_fake = D(G_sample) 66 | 67 | # EBGAN D loss. D_real and D_fake is energy, i.e. a number 68 | D_loss = D_real + nn.relu(m - D_fake) 69 | 70 | # Reuse D_fake for generator loss 71 | D_loss.backward() 72 | D_solver.step() 73 | reset_grad() 74 | 75 | # Generator 76 | G_sample = G(z) 77 | D_fake = D(G_sample) 78 | 79 | G_loss = D_fake 80 | 81 | G_loss.backward() 82 | G_solver.step() 83 | reset_grad() 84 | 85 | # Print and plot every now and then 86 | if it % 1000 == 0: 87 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 88 | .format(it, D_loss.data[0], G_loss.data[0])) 89 | 90 | samples = G(z).data.numpy()[:16] 91 | 92 | fig = plt.figure(figsize=(4, 4)) 93 | gs = gridspec.GridSpec(4, 4) 94 | gs.update(wspace=0.05, hspace=0.05) 95 | 96 | for i, sample in enumerate(samples): 97 | ax = plt.subplot(gs[i]) 98 | plt.axis('off') 99 | ax.set_xticklabels([]) 100 | ax.set_yticklabels([]) 101 | ax.set_aspect('equal') 102 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 103 | 104 | if not os.path.exists('out/'): 105 | os.makedirs('out/') 106 | 107 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 108 | cnt += 1 109 | plt.close(fig) 110 | -------------------------------------------------------------------------------- /GAN/boundary_equilibrium_gan/began_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | d_step = 3 22 | lr = 1e-3 23 | m = 5 24 | lam = 1e-3 25 | k = 0 26 | gamma = 0.5 27 | 28 | 29 | G = torch.nn.Sequential( 30 | torch.nn.Linear(z_dim, h_dim), 31 | torch.nn.ReLU(), 32 | torch.nn.Linear(h_dim, X_dim), 33 | torch.nn.Sigmoid() 34 | ) 35 | 36 | D_ = torch.nn.Sequential( 37 | torch.nn.Linear(X_dim, h_dim), 38 | torch.nn.ReLU(), 39 | torch.nn.Linear(h_dim, X_dim), 40 | ) 41 | 42 | 43 | # D is an autoencoder, approximating Gaussian 44 | def D(X): 45 | X_recon = D_(X) 46 | # Use Laplace MLE as in the paper 47 | return torch.mean(torch.sum(torch.abs(X - X_recon), 1)) 48 | 49 | 50 | def reset_grad(): 51 | G.zero_grad() 52 | D_.zero_grad() 53 | 54 | 55 | G_solver = optim.Adam(G.parameters(), lr=lr) 56 | D_solver = optim.Adam(D_.parameters(), lr=lr) 57 | 58 | 59 | for it in range(1000000): 60 | # Sample data 61 | X, _ = mnist.train.next_batch(mb_size) 62 | X = Variable(torch.from_numpy(X)) 63 | 64 | # Dicriminator 65 | z_D = Variable(torch.randn(mb_size, z_dim)) 66 | 67 | D_loss = D(X) - k * D(G(z_D)) 68 | 69 | D_loss.backward() 70 | D_solver.step() 71 | reset_grad() 72 | 73 | # Generator 74 | z_G = Variable(torch.randn(mb_size, z_dim)) 75 | 76 | G_loss = D(G(z_G)) 77 | 78 | G_loss.backward() 79 | G_solver.step() 80 | reset_grad() 81 | 82 | # Update k, the equlibrium 83 | k = k + lam * (gamma*D(X) - D(G(z_G))) 84 | k = k.data[0] # k is variable, so unvariable it so that no gradient prop. 85 | 86 | # Print and plot every now and then 87 | if it % 1000 == 0: 88 | measure = D(X) + torch.abs(gamma*D(X) - D(G(z_G))) 89 | 90 | print('Iter-{}; Convergence measure: {:.4}' 91 | .format(it, measure.data[0])) 92 | 93 | samples = G(z_G).data.numpy()[:16] 94 | 95 | fig = plt.figure(figsize=(4, 4)) 96 | gs = gridspec.GridSpec(4, 4) 97 | gs.update(wspace=0.05, hspace=0.05) 98 | 99 | for i, sample in enumerate(samples): 100 | ax = plt.subplot(gs[i]) 101 | plt.axis('off') 102 | ax.set_xticklabels([]) 103 | ax.set_yticklabels([]) 104 | ax.set_aspect('equal') 105 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 106 | 107 | if not os.path.exists('out/'): 108 | os.makedirs('out/') 109 | 110 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 111 | cnt += 1 112 | plt.close(fig) 113 | -------------------------------------------------------------------------------- /GAN/wasserstein_gan/wgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-4 22 | 23 | 24 | G = torch.nn.Sequential( 25 | torch.nn.Linear(z_dim, h_dim), 26 | torch.nn.ReLU(), 27 | torch.nn.Linear(h_dim, X_dim), 28 | torch.nn.Sigmoid() 29 | ) 30 | 31 | 32 | D = torch.nn.Sequential( 33 | torch.nn.Linear(X_dim, h_dim), 34 | torch.nn.ReLU(), 35 | torch.nn.Linear(h_dim, 1), 36 | ) 37 | 38 | 39 | def reset_grad(): 40 | G.zero_grad() 41 | D.zero_grad() 42 | 43 | 44 | G_solver = optim.RMSprop(G.parameters(), lr=lr) 45 | D_solver = optim.RMSprop(D.parameters(), lr=lr) 46 | 47 | 48 | for it in range(1000000): 49 | for _ in range(5): 50 | # Sample data 51 | z = Variable(torch.randn(mb_size, z_dim)) 52 | X, _ = mnist.train.next_batch(mb_size) 53 | X = Variable(torch.from_numpy(X)) 54 | 55 | # Dicriminator forward-loss-backward-update 56 | G_sample = G(z) 57 | D_real = D(X) 58 | D_fake = D(G_sample) 59 | 60 | D_loss = -(torch.mean(D_real) - torch.mean(D_fake)) 61 | 62 | D_loss.backward() 63 | D_solver.step() 64 | 65 | # Weight clipping 66 | for p in D.parameters(): 67 | p.data.clamp_(-0.01, 0.01) 68 | 69 | # Housekeeping - reset gradient 70 | reset_grad() 71 | 72 | # Generator forward-loss-backward-update 73 | X, _ = mnist.train.next_batch(mb_size) 74 | X = Variable(torch.from_numpy(X)) 75 | z = Variable(torch.randn(mb_size, z_dim)) 76 | 77 | G_sample = G(z) 78 | D_fake = D(G_sample) 79 | 80 | G_loss = -torch.mean(D_fake) 81 | 82 | G_loss.backward() 83 | G_solver.step() 84 | 85 | # Housekeeping - reset gradient 86 | reset_grad() 87 | 88 | # Print and plot every now and then 89 | if it % 1000 == 0: 90 | print('Iter-{}; D_loss: {}; G_loss: {}' 91 | .format(it, D_loss.data.numpy(), G_loss.data.numpy())) 92 | 93 | samples = G(z).data.numpy()[:16] 94 | 95 | fig = plt.figure(figsize=(4, 4)) 96 | gs = gridspec.GridSpec(4, 4) 97 | gs.update(wspace=0.05, hspace=0.05) 98 | 99 | for i, sample in enumerate(samples): 100 | ax = plt.subplot(gs[i]) 101 | plt.axis('off') 102 | ax.set_xticklabels([]) 103 | ax.set_yticklabels([]) 104 | ax.set_aspect('equal') 105 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 106 | 107 | if not os.path.exists('out/'): 108 | os.makedirs('out/') 109 | 110 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 111 | cnt += 1 112 | plt.close(fig) 113 | -------------------------------------------------------------------------------- /GAN/ali_bigan/ali_bigan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | from itertools import * 13 | 14 | 15 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 16 | mb_size = 32 17 | z_dim = 10 18 | X_dim = mnist.train.images.shape[1] 19 | y_dim = mnist.train.labels.shape[1] 20 | h_dim = 128 21 | cnt = 0 22 | lr = 1e-3 23 | 24 | 25 | def log(x): 26 | return torch.log(x + 1e-8) 27 | 28 | 29 | # Inference net (Encoder) Q(z|X) 30 | Q = torch.nn.Sequential( 31 | torch.nn.Linear(X_dim, h_dim), 32 | torch.nn.ReLU(), 33 | torch.nn.Linear(h_dim, z_dim) 34 | ) 35 | 36 | # Generator net (Decoder) P(X|z) 37 | P = torch.nn.Sequential( 38 | torch.nn.Linear(z_dim, h_dim), 39 | torch.nn.ReLU(), 40 | torch.nn.Linear(h_dim, X_dim), 41 | torch.nn.Sigmoid() 42 | ) 43 | 44 | D_ = torch.nn.Sequential( 45 | torch.nn.Linear(X_dim + z_dim, h_dim), 46 | torch.nn.ReLU(), 47 | torch.nn.Linear(h_dim, 1), 48 | torch.nn.Sigmoid() 49 | ) 50 | 51 | 52 | def D(X, z): 53 | return D_(torch.cat([X, z], 1)) 54 | 55 | 56 | def reset_grad(): 57 | Q.zero_grad() 58 | P.zero_grad() 59 | D_.zero_grad() 60 | 61 | 62 | G_solver = optim.Adam(chain(Q.parameters(), P.parameters()), lr=lr) 63 | D_solver = optim.Adam(D_.parameters(), lr=lr) 64 | 65 | 66 | for it in range(1000000): 67 | # Sample data 68 | z = Variable(torch.randn(mb_size, z_dim)) 69 | X, _ = mnist.train.next_batch(mb_size) 70 | X = Variable(torch.from_numpy(X)) 71 | 72 | # Discriminator 73 | z_hat = Q(X) 74 | X_hat = P(z) 75 | 76 | D_enc = D(X, z_hat) 77 | D_gen = D(X_hat, z) 78 | 79 | D_loss = -torch.mean(log(D_enc) + log(1 - D_gen)) 80 | 81 | D_loss.backward() 82 | D_solver.step() 83 | G_solver.step() 84 | reset_grad() 85 | 86 | # Autoencoder Q, P 87 | z_hat = Q(X) 88 | X_hat = P(z) 89 | 90 | D_enc = D(X, z_hat) 91 | D_gen = D(X_hat, z) 92 | 93 | G_loss = -torch.mean(log(D_gen) + log(1 - D_enc)) 94 | 95 | G_loss.backward() 96 | G_solver.step() 97 | reset_grad() 98 | 99 | # Print and plot every now and then 100 | if it % 1000 == 0: 101 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 102 | .format(it, D_loss.data[0], G_loss.data[0])) 103 | 104 | samples = P(z).data.numpy()[:16] 105 | 106 | fig = plt.figure(figsize=(4, 4)) 107 | gs = gridspec.GridSpec(4, 4) 108 | gs.update(wspace=0.05, hspace=0.05) 109 | 110 | for i, sample in enumerate(samples): 111 | ax = plt.subplot(gs[i]) 112 | plt.axis('off') 113 | ax.set_xticklabels([]) 114 | ax.set_yticklabels([]) 115 | ax.set_aspect('equal') 116 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 117 | 118 | if not os.path.exists('out/'): 119 | os.makedirs('out/') 120 | 121 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 122 | cnt += 1 123 | plt.close(fig) 124 | -------------------------------------------------------------------------------- /GAN/ebgan/ebgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | m = 5 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 42 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 43 | 44 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 45 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 46 | D_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 47 | D_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 48 | 49 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 50 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 51 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | theta_G = [G_W1, G_W2, G_b1, G_b2] 55 | theta_D = [D_W1, D_W2, D_b1, D_b2] 56 | 57 | 58 | def sample_z(m, n): 59 | return np.random.uniform(-1., 1., size=[m, n]) 60 | 61 | 62 | def generator(z): 63 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 64 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 65 | G_prob = tf.nn.sigmoid(G_log_prob) 66 | return G_prob 67 | 68 | 69 | def discriminator(X): 70 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 71 | X_recon = tf.matmul(D_h1, D_W2) + D_b2 72 | mse = tf.reduce_mean(tf.reduce_sum((X - X_recon)**2, 1)) 73 | return mse 74 | 75 | 76 | G_sample = generator(z) 77 | 78 | D_real = discriminator(X) 79 | D_fake = discriminator(G_sample) 80 | 81 | D_loss = D_real + tf.maximum(0., m - D_fake) 82 | G_loss = D_fake 83 | 84 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 85 | .minimize(D_loss, var_list=theta_D)) 86 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 87 | .minimize(G_loss, var_list=theta_G)) 88 | 89 | sess = tf.Session() 90 | sess.run(tf.global_variables_initializer()) 91 | 92 | if not os.path.exists('out/'): 93 | os.makedirs('out/') 94 | 95 | i = 0 96 | 97 | for it in range(1000000): 98 | X_mb, _ = mnist.train.next_batch(mb_size) 99 | z_mb = sample_z(mb_size, z_dim) 100 | 101 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb}) 102 | 103 | _, G_loss_curr = sess.run( 104 | [G_solver, G_loss], feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 105 | ) 106 | 107 | if it % 1000 == 0: 108 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 109 | .format(it, D_loss_curr, G_loss_curr)) 110 | 111 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 112 | 113 | fig = plot(samples) 114 | plt.savefig('out/{}.png' 115 | .format(str(i).zfill(3)), bbox_inches='tight') 116 | i += 1 117 | plt.close(fig) 118 | -------------------------------------------------------------------------------- /GAN/boundary_seeking_gan/bgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | d_steps = 3 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | def log(x): 42 | return tf.log(x + 1e-8) 43 | 44 | 45 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 49 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 51 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 52 | 53 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 54 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 55 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 56 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 57 | 58 | theta_G = [G_W1, G_W2, G_b1, G_b2] 59 | theta_D = [D_W1, D_W2, D_b1, D_b2] 60 | 61 | 62 | def sample_z(m, n): 63 | return np.random.uniform(-1., 1., size=[m, n]) 64 | 65 | 66 | def generator(z): 67 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 68 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 69 | G_prob = tf.nn.sigmoid(G_log_prob) 70 | return G_prob 71 | 72 | 73 | def discriminator(x): 74 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 75 | out = tf.nn.sigmoid(tf.matmul(D_h1, D_W2) + D_b2) 76 | return out 77 | 78 | 79 | G_sample = generator(z) 80 | 81 | D_real = discriminator(X) 82 | D_fake = discriminator(G_sample) 83 | 84 | D_loss = -tf.reduce_mean(log(D_real) + log(1 - D_fake)) 85 | G_loss = 0.5 * tf.reduce_mean((log(D_fake) - log(1 - D_fake))**2) 86 | 87 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 88 | .minimize(D_loss, var_list=theta_D)) 89 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 90 | .minimize(G_loss, var_list=theta_G)) 91 | 92 | sess = tf.Session() 93 | sess.run(tf.global_variables_initializer()) 94 | 95 | if not os.path.exists('out/'): 96 | os.makedirs('out/') 97 | 98 | i = 0 99 | 100 | for it in range(1000000): 101 | X_mb, _ = mnist.train.next_batch(mb_size) 102 | z_mb = sample_z(mb_size, z_dim) 103 | 104 | _, D_loss_curr = sess.run( 105 | [D_solver, D_loss], 106 | feed_dict={X: X_mb, z: z_mb} 107 | ) 108 | 109 | _, G_loss_curr = sess.run( 110 | [G_solver, G_loss], 111 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 112 | ) 113 | 114 | if it % 1000 == 0: 115 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 116 | .format(it, D_loss_curr, G_loss_curr)) 117 | 118 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 119 | 120 | fig = plot(samples) 121 | plt.savefig('out/{}.png' 122 | .format(str(i).zfill(3)), bbox_inches='tight') 123 | i += 1 124 | plt.close(fig) 125 | -------------------------------------------------------------------------------- /VAE/adversarial_autoencoder/aae_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 5 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-3 22 | 23 | 24 | # Encoder 25 | Q = torch.nn.Sequential( 26 | torch.nn.Linear(X_dim, h_dim), 27 | torch.nn.ReLU(), 28 | torch.nn.Linear(h_dim, z_dim) 29 | ) 30 | 31 | # Decoder 32 | P = torch.nn.Sequential( 33 | torch.nn.Linear(z_dim, h_dim), 34 | torch.nn.ReLU(), 35 | torch.nn.Linear(h_dim, X_dim), 36 | torch.nn.Sigmoid() 37 | ) 38 | 39 | # Discriminator 40 | D = torch.nn.Sequential( 41 | torch.nn.Linear(z_dim, h_dim), 42 | torch.nn.ReLU(), 43 | torch.nn.Linear(h_dim, 1), 44 | torch.nn.Sigmoid() 45 | ) 46 | 47 | 48 | def reset_grad(): 49 | Q.zero_grad() 50 | P.zero_grad() 51 | D.zero_grad() 52 | 53 | 54 | def sample_X(size, include_y=False): 55 | X, y = mnist.train.next_batch(size) 56 | X = Variable(torch.from_numpy(X)) 57 | 58 | if include_y: 59 | y = np.argmax(y, axis=1).astype(np.int) 60 | y = Variable(torch.from_numpy(y)) 61 | return X, y 62 | 63 | return X 64 | 65 | 66 | Q_solver = optim.Adam(Q.parameters(), lr=lr) 67 | P_solver = optim.Adam(P.parameters(), lr=lr) 68 | D_solver = optim.Adam(D.parameters(), lr=lr) 69 | 70 | 71 | for it in range(1000000): 72 | X = sample_X(mb_size) 73 | 74 | """ Reconstruction phase """ 75 | z_sample = Q(X) 76 | X_sample = P(z_sample) 77 | 78 | recon_loss = nn.binary_cross_entropy(X_sample, X) 79 | 80 | recon_loss.backward() 81 | P_solver.step() 82 | Q_solver.step() 83 | reset_grad() 84 | 85 | """ Regularization phase """ 86 | # Discriminator 87 | z_real = Variable(torch.randn(mb_size, z_dim)) 88 | z_fake = Q(X) 89 | 90 | D_real = D(z_real) 91 | D_fake = D(z_fake) 92 | 93 | D_loss = -torch.mean(torch.log(D_real) + torch.log(1 - D_fake)) 94 | 95 | D_loss.backward() 96 | D_solver.step() 97 | reset_grad() 98 | 99 | # Generator 100 | z_fake = Q(X) 101 | D_fake = D(z_fake) 102 | 103 | G_loss = -torch.mean(torch.log(D_fake)) 104 | 105 | G_loss.backward() 106 | Q_solver.step() 107 | reset_grad() 108 | 109 | # Print and plot every now and then 110 | if it % 1000 == 0: 111 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}; recon_loss: {:.4}' 112 | .format(it, D_loss.data[0], G_loss.data[0], recon_loss.data[0])) 113 | 114 | samples = P(z_real).data.numpy()[:16] 115 | 116 | fig = plt.figure(figsize=(4, 4)) 117 | gs = gridspec.GridSpec(4, 4) 118 | gs.update(wspace=0.05, hspace=0.05) 119 | 120 | for i, sample in enumerate(samples): 121 | ax = plt.subplot(gs[i]) 122 | plt.axis('off') 123 | ax.set_xticklabels([]) 124 | ax.set_yticklabels([]) 125 | ax.set_aspect('equal') 126 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 127 | 128 | if not os.path.exists('out/'): 129 | os.makedirs('out/') 130 | 131 | plt.savefig('out/{}.png' 132 | .format(str(cnt).zfill(3)), bbox_inches='tight') 133 | cnt += 1 134 | plt.close(fig) 135 | -------------------------------------------------------------------------------- /GAN/softmax_gan/softmax_gan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | d_steps = 3 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | def log(x): 42 | return tf.log(x + 1e-8) 43 | 44 | 45 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 49 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 51 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 52 | 53 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 54 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 55 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 56 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 57 | 58 | theta_G = [G_W1, G_W2, G_b1, G_b2] 59 | theta_D = [D_W1, D_W2, D_b1, D_b2] 60 | 61 | 62 | def sample_z(m, n): 63 | return np.random.uniform(-1., 1., size=[m, n]) 64 | 65 | 66 | def G(z): 67 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 68 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 69 | G_prob = tf.nn.sigmoid(G_log_prob) 70 | return G_prob 71 | 72 | 73 | def D(X): 74 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 75 | out = tf.matmul(D_h1, D_W2) + D_b2 76 | return out 77 | 78 | 79 | G_sample = G(z) 80 | 81 | D_real = D(X) 82 | D_fake = D(G_sample) 83 | 84 | D_target = 1./mb_size 85 | G_target = 1./(mb_size*2) 86 | 87 | Z = tf.reduce_sum(tf.exp(-D_real)) + tf.reduce_sum(tf.exp(-D_fake)) 88 | 89 | D_loss = tf.reduce_sum(D_target * D_real) + log(Z) 90 | G_loss = tf.reduce_sum(G_target * D_real) + tf.reduce_sum(G_target * D_fake) + log(Z) 91 | 92 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 93 | .minimize(D_loss, var_list=theta_D)) 94 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 95 | .minimize(G_loss, var_list=theta_G)) 96 | 97 | sess = tf.Session() 98 | sess.run(tf.global_variables_initializer()) 99 | 100 | if not os.path.exists('out/'): 101 | os.makedirs('out/') 102 | 103 | i = 0 104 | 105 | for it in range(1000000): 106 | X_mb, _ = mnist.train.next_batch(mb_size) 107 | z_mb = sample_z(mb_size, z_dim) 108 | 109 | _, D_loss_curr = sess.run( 110 | [D_solver, D_loss], feed_dict={X: X_mb, z: z_mb} 111 | ) 112 | 113 | _, G_loss_curr = sess.run( 114 | [G_solver, G_loss], feed_dict={X: X_mb, z: z_mb} 115 | ) 116 | 117 | if it % 1000 == 0: 118 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 119 | .format(it, D_loss_curr, G_loss_curr)) 120 | 121 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 122 | 123 | fig = plot(samples) 124 | plt.savefig('out/{}.png' 125 | .format(str(i).zfill(3)), bbox_inches='tight') 126 | i += 1 127 | plt.close(fig) 128 | -------------------------------------------------------------------------------- /GAN/wasserstein_gan/wgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 10 12 | h_dim = 128 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | 16 | 17 | def plot(samples): 18 | fig = plt.figure(figsize=(4, 4)) 19 | gs = gridspec.GridSpec(4, 4) 20 | gs.update(wspace=0.05, hspace=0.05) 21 | 22 | for i, sample in enumerate(samples): 23 | ax = plt.subplot(gs[i]) 24 | plt.axis('off') 25 | ax.set_xticklabels([]) 26 | ax.set_yticklabels([]) 27 | ax.set_aspect('equal') 28 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 29 | 30 | return fig 31 | 32 | 33 | def xavier_init(size): 34 | in_dim = size[0] 35 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 36 | return tf.random_normal(shape=size, stddev=xavier_stddev) 37 | 38 | 39 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 40 | 41 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 42 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 43 | 44 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 45 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 46 | 47 | theta_D = [D_W1, D_W2, D_b1, D_b2] 48 | 49 | 50 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 51 | 52 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 53 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 54 | 55 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 56 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 57 | 58 | theta_G = [G_W1, G_W2, G_b1, G_b2] 59 | 60 | 61 | def sample_z(m, n): 62 | return np.random.uniform(-1., 1., size=[m, n]) 63 | 64 | 65 | def generator(z): 66 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 67 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 68 | G_prob = tf.nn.sigmoid(G_log_prob) 69 | return G_prob 70 | 71 | 72 | def discriminator(x): 73 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 74 | out = tf.matmul(D_h1, D_W2) + D_b2 75 | return out 76 | 77 | 78 | G_sample = generator(z) 79 | D_real = discriminator(X) 80 | D_fake = discriminator(G_sample) 81 | 82 | D_loss = tf.reduce_mean(D_real) - tf.reduce_mean(D_fake) 83 | G_loss = -tf.reduce_mean(D_fake) 84 | 85 | D_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4) 86 | .minimize(-D_loss, var_list=theta_D)) 87 | G_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4) 88 | .minimize(G_loss, var_list=theta_G)) 89 | 90 | clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in theta_D] 91 | 92 | sess = tf.Session() 93 | sess.run(tf.global_variables_initializer()) 94 | 95 | if not os.path.exists('out/'): 96 | os.makedirs('out/') 97 | 98 | i = 0 99 | 100 | for it in range(1000000): 101 | for _ in range(5): 102 | X_mb, _ = mnist.train.next_batch(mb_size) 103 | 104 | _, D_loss_curr, _ = sess.run( 105 | [D_solver, D_loss, clip_D], 106 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 107 | ) 108 | 109 | _, G_loss_curr = sess.run( 110 | [G_solver, G_loss], 111 | feed_dict={z: sample_z(mb_size, z_dim)} 112 | ) 113 | 114 | if it % 100 == 0: 115 | print('Iter: {}; D loss: {:.4}; G_loss: {:.4}' 116 | .format(it, D_loss_curr, G_loss_curr)) 117 | 118 | if it % 1000 == 0: 119 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 120 | 121 | fig = plot(samples) 122 | plt.savefig('out/{}.png' 123 | .format(str(i).zfill(3)), bbox_inches='tight') 124 | i += 1 125 | plt.close(fig) 126 | -------------------------------------------------------------------------------- /GAN/boundary_equilibrium_gan/began_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | m = 5 15 | lam = 1e-3 16 | gamma = 0.5 17 | k_curr = 0 18 | 19 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 20 | 21 | 22 | def plot(samples): 23 | fig = plt.figure(figsize=(4, 4)) 24 | gs = gridspec.GridSpec(4, 4) 25 | gs.update(wspace=0.05, hspace=0.05) 26 | 27 | for i, sample in enumerate(samples): 28 | ax = plt.subplot(gs[i]) 29 | plt.axis('off') 30 | ax.set_xticklabels([]) 31 | ax.set_yticklabels([]) 32 | ax.set_aspect('equal') 33 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 34 | 35 | return fig 36 | 37 | 38 | def xavier_init(size): 39 | in_dim = size[0] 40 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 41 | return tf.random_normal(shape=size, stddev=xavier_stddev) 42 | 43 | 44 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 45 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 46 | k = tf.placeholder(tf.float32) 47 | 48 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 49 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | D_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 51 | D_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 52 | 53 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 54 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 55 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 56 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 57 | 58 | theta_G = [G_W1, G_W2, G_b1, G_b2] 59 | theta_D = [D_W1, D_W2, D_b1, D_b2] 60 | 61 | 62 | def sample_z(m, n): 63 | return np.random.uniform(-1., 1., size=[m, n]) 64 | 65 | 66 | def G(z): 67 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 68 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 69 | G_prob = tf.nn.sigmoid(G_log_prob) 70 | return G_prob 71 | 72 | 73 | def D(X): 74 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 75 | X_recon = tf.matmul(D_h1, D_W2) + D_b2 76 | return tf.reduce_mean(tf.reduce_sum((X - X_recon)**2, 1)) 77 | 78 | 79 | G_sample = G(z) 80 | 81 | D_real = D(X) 82 | D_fake = D(G_sample) 83 | 84 | D_loss = D_real - k*D_fake 85 | G_loss = D_fake 86 | 87 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 88 | .minimize(D_loss, var_list=theta_D)) 89 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 90 | .minimize(G_loss, var_list=theta_G)) 91 | 92 | sess = tf.Session() 93 | sess.run(tf.global_variables_initializer()) 94 | 95 | if not os.path.exists('out/'): 96 | os.makedirs('out/') 97 | 98 | i = 0 99 | 100 | for it in range(1000000): 101 | X_mb, _ = mnist.train.next_batch(mb_size) 102 | 103 | _, D_real_curr = sess.run( 104 | [D_solver, D_real], 105 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim), k: k_curr} 106 | ) 107 | 108 | _, D_fake_curr = sess.run( 109 | [G_solver, D_fake], 110 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 111 | ) 112 | 113 | k_curr = k_curr + lam * (gamma*D_real_curr - D_fake_curr) 114 | 115 | if it % 1000 == 0: 116 | measure = D_real_curr + np.abs(gamma*D_real_curr - D_fake_curr) 117 | 118 | print('Iter-{}; Convergence measure: {:.4}' 119 | .format(it, measure)) 120 | 121 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 122 | 123 | fig = plot(samples) 124 | plt.savefig('out/{}.png' 125 | .format(str(i).zfill(3)), bbox_inches='tight') 126 | i += 1 127 | plt.close(fig) 128 | -------------------------------------------------------------------------------- /VAE/adversarial_vb/avb_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | eps_dim = 4 18 | X_dim = mnist.train.images.shape[1] 19 | y_dim = mnist.train.labels.shape[1] 20 | h_dim = 128 21 | cnt = 0 22 | lr = 1e-3 23 | 24 | 25 | def log(x): 26 | return torch.log(x + 1e-8) 27 | 28 | 29 | # Encoder: q(z|x,eps) 30 | Q = torch.nn.Sequential( 31 | torch.nn.Linear(X_dim + eps_dim, h_dim), 32 | torch.nn.ReLU(), 33 | torch.nn.Linear(h_dim, z_dim) 34 | ) 35 | 36 | # Decoder: p(x|z) 37 | P = torch.nn.Sequential( 38 | torch.nn.Linear(z_dim, h_dim), 39 | torch.nn.ReLU(), 40 | torch.nn.Linear(h_dim, X_dim), 41 | torch.nn.Sigmoid() 42 | ) 43 | 44 | # Discriminator: T(X, z) 45 | T = torch.nn.Sequential( 46 | torch.nn.Linear(X_dim + z_dim, h_dim), 47 | torch.nn.ReLU(), 48 | torch.nn.Linear(h_dim, 1) 49 | ) 50 | 51 | 52 | def reset_grad(): 53 | Q.zero_grad() 54 | P.zero_grad() 55 | T.zero_grad() 56 | 57 | 58 | def sample_X(size, include_y=False): 59 | X, y = mnist.train.next_batch(size) 60 | X = Variable(torch.from_numpy(X)) 61 | 62 | if include_y: 63 | y = np.argmax(y, axis=1).astype(np.int) 64 | y = Variable(torch.from_numpy(y)) 65 | return X, y 66 | 67 | return X 68 | 69 | 70 | Q_solver = optim.Adam(Q.parameters(), lr=lr) 71 | P_solver = optim.Adam(P.parameters(), lr=lr) 72 | T_solver = optim.Adam(T.parameters(), lr=lr) 73 | 74 | 75 | for it in range(1000000): 76 | X = sample_X(mb_size) 77 | eps = Variable(torch.randn(mb_size, eps_dim)) 78 | z = Variable(torch.randn(mb_size, z_dim)) 79 | 80 | # Optimize VAE 81 | z_sample = Q(torch.cat([X, eps], 1)) 82 | X_sample = P(z_sample) 83 | T_sample = T(torch.cat([X, z_sample], 1)) 84 | 85 | disc = torch.mean(-T_sample) 86 | loglike = -nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size 87 | 88 | elbo = -(disc + loglike) 89 | 90 | elbo.backward() 91 | Q_solver.step() 92 | P_solver.step() 93 | reset_grad() 94 | 95 | # Discriminator T(X, z) 96 | z_sample = Q(torch.cat([X, eps], 1)) 97 | T_q = nn.sigmoid(T(torch.cat([X, z_sample], 1))) 98 | T_prior = nn.sigmoid(T(torch.cat([X, z], 1))) 99 | 100 | T_loss = -torch.mean(log(T_q) + log(1. - T_prior)) 101 | 102 | T_loss.backward() 103 | T_solver.step() 104 | reset_grad() 105 | 106 | # Print and plot every now and then 107 | if it % 1000 == 0: 108 | print('Iter-{}; ELBO: {:.4}; T_loss: {:.4}' 109 | .format(it, -elbo.data[0], -T_loss.data[0])) 110 | 111 | samples = P(z).data.numpy()[:16] 112 | 113 | fig = plt.figure(figsize=(4, 4)) 114 | gs = gridspec.GridSpec(4, 4) 115 | gs.update(wspace=0.05, hspace=0.05) 116 | 117 | for i, sample in enumerate(samples): 118 | ax = plt.subplot(gs[i]) 119 | plt.axis('off') 120 | ax.set_xticklabels([]) 121 | ax.set_yticklabels([]) 122 | ax.set_aspect('equal') 123 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 124 | 125 | if not os.path.exists('out/'): 126 | os.makedirs('out/') 127 | 128 | plt.savefig('out/{}.png' 129 | .format(str(cnt).zfill(3)), bbox_inches='tight') 130 | cnt += 1 131 | plt.close(fig) 132 | -------------------------------------------------------------------------------- /GAN/least_squares_gan/lsgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | d_steps = 3 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 42 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 43 | 44 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 45 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 46 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 47 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 48 | 49 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 50 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 51 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | theta_G = [G_W1, G_W2, G_b1, G_b2] 55 | theta_D = [D_W1, D_W2, D_b1, D_b2] 56 | 57 | 58 | def sample_z(m, n): 59 | return np.random.uniform(-1., 1., size=[m, n]) 60 | 61 | 62 | def generator(z): 63 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 64 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 65 | G_prob = tf.nn.sigmoid(G_log_prob) 66 | return G_prob 67 | 68 | 69 | def discriminator(x): 70 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 71 | out = tf.matmul(D_h1, D_W2) + D_b2 72 | return out 73 | 74 | 75 | G_sample = generator(z) 76 | 77 | D_real = discriminator(X) 78 | D_fake = discriminator(G_sample) 79 | 80 | D_loss = 0.5 * (tf.reduce_mean((D_real - 1)**2) + tf.reduce_mean(D_fake**2)) 81 | G_loss = 0.5 * tf.reduce_mean((D_fake - 1)**2) 82 | 83 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 84 | .minimize(D_loss, var_list=theta_D)) 85 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 86 | .minimize(G_loss, var_list=theta_G)) 87 | 88 | sess = tf.Session() 89 | sess.run(tf.global_variables_initializer()) 90 | 91 | if not os.path.exists('out/'): 92 | os.makedirs('out/') 93 | 94 | i = 0 95 | 96 | for it in range(1000000): 97 | for _ in range(d_steps): 98 | X_mb, _ = mnist.train.next_batch(mb_size) 99 | z_mb = sample_z(mb_size, z_dim) 100 | 101 | _, D_loss_curr = sess.run( 102 | [D_solver, D_loss], 103 | feed_dict={X: X_mb, z: z_mb} 104 | ) 105 | 106 | X_mb, _ = mnist.train.next_batch(mb_size) 107 | z_mb = sample_z(mb_size, z_dim) 108 | 109 | _, G_loss_curr = sess.run( 110 | [G_solver, G_loss], 111 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 112 | ) 113 | 114 | if it % 1000 == 0: 115 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 116 | .format(it, D_loss_curr, G_loss_curr)) 117 | 118 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 119 | 120 | fig = plot(samples) 121 | plt.savefig('out/{}.png' 122 | .format(str(i).zfill(3)), bbox_inches='tight') 123 | i += 1 124 | plt.close(fig) 125 | -------------------------------------------------------------------------------- /VAE/vanilla_vae/vae_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 64 15 | Z_dim = 100 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | c = 0 20 | lr = 1e-3 21 | 22 | 23 | def xavier_init(size): 24 | in_dim = size[0] 25 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 26 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 27 | 28 | 29 | # =============================== Q(z|X) ====================================== 30 | 31 | Wxh = xavier_init(size=[X_dim, h_dim]) 32 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whz_mu = xavier_init(size=[h_dim, Z_dim]) 35 | bhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True) 36 | 37 | Whz_var = xavier_init(size=[h_dim, Z_dim]) 38 | bhz_var = Variable(torch.zeros(Z_dim), requires_grad=True) 39 | 40 | 41 | def Q(X): 42 | h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1)) 43 | z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1) 44 | z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1) 45 | return z_mu, z_var 46 | 47 | 48 | def sample_z(mu, log_var): 49 | eps = Variable(torch.randn(mb_size, Z_dim)) 50 | return mu + torch.exp(log_var / 2) * eps 51 | 52 | 53 | # =============================== P(X|z) ====================================== 54 | 55 | Wzh = xavier_init(size=[Z_dim, h_dim]) 56 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 57 | 58 | Whx = xavier_init(size=[h_dim, X_dim]) 59 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 60 | 61 | 62 | def P(z): 63 | h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1)) 64 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 65 | return X 66 | 67 | 68 | # =============================== TRAINING ==================================== 69 | 70 | params = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var, 71 | Wzh, bzh, Whx, bhx] 72 | 73 | solver = optim.Adam(params, lr=lr) 74 | 75 | for it in range(100000): 76 | X, _ = mnist.train.next_batch(mb_size) 77 | X = Variable(torch.from_numpy(X)) 78 | 79 | # Forward 80 | z_mu, z_var = Q(X) 81 | z = sample_z(z_mu, z_var) 82 | X_sample = P(z) 83 | 84 | # Loss 85 | recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size 86 | kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1)) 87 | loss = recon_loss + kl_loss 88 | 89 | # Backward 90 | loss.backward() 91 | 92 | # Update 93 | solver.step() 94 | 95 | # Housekeeping 96 | for p in params: 97 | p.grad.data.zero_() 98 | 99 | # Print and plot every now and then 100 | if it % 1000 == 0: 101 | print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0])) 102 | 103 | samples = P(z).data.numpy()[:16] 104 | 105 | fig = plt.figure(figsize=(4, 4)) 106 | gs = gridspec.GridSpec(4, 4) 107 | gs.update(wspace=0.05, hspace=0.05) 108 | 109 | for i, sample in enumerate(samples): 110 | ax = plt.subplot(gs[i]) 111 | plt.axis('off') 112 | ax.set_xticklabels([]) 113 | ax.set_yticklabels([]) 114 | ax.set_aspect('equal') 115 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 116 | 117 | if not os.path.exists('out/'): 118 | os.makedirs('out/') 119 | 120 | plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight') 121 | c += 1 122 | plt.close(fig) 123 | -------------------------------------------------------------------------------- /VAE/denoising_vae/dvae_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 64 15 | Z_dim = 100 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | c = 0 20 | lr = 1e-3 21 | noise_factor = .25 22 | 23 | 24 | def xavier_init(size): 25 | in_dim = size[0] 26 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 27 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 28 | 29 | 30 | """ Q(z|X) """ 31 | Wxh = xavier_init(size=[X_dim, h_dim]) 32 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whz_mu = xavier_init(size=[h_dim, Z_dim]) 35 | bhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True) 36 | 37 | Whz_var = xavier_init(size=[h_dim, Z_dim]) 38 | bhz_var = Variable(torch.zeros(Z_dim), requires_grad=True) 39 | 40 | 41 | def Q(X): 42 | h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1)) 43 | z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1) 44 | z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1) 45 | return z_mu, z_var 46 | 47 | 48 | def sample_z(mu, log_var): 49 | eps = Variable(torch.randn(mb_size, Z_dim)) 50 | return mu + torch.exp(log_var / 2) * eps 51 | 52 | 53 | """ P(X|z) """ 54 | Wzh = xavier_init(size=[Z_dim, h_dim]) 55 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 56 | 57 | Whx = xavier_init(size=[h_dim, X_dim]) 58 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 59 | 60 | 61 | def P(z): 62 | h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1)) 63 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 64 | return X 65 | 66 | 67 | """ Training """ 68 | params = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var, 69 | Wzh, bzh, Whx, bhx] 70 | 71 | solver = optim.Adam(params, lr=lr) 72 | 73 | for it in range(100000): 74 | X, _ = mnist.train.next_batch(mb_size) 75 | X = Variable(torch.from_numpy(X)) 76 | 77 | # Add noise 78 | X_noise = X + noise_factor * Variable(torch.randn(X.size())) 79 | X_noise.data.clamp_(0., 1.) 80 | 81 | # Forward 82 | z_mu, z_var = Q(X_noise) 83 | z = sample_z(z_mu, z_var) 84 | X_sample = P(z) 85 | 86 | torch.nn.BCELoss 87 | recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size 88 | kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1)) 89 | loss = recon_loss + kl_loss 90 | 91 | # Backward 92 | loss.backward() 93 | 94 | # Update 95 | solver.step() 96 | 97 | # Housekeeping 98 | for p in params: 99 | p.grad.data.zero_() 100 | 101 | # Print and plot every now and then 102 | if it % 1000 == 0: 103 | print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0])) 104 | 105 | z = Variable(torch.randn(mb_size, Z_dim)) 106 | samples = P(z).data.numpy()[:16] 107 | 108 | fig = plt.figure(figsize=(4, 4)) 109 | gs = gridspec.GridSpec(4, 4) 110 | gs.update(wspace=0.05, hspace=0.05) 111 | 112 | for i, sample in enumerate(samples): 113 | ax = plt.subplot(gs[i]) 114 | plt.axis('off') 115 | ax.set_xticklabels([]) 116 | ax.set_yticklabels([]) 117 | ax.set_aspect('equal') 118 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 119 | 120 | if not os.path.exists('out/'): 121 | os.makedirs('out/') 122 | 123 | plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight') 124 | c += 1 125 | plt.close(fig) 126 | -------------------------------------------------------------------------------- /VAE/denoising_vae/dvae_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.gridspec as gridspec 5 | import os 6 | from torch.autograd import Variable 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | mb_size = 64 12 | z_dim = 100 13 | X_dim = mnist.train.images.shape[1] 14 | y_dim = mnist.train.labels.shape[1] 15 | h_dim = 128 16 | c = 0 17 | lr = 1e-3 18 | noise_factor = 0.25 19 | 20 | 21 | def plot(samples): 22 | fig = plt.figure(figsize=(4, 4)) 23 | gs = gridspec.GridSpec(4, 4) 24 | gs.update(wspace=0.05, hspace=0.05) 25 | 26 | for i, sample in enumerate(samples): 27 | ax = plt.subplot(gs[i]) 28 | plt.axis('off') 29 | ax.set_xticklabels([]) 30 | ax.set_yticklabels([]) 31 | ax.set_aspect('equal') 32 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 33 | 34 | return fig 35 | 36 | 37 | def xavier_init(size): 38 | in_dim = size[0] 39 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 40 | return tf.random_normal(shape=size, stddev=xavier_stddev) 41 | 42 | 43 | """ Q(X|z) """ 44 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 45 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 46 | 47 | Q_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 48 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 49 | 50 | Q_W2_mu = tf.Variable(xavier_init([h_dim, z_dim])) 51 | Q_b2_mu = tf.Variable(tf.zeros(shape=[z_dim])) 52 | 53 | Q_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim])) 54 | Q_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim])) 55 | 56 | 57 | def Q(X): 58 | h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1) 59 | z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu 60 | z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma 61 | return z_mu, z_logvar 62 | 63 | 64 | def sample_z(mu, log_var): 65 | eps = tf.random_normal(shape=tf.shape(mu)) 66 | return mu + tf.exp(log_var / 2) * eps 67 | 68 | 69 | """ P(X|z) """ 70 | P_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 71 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 72 | 73 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 74 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 75 | 76 | 77 | def P(z): 78 | h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1) 79 | logits = tf.matmul(h, P_W2) + P_b2 80 | prob = tf.nn.sigmoid(logits) 81 | return prob, logits 82 | 83 | 84 | """ Training """ 85 | # Add noise to X 86 | X_noise = X + noise_factor * tf.random_normal(tf.shape(X)) 87 | X_noise = tf.clip_by_value(X_noise, 0., 1.) 88 | 89 | z_mu, z_logvar = Q(X_noise) 90 | z_sample = sample_z(z_mu, z_logvar) 91 | _, logits = P(z_sample) 92 | 93 | # Sample from random z 94 | X_samples, _ = P(z) 95 | 96 | # E[log P(X|z)] 97 | recon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1) 98 | # D_KL(Q(z|X_noise) || P(z|X)); calculate in closed form as both dist. are Gaussian 99 | kl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1) 100 | # VAE loss 101 | vae_loss = tf.reduce_mean(recon_loss + kl_loss) 102 | 103 | solver = tf.train.AdamOptimizer().minimize(vae_loss) 104 | 105 | sess = tf.Session() 106 | sess.run(tf.global_variables_initializer()) 107 | 108 | if not os.path.exists('out/'): 109 | os.makedirs('out/') 110 | 111 | i = 0 112 | 113 | for it in range(1000000): 114 | X_mb, _ = mnist.train.next_batch(mb_size) 115 | 116 | _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb}) 117 | 118 | if it % 1000 == 0: 119 | print('Iter: {}; Loss: {:.4}'.format(it, loss)) 120 | 121 | samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)}) 122 | 123 | fig = plot(samples) 124 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 125 | i += 1 126 | plt.close(fig) 127 | -------------------------------------------------------------------------------- /GAN/f_gan/f_gan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-3 22 | 23 | 24 | def log(x): 25 | return torch.log(x + 1e-8) 26 | 27 | 28 | G = torch.nn.Sequential( 29 | torch.nn.Linear(z_dim, h_dim), 30 | torch.nn.ReLU(), 31 | torch.nn.Linear(h_dim, X_dim), 32 | torch.nn.Sigmoid() 33 | ) 34 | 35 | 36 | D = torch.nn.Sequential( 37 | torch.nn.Linear(X_dim, h_dim), 38 | torch.nn.ReLU(), 39 | torch.nn.Linear(h_dim, 1), 40 | ) 41 | 42 | 43 | def reset_grad(): 44 | G.zero_grad() 45 | D.zero_grad() 46 | 47 | 48 | G_solver = optim.Adam(G.parameters(), lr=lr) 49 | D_solver = optim.Adam(D.parameters(), lr=lr) 50 | 51 | 52 | for it in range(1000000): 53 | # Sample data 54 | z = Variable(torch.randn(mb_size, z_dim)) 55 | X, _ = mnist.train.next_batch(mb_size) 56 | X = Variable(torch.from_numpy(X)) 57 | 58 | # Dicriminator 59 | G_sample = G(z) 60 | D_real = D(X) 61 | D_fake = D(G_sample) 62 | 63 | # Uncomment D_loss and its respective G_loss of your choice 64 | # --------------------------------------------------------- 65 | 66 | """ Total Variation """ 67 | # D_loss = -(torch.mean(0.5 * torch.tanh(D_real)) - 68 | # torch.mean(0.5 * torch.tanh(D_fake))) 69 | """ Forward KL """ 70 | # D_loss = -(torch.mean(D_real) - torch.mean(torch.exp(D_fake - 1))) 71 | """ Reverse KL """ 72 | D_loss = -(torch.mean(-torch.exp(D_real)) - torch.mean(-1 - D_fake)) 73 | """ Pearson Chi-squared """ 74 | # D_loss = -(torch.mean(D_real) - torch.mean(0.25*D_fake**2 + D_fake)) 75 | """ Squared Hellinger """ 76 | # D_loss = -(torch.mean(1 - torch.exp(D_real)) - 77 | # torch.mean((1 - torch.exp(D_fake)) / (torch.exp(D_fake)))) 78 | 79 | D_loss.backward() 80 | D_solver.step() 81 | reset_grad() 82 | 83 | # Generator 84 | G_sample = G(z) 85 | D_fake = D(G_sample) 86 | 87 | """ Total Variation """ 88 | # G_loss = -torch.mean(0.5 * torch.tanh(D_fake)) 89 | """ Forward KL """ 90 | # G_loss = -torch.mean(torch.exp(D_fake - 1)) 91 | """ Reverse KL """ 92 | G_loss = -torch.mean(-1 - D_fake) 93 | """ Pearson Chi-squared """ 94 | # G_loss = -torch.mean(0.25*D_fake**2 + D_fake) 95 | """ Squared Hellinger """ 96 | # G_loss = -torch.mean((1 - torch.exp(D_fake)) / (torch.exp(D_fake))) 97 | 98 | G_loss.backward() 99 | G_solver.step() 100 | reset_grad() 101 | 102 | # Print and plot every now and then 103 | if it % 1000 == 0: 104 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 105 | .format(it, D_loss.data[0], G_loss.data[0])) 106 | 107 | samples = G(z).data.numpy()[:16] 108 | 109 | fig = plt.figure(figsize=(4, 4)) 110 | gs = gridspec.GridSpec(4, 4) 111 | gs.update(wspace=0.05, hspace=0.05) 112 | 113 | for i, sample in enumerate(samples): 114 | ax = plt.subplot(gs[i]) 115 | plt.axis('off') 116 | ax.set_xticklabels([]) 117 | ax.set_yticklabels([]) 118 | ax.set_aspect('equal') 119 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 120 | 121 | if not os.path.exists('out/'): 122 | os.makedirs('out/') 123 | 124 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 125 | cnt += 1 126 | plt.close(fig) 127 | -------------------------------------------------------------------------------- /GAN/ali_bigan/ali_bigan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | d_steps = 3 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | def log(x): 42 | return tf.log(x + 1e-8) 43 | 44 | 45 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | D_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim])) 49 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 51 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 52 | 53 | Q_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 54 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 55 | Q_W2 = tf.Variable(xavier_init([h_dim, z_dim])) 56 | Q_b2 = tf.Variable(tf.zeros(shape=[z_dim])) 57 | 58 | P_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 59 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 60 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 61 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 62 | 63 | theta_G = [Q_W1, Q_W2, Q_b1, Q_b2, P_W1, P_W2, P_b1, P_b2] 64 | theta_D = [D_W1, D_W2, D_b1, D_b2] 65 | 66 | 67 | def sample_z(m, n): 68 | return np.random.uniform(-1., 1., size=[m, n]) 69 | 70 | 71 | def Q(X): 72 | h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1) 73 | h = tf.matmul(h, Q_W2) + Q_b2 74 | return h 75 | 76 | 77 | def P(z): 78 | h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1) 79 | h = tf.matmul(h, P_W2) + P_b2 80 | return tf.nn.sigmoid(h) 81 | 82 | 83 | def D(X, z): 84 | inputs = tf.concat([X, z], axis=1) 85 | h = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1) 86 | return tf.nn.sigmoid(tf.matmul(h, D_W2) + D_b2) 87 | 88 | 89 | z_hat = Q(X) 90 | X_hat = P(z) 91 | 92 | D_enc = D(X, z_hat) 93 | D_gen = D(X_hat, z) 94 | 95 | D_loss = -tf.reduce_mean(log(D_enc) + log(1 - D_gen)) 96 | G_loss = -tf.reduce_mean(log(D_gen) + log(1 - D_enc)) 97 | 98 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 99 | .minimize(D_loss, var_list=theta_D)) 100 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 101 | .minimize(G_loss, var_list=theta_G)) 102 | 103 | sess = tf.Session() 104 | sess.run(tf.global_variables_initializer()) 105 | 106 | if not os.path.exists('out/'): 107 | os.makedirs('out/') 108 | 109 | i = 0 110 | 111 | for it in range(1000000): 112 | X_mb, _ = mnist.train.next_batch(mb_size) 113 | z_mb = sample_z(mb_size, z_dim) 114 | 115 | _, D_loss_curr = sess.run( 116 | [D_solver, D_loss], feed_dict={X: X_mb, z: z_mb} 117 | ) 118 | 119 | _, G_loss_curr = sess.run( 120 | [G_solver, G_loss], feed_dict={X: X_mb, z: z_mb} 121 | ) 122 | 123 | if it % 1000 == 0: 124 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 125 | .format(it, D_loss_curr, G_loss_curr)) 126 | 127 | samples = sess.run(X_hat, feed_dict={z: sample_z(16, z_dim)}) 128 | 129 | fig = plot(samples) 130 | plt.savefig('out/{}.png' 131 | .format(str(i).zfill(3)), bbox_inches='tight') 132 | i += 1 133 | plt.close(fig) 134 | -------------------------------------------------------------------------------- /GAN/improved_wasserstein_gan/wgan_gp_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 10 12 | h_dim = 128 13 | lam = 10 14 | n_disc = 5 15 | lr = 1e-4 16 | 17 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 18 | 19 | 20 | def plot(samples): 21 | fig = plt.figure(figsize=(4, 4)) 22 | gs = gridspec.GridSpec(4, 4) 23 | gs.update(wspace=0.05, hspace=0.05) 24 | 25 | for i, sample in enumerate(samples): 26 | ax = plt.subplot(gs[i]) 27 | plt.axis('off') 28 | ax.set_xticklabels([]) 29 | ax.set_yticklabels([]) 30 | ax.set_aspect('equal') 31 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 32 | 33 | return fig 34 | 35 | 36 | def xavier_init(size): 37 | in_dim = size[0] 38 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 39 | return tf.random_normal(shape=size, stddev=xavier_stddev) 40 | 41 | 42 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 43 | 44 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 45 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 46 | 47 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 48 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 49 | 50 | theta_D = [D_W1, D_W2, D_b1, D_b2] 51 | 52 | 53 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 54 | 55 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 56 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 57 | 58 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 59 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 60 | 61 | theta_G = [G_W1, G_W2, G_b1, G_b2] 62 | 63 | 64 | def sample_z(m, n): 65 | return np.random.uniform(-1., 1., size=[m, n]) 66 | 67 | 68 | def G(z): 69 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 70 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 71 | G_prob = tf.nn.sigmoid(G_log_prob) 72 | return G_prob 73 | 74 | 75 | def D(X): 76 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 77 | out = tf.matmul(D_h1, D_W2) + D_b2 78 | return out 79 | 80 | 81 | G_sample = G(z) 82 | D_real = D(X) 83 | D_fake = D(G_sample) 84 | 85 | eps = tf.random_uniform([mb_size, 1], minval=0., maxval=1.) 86 | X_inter = eps*X + (1. - eps)*G_sample 87 | grad = tf.gradients(D(X_inter), [X_inter])[0] 88 | grad_norm = tf.sqrt(tf.reduce_sum((grad)**2, axis=1)) 89 | grad_pen = lam * tf.reduce_mean(grad_norm - 1.)**2 90 | 91 | D_loss = tf.reduce_mean(D_fake) - tf.reduce_mean(D_real) + grad_pen 92 | G_loss = -tf.reduce_mean(D_fake) 93 | 94 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5) 95 | .minimize(D_loss, var_list=theta_D)) 96 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr, beta1=0.5) 97 | .minimize(G_loss, var_list=theta_G)) 98 | 99 | sess = tf.Session() 100 | sess.run(tf.global_variables_initializer()) 101 | 102 | if not os.path.exists('out/'): 103 | os.makedirs('out/') 104 | 105 | i = 0 106 | 107 | for it in range(1000000): 108 | for _ in range(n_disc): 109 | X_mb, _ = mnist.train.next_batch(mb_size) 110 | 111 | _, D_loss_curr = sess.run( 112 | [D_solver, D_loss], 113 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 114 | ) 115 | 116 | _, G_loss_curr = sess.run( 117 | [G_solver, G_loss], 118 | feed_dict={z: sample_z(mb_size, z_dim)} 119 | ) 120 | 121 | if it % 1000 == 0: 122 | print('Iter: {}; D loss: {:.4}; G_loss: {:.4}' 123 | .format(it, D_loss_curr, G_loss_curr)) 124 | 125 | if it % 1000 == 0: 126 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 127 | 128 | fig = plot(samples) 129 | plt.savefig('out/{}.png' 130 | .format(str(i).zfill(3)), bbox_inches='tight') 131 | i += 1 132 | plt.close(fig) 133 | -------------------------------------------------------------------------------- /VAE/vanilla_vae/vae_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.gridspec as gridspec 5 | import os 6 | from torch.autograd import Variable 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | mb_size = 64 12 | z_dim = 100 13 | X_dim = mnist.train.images.shape[1] 14 | y_dim = mnist.train.labels.shape[1] 15 | h_dim = 128 16 | c = 0 17 | lr = 1e-3 18 | 19 | 20 | def plot(samples): 21 | fig = plt.figure(figsize=(4, 4)) 22 | gs = gridspec.GridSpec(4, 4) 23 | gs.update(wspace=0.05, hspace=0.05) 24 | 25 | for i, sample in enumerate(samples): 26 | ax = plt.subplot(gs[i]) 27 | plt.axis('off') 28 | ax.set_xticklabels([]) 29 | ax.set_yticklabels([]) 30 | ax.set_aspect('equal') 31 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 32 | 33 | return fig 34 | 35 | 36 | def xavier_init(size): 37 | in_dim = size[0] 38 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 39 | return tf.random_normal(shape=size, stddev=xavier_stddev) 40 | 41 | 42 | # =============================== Q(z|X) ====================================== 43 | 44 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 45 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 46 | 47 | Q_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 48 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 49 | 50 | Q_W2_mu = tf.Variable(xavier_init([h_dim, z_dim])) 51 | Q_b2_mu = tf.Variable(tf.zeros(shape=[z_dim])) 52 | 53 | Q_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim])) 54 | Q_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim])) 55 | 56 | 57 | def Q(X): 58 | h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1) 59 | z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu 60 | z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma 61 | return z_mu, z_logvar 62 | 63 | 64 | def sample_z(mu, log_var): 65 | eps = tf.random_normal(shape=tf.shape(mu)) 66 | return mu + tf.exp(log_var / 2) * eps 67 | 68 | 69 | # =============================== P(X|z) ====================================== 70 | 71 | P_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 72 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 73 | 74 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 75 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 76 | 77 | 78 | def P(z): 79 | h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1) 80 | logits = tf.matmul(h, P_W2) + P_b2 81 | prob = tf.nn.sigmoid(logits) 82 | return prob, logits 83 | 84 | 85 | # =============================== TRAINING ==================================== 86 | 87 | z_mu, z_logvar = Q(X) 88 | z_sample = sample_z(z_mu, z_logvar) 89 | _, logits = P(z_sample) 90 | 91 | # Sampling from random z 92 | X_samples, _ = P(z) 93 | 94 | # E[log P(X|z)] 95 | recon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1) 96 | # D_KL(Q(z|X) || P(z|X)); calculate in closed form as both dist. are Gaussian 97 | kl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1) 98 | # VAE loss 99 | vae_loss = tf.reduce_mean(recon_loss + kl_loss) 100 | 101 | solver = tf.train.AdamOptimizer().minimize(vae_loss) 102 | 103 | sess = tf.Session() 104 | sess.run(tf.global_variables_initializer()) 105 | 106 | if not os.path.exists('out/'): 107 | os.makedirs('out/') 108 | 109 | i = 0 110 | 111 | for it in range(1000000): 112 | X_mb, _ = mnist.train.next_batch(mb_size) 113 | 114 | _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb}) 115 | 116 | if it % 1000 == 0: 117 | print('Iter: {}'.format(it)) 118 | print('Loss: {:.4}'. format(loss)) 119 | print() 120 | 121 | samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)}) 122 | 123 | fig = plot(samples) 124 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 125 | i += 1 126 | plt.close(fig) 127 | -------------------------------------------------------------------------------- /GAN/vanilla_gan/gan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | def xavier_init(size): 10 | in_dim = size[0] 11 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 12 | return tf.random_normal(shape=size, stddev=xavier_stddev) 13 | 14 | 15 | X = tf.placeholder(tf.float32, shape=[None, 784]) 16 | 17 | D_W1 = tf.Variable(xavier_init([784, 128])) 18 | D_b1 = tf.Variable(tf.zeros(shape=[128])) 19 | 20 | D_W2 = tf.Variable(xavier_init([128, 1])) 21 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 22 | 23 | theta_D = [D_W1, D_W2, D_b1, D_b2] 24 | 25 | 26 | Z = tf.placeholder(tf.float32, shape=[None, 100]) 27 | 28 | G_W1 = tf.Variable(xavier_init([100, 128])) 29 | G_b1 = tf.Variable(tf.zeros(shape=[128])) 30 | 31 | G_W2 = tf.Variable(xavier_init([128, 784])) 32 | G_b2 = tf.Variable(tf.zeros(shape=[784])) 33 | 34 | theta_G = [G_W1, G_W2, G_b1, G_b2] 35 | 36 | 37 | def sample_Z(m, n): 38 | return np.random.uniform(-1., 1., size=[m, n]) 39 | 40 | 41 | def generator(z): 42 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 43 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 44 | G_prob = tf.nn.sigmoid(G_log_prob) 45 | 46 | return G_prob 47 | 48 | 49 | def discriminator(x): 50 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 51 | D_logit = tf.matmul(D_h1, D_W2) + D_b2 52 | D_prob = tf.nn.sigmoid(D_logit) 53 | 54 | return D_prob, D_logit 55 | 56 | 57 | def plot(samples): 58 | fig = plt.figure(figsize=(4, 4)) 59 | gs = gridspec.GridSpec(4, 4) 60 | gs.update(wspace=0.05, hspace=0.05) 61 | 62 | for i, sample in enumerate(samples): 63 | ax = plt.subplot(gs[i]) 64 | plt.axis('off') 65 | ax.set_xticklabels([]) 66 | ax.set_yticklabels([]) 67 | ax.set_aspect('equal') 68 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 69 | 70 | return fig 71 | 72 | 73 | G_sample = generator(Z) 74 | D_real, D_logit_real = discriminator(X) 75 | D_fake, D_logit_fake = discriminator(G_sample) 76 | 77 | # D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake)) 78 | # G_loss = -tf.reduce_mean(tf.log(D_fake)) 79 | 80 | # Alternative losses: 81 | # ------------------- 82 | D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real))) 83 | D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake))) 84 | D_loss = D_loss_real + D_loss_fake 85 | G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake))) 86 | 87 | D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D) 88 | G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G) 89 | 90 | mb_size = 128 91 | Z_dim = 100 92 | 93 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 94 | 95 | sess = tf.Session() 96 | sess.run(tf.global_variables_initializer()) 97 | 98 | if not os.path.exists('out/'): 99 | os.makedirs('out/') 100 | 101 | i = 0 102 | 103 | for it in range(1000000): 104 | if it % 1000 == 0: 105 | samples = sess.run(G_sample, feed_dict={Z: sample_Z(16, Z_dim)}) 106 | 107 | fig = plot(samples) 108 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 109 | i += 1 110 | plt.close(fig) 111 | 112 | X_mb, _ = mnist.train.next_batch(mb_size) 113 | 114 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: sample_Z(mb_size, Z_dim)}) 115 | _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: sample_Z(mb_size, Z_dim)}) 116 | 117 | if it % 1000 == 0: 118 | print('Iter: {}'.format(it)) 119 | print('D loss: {:.4}'. format(D_loss_curr)) 120 | print('G_loss: {:.4}'.format(G_loss_curr)) 121 | print() 122 | -------------------------------------------------------------------------------- /GAN/generative_adversarial_parallelization/gap_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | import random 11 | from torch.autograd import Variable 12 | from tensorflow.examples.tutorials.mnist import input_data 13 | 14 | 15 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 16 | mb_size = 32 17 | z_dim = 10 18 | X_dim = mnist.train.images.shape[1] 19 | y_dim = mnist.train.labels.shape[1] 20 | h_dim = 128 21 | cnt = 0 22 | lr = 1e-3 23 | K = 100 24 | 25 | 26 | def log(x): 27 | return torch.log(x + 1e-8) 28 | 29 | 30 | G1_ = torch.nn.Sequential( 31 | torch.nn.Linear(z_dim, h_dim), 32 | torch.nn.ReLU(), 33 | torch.nn.Linear(h_dim, X_dim), 34 | torch.nn.Sigmoid() 35 | ) 36 | 37 | 38 | D1_ = torch.nn.Sequential( 39 | torch.nn.Linear(X_dim, h_dim), 40 | torch.nn.ReLU(), 41 | torch.nn.Linear(h_dim, 1), 42 | torch.nn.Sigmoid() 43 | ) 44 | 45 | G2_ = torch.nn.Sequential( 46 | torch.nn.Linear(z_dim, h_dim), 47 | torch.nn.ReLU(), 48 | torch.nn.Linear(h_dim, X_dim), 49 | torch.nn.Sigmoid() 50 | ) 51 | 52 | 53 | D2_ = torch.nn.Sequential( 54 | torch.nn.Linear(X_dim, h_dim), 55 | torch.nn.ReLU(), 56 | torch.nn.Linear(h_dim, 1), 57 | torch.nn.Sigmoid() 58 | ) 59 | 60 | nets = [G1_, D1_, G2_, D2_] 61 | 62 | 63 | def reset_grad(): 64 | for net in nets: 65 | net.zero_grad() 66 | 67 | 68 | G1_solver = optim.Adam(G1_.parameters(), lr=lr) 69 | D1_solver = optim.Adam(D1_.parameters(), lr=lr) 70 | G2_solver = optim.Adam(G2_.parameters(), lr=lr) 71 | D2_solver = optim.Adam(D2_.parameters(), lr=lr) 72 | 73 | D1 = {'model': D1_, 'solver': D1_solver} 74 | G1 = {'model': G1_, 'solver': G1_solver} 75 | D2 = {'model': D2_, 'solver': D2_solver} 76 | G2 = {'model': G2_, 'solver': G2_solver} 77 | 78 | GAN_pairs = [(D1, G1), (D2, G2)] 79 | 80 | for it in range(1000000): 81 | # Sample data 82 | z = Variable(torch.randn(mb_size, z_dim)) 83 | X, _ = mnist.train.next_batch(mb_size) 84 | X = Variable(torch.from_numpy(X)) 85 | 86 | for D, G in GAN_pairs: 87 | # Discriminator 88 | G_sample = G['model'](z) 89 | D_real = D['model'](X) 90 | D_fake = D['model'](G_sample) 91 | 92 | D_loss = -torch.mean(log(D_real) + log(1 - D_fake)) 93 | 94 | D_loss.backward() 95 | D['solver'].step() 96 | reset_grad() 97 | 98 | # Generator 99 | G_sample = G['model'](z) 100 | D_fake = D['model'](G_sample) 101 | 102 | G_loss = -torch.mean(log(D_fake)) 103 | 104 | G_loss.backward() 105 | G['solver'].step() 106 | reset_grad() 107 | 108 | if it != 0 and it % K == 0: 109 | # Swap (D, G) pairs 110 | new_D1, new_D2 = GAN_pairs[1][0], GAN_pairs[0][0] 111 | GAN_pairs = [(new_D1, G1), (new_D2, G2)] 112 | 113 | # Print and plot every now and then 114 | if it % 1000 == 0: 115 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 116 | .format(it, D_loss.data[0], G_loss.data[0])) 117 | 118 | # Pick G randomly 119 | G_rand = random.choice([G1_, G2_]) 120 | samples = G_rand(z).data.numpy()[:16] 121 | 122 | fig = plt.figure(figsize=(4, 4)) 123 | gs = gridspec.GridSpec(4, 4) 124 | gs.update(wspace=0.05, hspace=0.05) 125 | 126 | for i, sample in enumerate(samples): 127 | ax = plt.subplot(gs[i]) 128 | plt.axis('off') 129 | ax.set_xticklabels([]) 130 | ax.set_yticklabels([]) 131 | ax.set_aspect('equal') 132 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 133 | 134 | if not os.path.exists('out/'): 135 | os.makedirs('out/') 136 | 137 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 138 | cnt += 1 139 | plt.close(fig) 140 | -------------------------------------------------------------------------------- /GAN/vanilla_gan/gan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 64 15 | Z_dim = 100 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | c = 0 20 | lr = 1e-3 21 | 22 | 23 | def xavier_init(size): 24 | in_dim = size[0] 25 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 26 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 27 | 28 | 29 | """ ==================== GENERATOR ======================== """ 30 | 31 | Wzh = xavier_init(size=[Z_dim, h_dim]) 32 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whx = xavier_init(size=[h_dim, X_dim]) 35 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 36 | 37 | 38 | def G(z): 39 | h = nn.relu(z @ Wzh + bzh.repeat(z.size(0), 1)) 40 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 41 | return X 42 | 43 | 44 | """ ==================== DISCRIMINATOR ======================== """ 45 | 46 | Wxh = xavier_init(size=[X_dim, h_dim]) 47 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 48 | 49 | Why = xavier_init(size=[h_dim, 1]) 50 | bhy = Variable(torch.zeros(1), requires_grad=True) 51 | 52 | 53 | def D(X): 54 | h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1)) 55 | y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1)) 56 | return y 57 | 58 | 59 | G_params = [Wzh, bzh, Whx, bhx] 60 | D_params = [Wxh, bxh, Why, bhy] 61 | params = G_params + D_params 62 | 63 | 64 | """ ===================== TRAINING ======================== """ 65 | 66 | 67 | def reset_grad(): 68 | for p in params: 69 | p.grad.data.zero_() 70 | 71 | 72 | G_solver = optim.Adam(G_params, lr=1e-3) 73 | D_solver = optim.Adam(D_params, lr=1e-3) 74 | 75 | ones_label = Variable(torch.ones(mb_size)) 76 | zeros_label = Variable(torch.zeros(mb_size)) 77 | 78 | 79 | for it in range(100000): 80 | # Sample data 81 | z = Variable(torch.randn(mb_size, Z_dim)) 82 | X, _ = mnist.train.next_batch(mb_size) 83 | X = Variable(torch.from_numpy(X)) 84 | 85 | # Dicriminator forward-loss-backward-update 86 | G_sample = G(z) 87 | D_real = D(X) 88 | D_fake = D(G_sample) 89 | 90 | D_loss_real = nn.binary_cross_entropy(D_real, ones_label) 91 | D_loss_fake = nn.binary_cross_entropy(D_fake, zeros_label) 92 | D_loss = D_loss_real + D_loss_fake 93 | 94 | D_loss.backward() 95 | D_solver.step() 96 | 97 | # Housekeeping - reset gradient 98 | reset_grad() 99 | 100 | # Generator forward-loss-backward-update 101 | z = Variable(torch.randn(mb_size, Z_dim)) 102 | G_sample = G(z) 103 | D_fake = D(G_sample) 104 | 105 | G_loss = nn.binary_cross_entropy(D_fake, ones_label) 106 | 107 | G_loss.backward() 108 | G_solver.step() 109 | 110 | # Housekeeping - reset gradient 111 | reset_grad() 112 | 113 | # Print and plot every now and then 114 | if it % 1000 == 0: 115 | print('Iter-{}; D_loss: {}; G_loss: {}'.format(it, D_loss.data.numpy(), G_loss.data.numpy())) 116 | 117 | samples = G(z).data.numpy()[:16] 118 | 119 | fig = plt.figure(figsize=(4, 4)) 120 | gs = gridspec.GridSpec(4, 4) 121 | gs.update(wspace=0.05, hspace=0.05) 122 | 123 | for i, sample in enumerate(samples): 124 | ax = plt.subplot(gs[i]) 125 | plt.axis('off') 126 | ax.set_xticklabels([]) 127 | ax.set_yticklabels([]) 128 | ax.set_aspect('equal') 129 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 130 | 131 | if not os.path.exists('out/'): 132 | os.makedirs('out/') 133 | 134 | plt.savefig('out/{}.png'.format(str(c).zfill(3)), bbox_inches='tight') 135 | c += 1 136 | plt.close(fig) 137 | -------------------------------------------------------------------------------- /GAN/magan/magan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 10 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | d_step = 3 22 | lr = 5e-4 23 | m = 5 24 | n_iter = 1000 25 | n_epoch = 1000 26 | N = n_iter * mb_size # N data per epoch 27 | 28 | 29 | G = torch.nn.Sequential( 30 | torch.nn.Linear(z_dim, h_dim), 31 | torch.nn.ReLU(), 32 | torch.nn.Linear(h_dim, X_dim), 33 | torch.nn.Sigmoid() 34 | ) 35 | 36 | # D is an autoencoder 37 | D_ = torch.nn.Sequential( 38 | torch.nn.Linear(X_dim, h_dim), 39 | torch.nn.ReLU(), 40 | torch.nn.Linear(h_dim, X_dim), 41 | ) 42 | 43 | 44 | # Energy is the MSE of autoencoder 45 | def D(X): 46 | X_recon = D_(X) 47 | return torch.sum((X - X_recon)**2, 1) 48 | 49 | 50 | def reset_grad(): 51 | G.zero_grad() 52 | D_.zero_grad() 53 | 54 | 55 | G_solver = optim.Adamax(G.parameters(), lr=lr) 56 | D_solver = optim.Adamax(D_.parameters(), lr=lr) 57 | 58 | 59 | # Pretrain discriminator 60 | for it in range(2*n_iter): 61 | X, _ = mnist.train.next_batch(mb_size) 62 | X = Variable(torch.from_numpy(X)) 63 | 64 | loss = torch.mean(D(X)) # Minimize real samples energy 65 | 66 | loss.backward() 67 | D_solver.step() 68 | reset_grad() 69 | 70 | if it % 1000 == 0: 71 | print('Iter-{}; Pretrained D loss: {:.4}'.format(it, loss.data[0])) 72 | 73 | 74 | # Initial margin, expected energy of real data 75 | m = torch.mean(D(Variable(torch.from_numpy(mnist.train.images)))).data[0] 76 | s_z_before = torch.from_numpy(np.array([np.inf], dtype='float32')) 77 | 78 | 79 | # GAN training 80 | for t in range(n_epoch): 81 | s_x, s_z = torch.zeros(1), torch.zeros(1) 82 | 83 | for it in range(n_iter): 84 | # Sample data 85 | z = Variable(torch.randn(mb_size, z_dim)) 86 | X, _ = mnist.train.next_batch(mb_size) 87 | X = Variable(torch.from_numpy(X)) 88 | 89 | # Dicriminator 90 | G_sample = G(z) 91 | D_real = D(X) 92 | D_fake = D(G_sample) 93 | 94 | D_loss = torch.mean(D_real) + nn.relu(m - torch.mean(D_fake)) 95 | 96 | D_loss.backward() 97 | D_solver.step() 98 | 99 | # Update real samples statistics 100 | s_x += torch.sum(D_real.data) 101 | 102 | reset_grad() 103 | 104 | # Generator 105 | z = Variable(torch.randn(mb_size, z_dim)) 106 | G_sample = G(z) 107 | D_fake = D(G_sample) 108 | 109 | G_loss = torch.mean(D_fake) 110 | 111 | G_loss.backward() 112 | G_solver.step() 113 | 114 | # Update fake samples statistics 115 | s_z += torch.sum(D_fake.data) 116 | 117 | reset_grad() 118 | 119 | # Update margin 120 | if (((s_x[0] / N) < m) and (s_x[0] < s_z[0]) and (s_z_before[0] < s_z[0])): 121 | m = s_x[0] / N 122 | 123 | s_z_before = s_z 124 | 125 | # Convergence measure 126 | Ex = s_x[0] / N 127 | Ez = s_z[0] / N 128 | L = Ex + np.abs(Ex - Ez) 129 | 130 | # Visualize 131 | print('Epoch-{}; m = {:.4}; L = {:.4}' 132 | .format(t, m, L)) 133 | 134 | samples = G(z).data.numpy()[:16] 135 | 136 | fig = plt.figure(figsize=(4, 4)) 137 | gs = gridspec.GridSpec(4, 4) 138 | gs.update(wspace=0.05, hspace=0.05) 139 | 140 | for i, sample in enumerate(samples): 141 | ax = plt.subplot(gs[i]) 142 | plt.axis('off') 143 | ax.set_xticklabels([]) 144 | ax.set_yticklabels([]) 145 | ax.set_aspect('equal') 146 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 147 | 148 | if not os.path.exists('out/'): 149 | os.makedirs('out/') 150 | 151 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 152 | cnt += 1 153 | plt.close(fig) 154 | -------------------------------------------------------------------------------- /VAE/conditional_vae/cvae_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 64 15 | Z_dim = 100 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | cnt = 0 20 | lr = 1e-3 21 | 22 | 23 | def xavier_init(size): 24 | in_dim = size[0] 25 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 26 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 27 | 28 | 29 | # =============================== Q(z|X) ====================================== 30 | 31 | Wxh = xavier_init(size=[X_dim + y_dim, h_dim]) 32 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whz_mu = xavier_init(size=[h_dim, Z_dim]) 35 | bhz_mu = Variable(torch.zeros(Z_dim), requires_grad=True) 36 | 37 | Whz_var = xavier_init(size=[h_dim, Z_dim]) 38 | bhz_var = Variable(torch.zeros(Z_dim), requires_grad=True) 39 | 40 | 41 | def Q(X, c): 42 | inputs = torch.cat([X, c], 1) 43 | h = nn.relu(inputs @ Wxh + bxh.repeat(inputs.size(0), 1)) 44 | z_mu = h @ Whz_mu + bhz_mu.repeat(h.size(0), 1) 45 | z_var = h @ Whz_var + bhz_var.repeat(h.size(0), 1) 46 | return z_mu, z_var 47 | 48 | 49 | def sample_z(mu, log_var): 50 | eps = Variable(torch.randn(mb_size, Z_dim)) 51 | return mu + torch.exp(log_var / 2) * eps 52 | 53 | 54 | # =============================== P(X|z) ====================================== 55 | 56 | Wzh = xavier_init(size=[Z_dim + y_dim, h_dim]) 57 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 58 | 59 | Whx = xavier_init(size=[h_dim, X_dim]) 60 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 61 | 62 | 63 | def P(z, c): 64 | inputs = torch.cat([z, c], 1) 65 | h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1)) 66 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 67 | return X 68 | 69 | 70 | # =============================== TRAINING ==================================== 71 | 72 | params = [Wxh, bxh, Whz_mu, bhz_mu, Whz_var, bhz_var, 73 | Wzh, bzh, Whx, bhx] 74 | 75 | solver = optim.Adam(params, lr=lr) 76 | 77 | for it in range(100000): 78 | X, c = mnist.train.next_batch(mb_size) 79 | X = Variable(torch.from_numpy(X)) 80 | c = Variable(torch.from_numpy(c.astype('float32'))) 81 | 82 | # Forward 83 | z_mu, z_var = Q(X, c) 84 | z = sample_z(z_mu, z_var) 85 | X_sample = P(z, c) 86 | 87 | # Loss 88 | recon_loss = nn.binary_cross_entropy(X_sample, X, size_average=False) / mb_size 89 | kl_loss = torch.mean(0.5 * torch.sum(torch.exp(z_var) + z_mu**2 - 1. - z_var, 1)) 90 | loss = recon_loss + kl_loss 91 | 92 | # Backward 93 | loss.backward() 94 | 95 | # Update 96 | solver.step() 97 | 98 | # Housekeeping 99 | for p in params: 100 | p.grad.data.zero_() 101 | 102 | # Print and plot every now and then 103 | if it % 1000 == 0: 104 | print('Iter-{}; Loss: {:.4}'.format(it, loss.data[0])) 105 | 106 | c = np.zeros(shape=[mb_size, y_dim], dtype='float32') 107 | c[:, np.random.randint(0, 10)] = 1. 108 | c = Variable(torch.from_numpy(c)) 109 | z = Variable(torch.randn(mb_size, Z_dim)) 110 | samples = P(z, c).data.numpy()[:16] 111 | 112 | fig = plt.figure(figsize=(4, 4)) 113 | gs = gridspec.GridSpec(4, 4) 114 | gs.update(wspace=0.05, hspace=0.05) 115 | 116 | for i, sample in enumerate(samples): 117 | ax = plt.subplot(gs[i]) 118 | plt.axis('off') 119 | ax.set_xticklabels([]) 120 | ax.set_yticklabels([]) 121 | ax.set_aspect('equal') 122 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 123 | 124 | if not os.path.exists('out/'): 125 | os.makedirs('out/') 126 | 127 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 128 | cnt += 1 129 | plt.close(fig) 130 | -------------------------------------------------------------------------------- /VAE/adversarial_autoencoder/aae_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.gridspec as gridspec 5 | import os 6 | from tensorflow.examples.tutorials.mnist import input_data 7 | 8 | 9 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 10 | mb_size = 32 11 | z_dim = 10 12 | X_dim = mnist.train.images.shape[1] 13 | y_dim = mnist.train.labels.shape[1] 14 | h_dim = 128 15 | c = 0 16 | lr = 1e-3 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | """ Q(z|X) """ 42 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 43 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 44 | 45 | Q_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 46 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 47 | 48 | Q_W2 = tf.Variable(xavier_init([h_dim, z_dim])) 49 | Q_b2 = tf.Variable(tf.zeros(shape=[z_dim])) 50 | 51 | theta_Q = [Q_W1, Q_W2, Q_b1, Q_b2] 52 | 53 | 54 | def Q(X): 55 | h = tf.nn.relu(tf.matmul(X, Q_W1) + Q_b1) 56 | z = tf.matmul(h, Q_W2) + Q_b2 57 | return z 58 | 59 | 60 | """ P(X|z) """ 61 | P_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 62 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 63 | 64 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 65 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 66 | 67 | theta_P = [P_W1, P_W2, P_b1, P_b2] 68 | 69 | 70 | def P(z): 71 | h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1) 72 | logits = tf.matmul(h, P_W2) + P_b2 73 | prob = tf.nn.sigmoid(logits) 74 | return prob, logits 75 | 76 | 77 | """ D(z) """ 78 | D_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 79 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 80 | 81 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 82 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 83 | 84 | theta_D = [D_W1, D_W2, D_b1, D_b2] 85 | 86 | 87 | def D(z): 88 | h = tf.nn.relu(tf.matmul(z, D_W1) + D_b1) 89 | logits = tf.matmul(h, D_W2) + D_b2 90 | prob = tf.nn.sigmoid(logits) 91 | return prob 92 | 93 | 94 | """ Training """ 95 | z_sample = Q(X) 96 | _, logits = P(z_sample) 97 | 98 | # Sample from random z 99 | X_samples, _ = P(z) 100 | 101 | # E[log P(X|z)] 102 | recon_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X)) 103 | 104 | # Adversarial loss to approx. Q(z|X) 105 | D_real = D(z) 106 | D_fake = D(z_sample) 107 | 108 | D_loss = -tf.reduce_mean(tf.log(D_real) + tf.log(1. - D_fake)) 109 | G_loss = -tf.reduce_mean(tf.log(D_fake)) 110 | 111 | AE_solver = tf.train.AdamOptimizer().minimize(recon_loss, var_list=theta_P + theta_Q) 112 | D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D) 113 | G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_Q) 114 | 115 | sess = tf.Session() 116 | sess.run(tf.global_variables_initializer()) 117 | 118 | if not os.path.exists('out/'): 119 | os.makedirs('out/') 120 | 121 | i = 0 122 | 123 | for it in range(1000000): 124 | X_mb, _ = mnist.train.next_batch(mb_size) 125 | z_mb = np.random.randn(mb_size, z_dim) 126 | 127 | _, recon_loss_curr = sess.run([AE_solver, recon_loss], feed_dict={X: X_mb}) 128 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb}) 129 | _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={X: X_mb}) 130 | 131 | if it % 1000 == 0: 132 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}; Recon_loss: {:.4}' 133 | .format(it, D_loss_curr, G_loss_curr, recon_loss_curr)) 134 | 135 | samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)}) 136 | 137 | fig = plot(samples) 138 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 139 | i += 1 140 | plt.close(fig) 141 | -------------------------------------------------------------------------------- /VAE/conditional_vae/cvae_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.gridspec as gridspec 5 | import os 6 | from torch.autograd import Variable 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | mb_size = 64 12 | z_dim = 100 13 | X_dim = mnist.train.images.shape[1] 14 | y_dim = mnist.train.labels.shape[1] 15 | h_dim = 128 16 | c = 0 17 | lr = 1e-3 18 | 19 | 20 | def plot(samples): 21 | fig = plt.figure(figsize=(4, 4)) 22 | gs = gridspec.GridSpec(4, 4) 23 | gs.update(wspace=0.05, hspace=0.05) 24 | 25 | for i, sample in enumerate(samples): 26 | ax = plt.subplot(gs[i]) 27 | plt.axis('off') 28 | ax.set_xticklabels([]) 29 | ax.set_yticklabels([]) 30 | ax.set_aspect('equal') 31 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 32 | 33 | return fig 34 | 35 | 36 | def xavier_init(size): 37 | in_dim = size[0] 38 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 39 | return tf.random_normal(shape=size, stddev=xavier_stddev) 40 | 41 | 42 | # =============================== Q(z|X) ====================================== 43 | 44 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 45 | c = tf.placeholder(tf.float32, shape=[None, y_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | Q_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim])) 49 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | 51 | Q_W2_mu = tf.Variable(xavier_init([h_dim, z_dim])) 52 | Q_b2_mu = tf.Variable(tf.zeros(shape=[z_dim])) 53 | 54 | Q_W2_sigma = tf.Variable(xavier_init([h_dim, z_dim])) 55 | Q_b2_sigma = tf.Variable(tf.zeros(shape=[z_dim])) 56 | 57 | 58 | def Q(X, c): 59 | inputs = tf.concat(axis=1, values=[X, c]) 60 | h = tf.nn.relu(tf.matmul(inputs, Q_W1) + Q_b1) 61 | z_mu = tf.matmul(h, Q_W2_mu) + Q_b2_mu 62 | z_logvar = tf.matmul(h, Q_W2_sigma) + Q_b2_sigma 63 | return z_mu, z_logvar 64 | 65 | 66 | def sample_z(mu, log_var): 67 | eps = tf.random_normal(shape=tf.shape(mu)) 68 | return mu + tf.exp(log_var / 2) * eps 69 | 70 | 71 | # =============================== P(X|z) ====================================== 72 | 73 | P_W1 = tf.Variable(xavier_init([z_dim + y_dim, h_dim])) 74 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 75 | 76 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 77 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 78 | 79 | 80 | def P(z, c): 81 | inputs = tf.concat(axis=1, values=[z, c]) 82 | h = tf.nn.relu(tf.matmul(inputs, P_W1) + P_b1) 83 | logits = tf.matmul(h, P_W2) + P_b2 84 | prob = tf.nn.sigmoid(logits) 85 | return prob, logits 86 | 87 | 88 | # =============================== TRAINING ==================================== 89 | 90 | z_mu, z_logvar = Q(X, c) 91 | z_sample = sample_z(z_mu, z_logvar) 92 | _, logits = P(z_sample, c) 93 | 94 | # Sampling from random z 95 | X_samples, _ = P(z, c) 96 | 97 | # E[log P(X|z)] 98 | recon_loss = tf.reduce_sum(tf.nn.sigmoid_cross_entropy_with_logits(logits=logits, labels=X), 1) 99 | # D_KL(Q(z|X) || P(z|X)); calculate in closed form as both dist. are Gaussian 100 | kl_loss = 0.5 * tf.reduce_sum(tf.exp(z_logvar) + z_mu**2 - 1. - z_logvar, 1) 101 | # VAE loss 102 | vae_loss = tf.reduce_mean(recon_loss + kl_loss) 103 | 104 | solver = tf.train.AdamOptimizer().minimize(vae_loss) 105 | 106 | sess = tf.Session() 107 | sess.run(tf.global_variables_initializer()) 108 | 109 | if not os.path.exists('out/'): 110 | os.makedirs('out/') 111 | 112 | i = 0 113 | 114 | for it in range(1000000): 115 | X_mb, y_mb = mnist.train.next_batch(mb_size) 116 | 117 | _, loss = sess.run([solver, vae_loss], feed_dict={X: X_mb, c: y_mb}) 118 | 119 | if it % 1000 == 0: 120 | print('Iter: {}'.format(it)) 121 | print('Loss: {:.4}'. format(loss)) 122 | print() 123 | 124 | y = np.zeros(shape=[16, y_dim]) 125 | y[:, np.random.randint(0, y_dim)] = 1. 126 | 127 | samples = sess.run(X_samples, 128 | feed_dict={z: np.random.randn(16, z_dim), c: y}) 129 | 130 | fig = plot(samples) 131 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 132 | i += 1 133 | plt.close(fig) 134 | -------------------------------------------------------------------------------- /GAN/mode_regularized_gan/mode_reg_gan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | 13 | 14 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 15 | mb_size = 32 16 | z_dim = 128 17 | X_dim = mnist.train.images.shape[1] 18 | y_dim = mnist.train.labels.shape[1] 19 | h_dim = 128 20 | cnt = 0 21 | lr = 1e-4 22 | lam1 = 1e-2 23 | lam2 = 1e-2 24 | 25 | 26 | def log(x): 27 | return torch.log(x + 1e-8) 28 | 29 | 30 | E = torch.nn.Sequential( 31 | torch.nn.Linear(X_dim, h_dim), 32 | torch.nn.ReLU(), 33 | torch.nn.Linear(h_dim, z_dim) 34 | ) 35 | 36 | G = torch.nn.Sequential( 37 | torch.nn.Linear(z_dim, h_dim), 38 | torch.nn.ReLU(), 39 | torch.nn.Linear(h_dim, X_dim), 40 | torch.nn.Sigmoid() 41 | ) 42 | 43 | D = torch.nn.Sequential( 44 | torch.nn.Linear(X_dim, h_dim), 45 | torch.nn.ReLU(), 46 | torch.nn.Linear(h_dim, 1), 47 | torch.nn.Sigmoid() 48 | ) 49 | 50 | 51 | def reset_grad(): 52 | G.zero_grad() 53 | D.zero_grad() 54 | E.zero_grad() 55 | 56 | 57 | def sample_X(size, include_y=False): 58 | X, y = mnist.train.next_batch(size) 59 | X = Variable(torch.from_numpy(X)) 60 | 61 | if include_y: 62 | y = np.argmax(y, axis=1).astype(np.int) 63 | y = Variable(torch.from_numpy(y)) 64 | return X, y 65 | 66 | return X 67 | 68 | 69 | E_solver = optim.Adam(E.parameters(), lr=lr) 70 | G_solver = optim.Adam(G.parameters(), lr=lr) 71 | D_solver = optim.Adam(D.parameters(), lr=lr) 72 | 73 | 74 | for it in range(1000000): 75 | """ Discriminator """ 76 | # Sample data 77 | X = sample_X(mb_size) 78 | z = Variable(torch.randn(mb_size, z_dim)) 79 | 80 | # Dicriminator_1 forward-loss-backward-update 81 | G_sample = G(z) 82 | D_real = D(X) 83 | D_fake = D(G_sample) 84 | 85 | D_loss = -torch.mean(log(D_real) + log(1 - D_fake)) 86 | 87 | D_loss.backward() 88 | D_solver.step() 89 | 90 | # Housekeeping - reset gradient 91 | reset_grad() 92 | 93 | """ Generator """ 94 | # Sample data 95 | X = sample_X(mb_size) 96 | z = Variable(torch.randn(mb_size, z_dim)) 97 | 98 | # Generator forward-loss-backward-update 99 | G_sample = G(z) 100 | G_sample_reg = G(E(X)) 101 | D_fake = D(G_sample) 102 | D_reg = D(G_sample_reg) 103 | 104 | mse = torch.sum((X - G_sample_reg)**2, 1) 105 | reg = torch.mean(lam1 * mse + lam2 * log(D_reg)) 106 | G_loss = -torch.mean(log(D_fake)) + reg 107 | 108 | G_loss.backward() 109 | G_solver.step() 110 | 111 | # Housekeeping - reset gradient 112 | reset_grad() 113 | 114 | """ Encoder """ 115 | # Sample data 116 | X = sample_X(mb_size) 117 | z = Variable(torch.randn(mb_size, z_dim)) 118 | 119 | G_sample_reg = G(E(X)) 120 | D_reg = D(G_sample_reg) 121 | 122 | mse = torch.sum((X - G_sample_reg)**2, 1) 123 | E_loss = torch.mean(lam1 * mse + lam2 * log(D_reg)) 124 | 125 | E_loss.backward() 126 | E_solver.step() 127 | 128 | # Housekeeping - reset gradient 129 | reset_grad() 130 | 131 | # Print and plot every now and then 132 | if it % 1000 == 0: 133 | print('Iter-{}; D_loss: {}; E_loss: {}; G_loss: {}' 134 | .format(it, D_loss.data.numpy(), E_loss.data.numpy(), G_loss.data.numpy())) 135 | 136 | samples = G(z).data.numpy()[:16] 137 | 138 | fig = plt.figure(figsize=(4, 4)) 139 | gs = gridspec.GridSpec(4, 4) 140 | gs.update(wspace=0.05, hspace=0.05) 141 | 142 | for i, sample in enumerate(samples): 143 | ax = plt.subplot(gs[i]) 144 | plt.axis('off') 145 | ax.set_xticklabels([]) 146 | ax.set_yticklabels([]) 147 | ax.set_aspect('equal') 148 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 149 | 150 | if not os.path.exists('out/'): 151 | os.makedirs('out/') 152 | 153 | plt.savefig('out/{}.png' 154 | .format(str(cnt).zfill(3)), bbox_inches='tight') 155 | cnt += 1 156 | plt.close(fig) 157 | -------------------------------------------------------------------------------- /GAN/conditional_gan/cgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 10 | mb_size = 64 11 | Z_dim = 100 12 | X_dim = mnist.train.images.shape[1] 13 | y_dim = mnist.train.labels.shape[1] 14 | h_dim = 128 15 | 16 | 17 | def xavier_init(size): 18 | in_dim = size[0] 19 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 20 | return tf.random_normal(shape=size, stddev=xavier_stddev) 21 | 22 | 23 | """ Discriminator Net model """ 24 | X = tf.placeholder(tf.float32, shape=[None, 784]) 25 | y = tf.placeholder(tf.float32, shape=[None, y_dim]) 26 | 27 | D_W1 = tf.Variable(xavier_init([X_dim + y_dim, h_dim])) 28 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 29 | 30 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 31 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 32 | 33 | theta_D = [D_W1, D_W2, D_b1, D_b2] 34 | 35 | 36 | def discriminator(x, y): 37 | inputs = tf.concat(axis=1, values=[x, y]) 38 | D_h1 = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1) 39 | D_logit = tf.matmul(D_h1, D_W2) + D_b2 40 | D_prob = tf.nn.sigmoid(D_logit) 41 | 42 | return D_prob, D_logit 43 | 44 | 45 | """ Generator Net model """ 46 | Z = tf.placeholder(tf.float32, shape=[None, Z_dim]) 47 | 48 | G_W1 = tf.Variable(xavier_init([Z_dim + y_dim, h_dim])) 49 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | 51 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | theta_G = [G_W1, G_W2, G_b1, G_b2] 55 | 56 | 57 | def generator(z, y): 58 | inputs = tf.concat(axis=1, values=[z, y]) 59 | G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1) 60 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 61 | G_prob = tf.nn.sigmoid(G_log_prob) 62 | 63 | return G_prob 64 | 65 | 66 | def sample_Z(m, n): 67 | return np.random.uniform(-1., 1., size=[m, n]) 68 | 69 | 70 | def plot(samples): 71 | fig = plt.figure(figsize=(4, 4)) 72 | gs = gridspec.GridSpec(4, 4) 73 | gs.update(wspace=0.05, hspace=0.05) 74 | 75 | for i, sample in enumerate(samples): 76 | ax = plt.subplot(gs[i]) 77 | plt.axis('off') 78 | ax.set_xticklabels([]) 79 | ax.set_yticklabels([]) 80 | ax.set_aspect('equal') 81 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 82 | 83 | return fig 84 | 85 | 86 | G_sample = generator(Z, y) 87 | D_real, D_logit_real = discriminator(X, y) 88 | D_fake, D_logit_fake = discriminator(G_sample, y) 89 | 90 | D_loss_real = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_real, labels=tf.ones_like(D_logit_real))) 91 | D_loss_fake = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.zeros_like(D_logit_fake))) 92 | D_loss = D_loss_real + D_loss_fake 93 | G_loss = tf.reduce_mean(tf.nn.sigmoid_cross_entropy_with_logits(logits=D_logit_fake, labels=tf.ones_like(D_logit_fake))) 94 | 95 | D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D) 96 | G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G) 97 | 98 | 99 | sess = tf.Session() 100 | sess.run(tf.global_variables_initializer()) 101 | 102 | if not os.path.exists('out/'): 103 | os.makedirs('out/') 104 | 105 | i = 0 106 | 107 | for it in range(1000000): 108 | if it % 1000 == 0: 109 | n_sample = 16 110 | 111 | Z_sample = sample_Z(n_sample, Z_dim) 112 | y_sample = np.zeros(shape=[n_sample, y_dim]) 113 | y_sample[:, 7] = 1 114 | 115 | samples = sess.run(G_sample, feed_dict={Z: Z_sample, y:y_sample}) 116 | 117 | fig = plot(samples) 118 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 119 | i += 1 120 | plt.close(fig) 121 | 122 | X_mb, y_mb = mnist.train.next_batch(mb_size) 123 | 124 | Z_sample = sample_Z(mb_size, Z_dim) 125 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, Z: Z_sample, y:y_mb}) 126 | _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={Z: Z_sample, y:y_mb}) 127 | 128 | if it % 1000 == 0: 129 | print('Iter: {}'.format(it)) 130 | print('D loss: {:.4}'. format(D_loss_curr)) 131 | print('G_loss: {:.4}'.format(G_loss_curr)) 132 | print() 133 | -------------------------------------------------------------------------------- /GAN/auxiliary_classifier_gan/ac_gan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 32 15 | z_dim = 16 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | cnt = 0 20 | lr = 1e-3 21 | eps = 1e-8 22 | 23 | 24 | G_ = torch.nn.Sequential( 25 | torch.nn.Linear(z_dim + y_dim, h_dim), 26 | torch.nn.PReLU(), 27 | torch.nn.Linear(h_dim, X_dim), 28 | torch.nn.Sigmoid() 29 | ) 30 | 31 | 32 | def G(z, c): 33 | inputs = torch.cat([z, c], 1) 34 | return G_(inputs) 35 | 36 | 37 | D_shared = torch.nn.Sequential( 38 | torch.nn.Linear(X_dim, h_dim), 39 | torch.nn.PReLU() 40 | ) 41 | 42 | D_gan = torch.nn.Sequential( 43 | torch.nn.Linear(h_dim, 1), 44 | torch.nn.Sigmoid() 45 | ) 46 | 47 | D_aux = torch.nn.Sequential( 48 | torch.nn.Linear(h_dim, y_dim), 49 | torch.nn.Softmax() 50 | ) 51 | 52 | 53 | def D(X): 54 | h = D_shared(X) 55 | return D_gan(h), D_aux(h) 56 | 57 | 58 | nets = [G_, D_shared, D_gan, D_aux] 59 | 60 | G_params = G_.parameters() 61 | D_params = (list(D_shared.parameters()) + list(D_gan.parameters()) + 62 | list(D_aux.parameters())) 63 | 64 | 65 | def reset_grad(): 66 | for net in nets: 67 | net.zero_grad() 68 | 69 | 70 | G_solver = optim.Adam(G_params, lr=lr) 71 | D_solver = optim.Adam(D_params, lr=lr) 72 | 73 | 74 | for it in range(100000): 75 | # Sample data 76 | X, y = mnist.train.next_batch(mb_size) 77 | X = Variable(torch.from_numpy(X)) 78 | # c is one-hot 79 | c = Variable(torch.from_numpy(y.astype('float32'))) 80 | # y_true is not one-hot (requirement from nn.cross_entropy) 81 | y_true = Variable(torch.from_numpy(y.argmax(axis=1).astype('int'))) 82 | # z noise 83 | z = Variable(torch.randn(mb_size, z_dim)) 84 | 85 | """ Discriminator """ 86 | G_sample = G(z, c) 87 | D_real, C_real = D(X) 88 | D_fake, C_fake = D(G_sample) 89 | 90 | # GAN's D loss 91 | D_loss = torch.mean(torch.log(D_real + eps) + torch.log(1 - D_fake + eps)) 92 | # Cross entropy aux loss 93 | C_loss = -nn.cross_entropy(C_real, y_true) - nn.cross_entropy(C_fake, y_true) 94 | 95 | # Maximize 96 | DC_loss = -(D_loss + C_loss) 97 | 98 | DC_loss.backward() 99 | D_solver.step() 100 | 101 | reset_grad() 102 | 103 | """ Generator """ 104 | G_sample = G(z, c) 105 | D_fake, C_fake = D(G_sample) 106 | _, C_real = D(X) 107 | 108 | # GAN's G loss 109 | G_loss = torch.mean(torch.log(D_fake + eps)) 110 | # Cross entropy aux loss 111 | C_loss = -nn.cross_entropy(C_real, y_true) - nn.cross_entropy(C_fake, y_true) 112 | 113 | # Maximize 114 | GC_loss = -(G_loss + C_loss) 115 | 116 | GC_loss.backward() 117 | G_solver.step() 118 | 119 | reset_grad() 120 | 121 | # Print and plot every now and then 122 | if it % 1000 == 0: 123 | idx = np.random.randint(0, 10) 124 | c = np.zeros([16, y_dim]) 125 | c[range(16), idx] = 1 126 | c = Variable(torch.from_numpy(c.astype('float32'))) 127 | 128 | z = Variable(torch.randn(16, z_dim)) 129 | 130 | samples = G(z, c).data.numpy() 131 | 132 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}; Idx: {}' 133 | .format(it, -D_loss.data[0], -G_loss.data[0], idx)) 134 | 135 | fig = plt.figure(figsize=(4, 4)) 136 | gs = gridspec.GridSpec(4, 4) 137 | gs.update(wspace=0.05, hspace=0.05) 138 | 139 | for i, sample in enumerate(samples): 140 | ax = plt.subplot(gs[i]) 141 | plt.axis('off') 142 | ax.set_xticklabels([]) 143 | ax.set_yticklabels([]) 144 | ax.set_aspect('equal') 145 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 146 | 147 | if not os.path.exists('out/'): 148 | os.makedirs('out/') 149 | 150 | plt.savefig('out/{}.png' 151 | .format(str(cnt).zfill(3)), bbox_inches='tight') 152 | cnt += 1 153 | plt.close(fig) 154 | -------------------------------------------------------------------------------- /GAN/f_gan/f_gan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 1e-3 14 | d_steps = 3 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 42 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 43 | 44 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 45 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 46 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 47 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 48 | 49 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 50 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 51 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | theta_G = [G_W1, G_W2, G_b1, G_b2] 55 | theta_D = [D_W1, D_W2, D_b1, D_b2] 56 | 57 | 58 | def sample_z(m, n): 59 | return np.random.uniform(-1., 1., size=[m, n]) 60 | 61 | 62 | def generator(z): 63 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 64 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 65 | G_prob = tf.nn.sigmoid(G_log_prob) 66 | return G_prob 67 | 68 | 69 | def discriminator(x): 70 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 71 | out = tf.matmul(D_h1, D_W2) + D_b2 72 | return out 73 | 74 | 75 | G_sample = generator(z) 76 | 77 | D_real = discriminator(X) 78 | D_fake = discriminator(G_sample) 79 | 80 | # Uncomment D_loss and its respective G_loss of your choice 81 | # --------------------------------------------------------- 82 | 83 | """ Total Variation """ 84 | # D_loss = -(tf.reduce_mean(0.5 * tf.nn.tanh(D_real)) - 85 | # tf.reduce_mean(0.5 * tf.nn.tanh(D_fake))) 86 | # G_loss = -tf.reduce_mean(0.5 * tf.nn.tanh(D_fake)) 87 | 88 | """ Forward KL """ 89 | # D_loss = -(tf.reduce_mean(D_real) - tf.reduce_mean(tf.exp(D_fake - 1))) 90 | # G_loss = -tf.reduce_mean(tf.exp(D_fake - 1)) 91 | 92 | """ Reverse KL """ 93 | # D_loss = -(tf.reduce_mean(-tf.exp(D_real)) - tf.reduce_mean(-1 - D_fake)) 94 | # G_loss = -tf.reduce_mean(-1 - D_fake) 95 | 96 | """ Pearson Chi-squared """ 97 | D_loss = -(tf.reduce_mean(D_real) - tf.reduce_mean(0.25*D_fake**2 + D_fake)) 98 | G_loss = -tf.reduce_mean(0.25*D_fake**2 + D_fake) 99 | 100 | """ Squared Hellinger """ 101 | # D_loss = -(tf.reduce_mean(1 - tf.exp(D_real)) - 102 | # tf.reduce_mean((1 - tf.exp(D_fake)) / (tf.exp(D_fake)))) 103 | # G_loss = -tf.reduce_mean((1 - tf.exp(D_fake)) / (tf.exp(D_fake))) 104 | 105 | 106 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 107 | .minimize(D_loss, var_list=theta_D)) 108 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 109 | .minimize(G_loss, var_list=theta_G)) 110 | 111 | sess = tf.Session() 112 | sess.run(tf.global_variables_initializer()) 113 | 114 | if not os.path.exists('out/'): 115 | os.makedirs('out/') 116 | 117 | i = 0 118 | 119 | for it in range(1000000): 120 | X_mb, _ = mnist.train.next_batch(mb_size) 121 | z_mb = sample_z(mb_size, z_dim) 122 | 123 | _, D_loss_curr = sess.run([D_solver, D_loss], feed_dict={X: X_mb, z: z_mb}) 124 | _, G_loss_curr = sess.run([G_solver, G_loss], feed_dict={z: z_mb}) 125 | 126 | if it % 1000 == 0: 127 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 128 | .format(it, D_loss_curr, G_loss_curr)) 129 | 130 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 131 | 132 | fig = plot(samples) 133 | plt.savefig('out/{}.png' 134 | .format(str(i).zfill(3)), bbox_inches='tight') 135 | i += 1 136 | plt.close(fig) 137 | -------------------------------------------------------------------------------- /GAN/conditional_gan/cgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 64 15 | Z_dim = 100 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | cnt = 0 20 | lr = 1e-3 21 | 22 | 23 | def xavier_init(size): 24 | in_dim = size[0] 25 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 26 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 27 | 28 | 29 | """ ==================== GENERATOR ======================== """ 30 | 31 | Wzh = xavier_init(size=[Z_dim + y_dim, h_dim]) 32 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whx = xavier_init(size=[h_dim, X_dim]) 35 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 36 | 37 | 38 | def G(z, c): 39 | inputs = torch.cat([z, c], 1) 40 | h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1)) 41 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 42 | return X 43 | 44 | 45 | """ ==================== DISCRIMINATOR ======================== """ 46 | 47 | Wxh = xavier_init(size=[X_dim + y_dim, h_dim]) 48 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 49 | 50 | Why = xavier_init(size=[h_dim, 1]) 51 | bhy = Variable(torch.zeros(1), requires_grad=True) 52 | 53 | 54 | def D(X, c): 55 | inputs = torch.cat([X, c], 1) 56 | h = nn.relu(inputs @ Wxh + bxh.repeat(inputs.size(0), 1)) 57 | y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1)) 58 | return y 59 | 60 | 61 | G_params = [Wzh, bzh, Whx, bhx] 62 | D_params = [Wxh, bxh, Why, bhy] 63 | params = G_params + D_params 64 | 65 | 66 | """ ===================== TRAINING ======================== """ 67 | 68 | 69 | def reset_grad(): 70 | for p in params: 71 | p.grad.data.zero_() 72 | 73 | 74 | G_solver = optim.Adam(G_params, lr=1e-3) 75 | D_solver = optim.Adam(D_params, lr=1e-3) 76 | 77 | ones_label = Variable(torch.ones(mb_size)) 78 | zeros_label = Variable(torch.zeros(mb_size)) 79 | 80 | 81 | for it in range(100000): 82 | # Sample data 83 | z = Variable(torch.randn(mb_size, Z_dim)) 84 | X, c = mnist.train.next_batch(mb_size) 85 | X = Variable(torch.from_numpy(X)) 86 | c = Variable(torch.from_numpy(c.astype('float32'))) 87 | 88 | # Dicriminator forward-loss-backward-update 89 | G_sample = G(z, c) 90 | D_real = D(X, c) 91 | D_fake = D(G_sample, c) 92 | 93 | D_loss_real = nn.binary_cross_entropy(D_real, ones_label) 94 | D_loss_fake = nn.binary_cross_entropy(D_fake, zeros_label) 95 | D_loss = D_loss_real + D_loss_fake 96 | 97 | D_loss.backward() 98 | D_solver.step() 99 | 100 | # Housekeeping - reset gradient 101 | reset_grad() 102 | 103 | # Generator forward-loss-backward-update 104 | z = Variable(torch.randn(mb_size, Z_dim)) 105 | G_sample = G(z, c) 106 | D_fake = D(G_sample, c) 107 | 108 | G_loss = nn.binary_cross_entropy(D_fake, ones_label) 109 | 110 | G_loss.backward() 111 | G_solver.step() 112 | 113 | # Housekeeping - reset gradient 114 | reset_grad() 115 | 116 | # Print and plot every now and then 117 | if it % 1000 == 0: 118 | print('Iter-{}; D_loss: {}; G_loss: {}'.format(it, D_loss.data.numpy(), G_loss.data.numpy())) 119 | 120 | c = np.zeros(shape=[mb_size, y_dim], dtype='float32') 121 | c[:, np.random.randint(0, 10)] = 1. 122 | c = Variable(torch.from_numpy(c)) 123 | samples = G(z, c).data.numpy()[:16] 124 | 125 | fig = plt.figure(figsize=(4, 4)) 126 | gs = gridspec.GridSpec(4, 4) 127 | gs.update(wspace=0.05, hspace=0.05) 128 | 129 | for i, sample in enumerate(samples): 130 | ax = plt.subplot(gs[i]) 131 | plt.axis('off') 132 | ax.set_xticklabels([]) 133 | ax.set_yticklabels([]) 134 | ax.set_aspect('equal') 135 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 136 | 137 | if not os.path.exists('out/'): 138 | os.makedirs('out/') 139 | 140 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 141 | cnt += 1 142 | plt.close(fig) 143 | -------------------------------------------------------------------------------- /VAE/adversarial_vb/avb_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import matplotlib.pyplot as plt 4 | import matplotlib.gridspec as gridspec 5 | import os 6 | from torch.autograd import Variable 7 | from tensorflow.examples.tutorials.mnist import input_data 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | mb_size = 32 12 | z_dim = 10 13 | eps_dim = 4 14 | X_dim = mnist.train.images.shape[1] 15 | y_dim = mnist.train.labels.shape[1] 16 | h_dim = 128 17 | c = 0 18 | lr = 1e-3 19 | 20 | 21 | def log(x): 22 | return tf.log(x + 1e-8) 23 | 24 | 25 | def plot(samples): 26 | fig = plt.figure(figsize=(4, 4)) 27 | gs = gridspec.GridSpec(4, 4) 28 | gs.update(wspace=0.05, hspace=0.05) 29 | 30 | for i, sample in enumerate(samples): 31 | ax = plt.subplot(gs[i]) 32 | plt.axis('off') 33 | ax.set_xticklabels([]) 34 | ax.set_yticklabels([]) 35 | ax.set_aspect('equal') 36 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 37 | 38 | return fig 39 | 40 | 41 | def xavier_init(size): 42 | in_dim = size[0] 43 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 44 | return tf.random_normal(shape=size, stddev=xavier_stddev) 45 | 46 | 47 | """ Q(z|X,eps) """ 48 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 49 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 50 | eps = tf.placeholder(tf.float32, shape=[None, eps_dim]) 51 | 52 | Q_W1 = tf.Variable(xavier_init([X_dim + eps_dim, h_dim])) 53 | Q_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 54 | Q_W2 = tf.Variable(xavier_init([h_dim, z_dim])) 55 | Q_b2 = tf.Variable(tf.zeros(shape=[z_dim])) 56 | 57 | theta_Q = [Q_W1, Q_W2, Q_b1, Q_b2] 58 | 59 | 60 | def Q(X, eps): 61 | inputs = tf.concat(axis=1, values=[X, eps]) 62 | h = tf.nn.relu(tf.matmul(inputs, Q_W1) + Q_b1) 63 | z = tf.matmul(h, Q_W2) + Q_b2 64 | return z 65 | 66 | 67 | """ P(X|z) """ 68 | P_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 69 | P_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 70 | P_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 71 | P_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 72 | 73 | theta_P = [P_W1, P_W2, P_b1, P_b2] 74 | 75 | 76 | def P(z): 77 | h = tf.nn.relu(tf.matmul(z, P_W1) + P_b1) 78 | logits = tf.matmul(h, P_W2) + P_b2 79 | prob = tf.nn.sigmoid(logits) 80 | return prob, logits 81 | 82 | 83 | """ D(z) """ 84 | D_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim])) 85 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 86 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 87 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 88 | 89 | theta_D = [D_W1, D_W2, D_b1, D_b2] 90 | 91 | 92 | def D(X, z): 93 | inputs = tf.concat([X, z], axis=1) 94 | h = tf.nn.relu(tf.matmul(inputs, D_W1) + D_b1) 95 | return tf.matmul(h, D_W2) + D_b2 96 | 97 | 98 | """ Training """ 99 | z_sample = Q(X, eps) 100 | _, X_logits = P(z_sample) 101 | D_sample = D(X, z_sample) 102 | 103 | D_q = tf.nn.sigmoid(D(X, z_sample)) 104 | D_prior = tf.nn.sigmoid(D(X, z)) 105 | 106 | # Sample from random z 107 | X_samples, _ = P(z) 108 | 109 | disc = tf.reduce_mean(-D_sample) 110 | nll = tf.reduce_sum( 111 | tf.nn.sigmoid_cross_entropy_with_logits(logits=X_logits, labels=X), 112 | axis=1 113 | ) 114 | loglike = -tf.reduce_mean(nll) 115 | 116 | elbo = disc + loglike 117 | D_loss = tf.reduce_mean(log(D_q) + log(1. - D_prior)) 118 | 119 | VAE_solver = tf.train.AdamOptimizer().minimize(-elbo, var_list=theta_P+theta_Q) 120 | D_solver = tf.train.AdamOptimizer().minimize(-D_loss, var_list=theta_D) 121 | 122 | sess = tf.Session() 123 | sess.run(tf.global_variables_initializer()) 124 | 125 | if not os.path.exists('out/'): 126 | os.makedirs('out/') 127 | 128 | i = 0 129 | 130 | for it in range(1000000): 131 | X_mb, _ = mnist.train.next_batch(mb_size) 132 | eps_mb = np.random.randn(mb_size, eps_dim) 133 | z_mb = np.random.randn(mb_size, z_dim) 134 | 135 | _, elbo_curr = sess.run([VAE_solver, elbo], 136 | feed_dict={X: X_mb, eps: eps_mb, z: z_mb}) 137 | 138 | _, D_loss_curr = sess.run([D_solver, D_loss], 139 | feed_dict={X: X_mb, eps: eps_mb, z: z_mb}) 140 | 141 | if it % 1000 == 0: 142 | print('Iter: {}; ELBO: {:.4}; D_Loss: {:.4}' 143 | .format(it, elbo_curr, D_loss_curr)) 144 | 145 | samples = sess.run(X_samples, feed_dict={z: np.random.randn(16, z_dim)}) 146 | 147 | fig = plot(samples) 148 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 149 | i += 1 150 | plt.close(fig) 151 | -------------------------------------------------------------------------------- /GAN/mode_regularized_gan/mode_reg_gan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 10 12 | h_dim = 128 13 | lam1 = 1e-2 14 | lam2 = 1e-2 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | 18 | 19 | def plot(samples): 20 | fig = plt.figure(figsize=(4, 4)) 21 | gs = gridspec.GridSpec(4, 4) 22 | gs.update(wspace=0.05, hspace=0.05) 23 | 24 | for i, sample in enumerate(samples): 25 | ax = plt.subplot(gs[i]) 26 | plt.axis('off') 27 | ax.set_xticklabels([]) 28 | ax.set_yticklabels([]) 29 | ax.set_aspect('equal') 30 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 31 | 32 | return fig 33 | 34 | 35 | def xavier_init(size): 36 | in_dim = size[0] 37 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 38 | return tf.random_normal(shape=size, stddev=xavier_stddev) 39 | 40 | 41 | def log(x): 42 | return tf.log(x + 1e-8) 43 | 44 | 45 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | E_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 49 | E_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | E_W2 = tf.Variable(xavier_init([h_dim, z_dim])) 51 | E_b2 = tf.Variable(tf.zeros(shape=[z_dim])) 52 | 53 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 54 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 55 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 56 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 57 | 58 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 59 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 60 | 61 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 62 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 63 | 64 | theta_E = [E_W1, E_W2, E_b1, E_b2] 65 | theta_G = [G_W1, G_W2, G_b1, G_b2] 66 | theta_D = [D_W1, D_W2, D_b1, D_b2] 67 | 68 | 69 | def sample_z(m, n): 70 | return np.random.uniform(-1., 1., size=[m, n]) 71 | 72 | 73 | def encoder(x): 74 | E_h1 = tf.nn.relu(tf.matmul(x, E_W1) + E_b1) 75 | out = tf.matmul(E_h1, E_W2) + E_b2 76 | return out 77 | 78 | 79 | def generator(z): 80 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 81 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 82 | G_prob = tf.nn.sigmoid(G_log_prob) 83 | return G_prob 84 | 85 | 86 | def discriminator(x): 87 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 88 | D_log_prob = tf.matmul(D_h1, D_W2) + D_b2 89 | D_prob = tf.nn.sigmoid(D_log_prob) 90 | return D_prob 91 | 92 | 93 | G_sample = generator(z) 94 | G_sample_reg = generator(encoder(X)) 95 | 96 | D_real = discriminator(X) 97 | D_fake = discriminator(G_sample) 98 | D_reg = discriminator(G_sample_reg) 99 | 100 | mse = tf.reduce_sum((X - G_sample_reg)**2, 1) 101 | 102 | D_loss = -tf.reduce_mean(log(D_real) + log(1 - D_fake)) 103 | E_loss = tf.reduce_mean(lam1 * mse + lam2 * log(D_reg)) 104 | G_loss = -tf.reduce_mean(log(D_fake)) + E_loss 105 | 106 | E_solver = (tf.train.AdamOptimizer(learning_rate=1e-3) 107 | .minimize(E_loss, var_list=theta_E)) 108 | D_solver = (tf.train.AdamOptimizer(learning_rate=1e-3) 109 | .minimize(D_loss, var_list=theta_D)) 110 | G_solver = (tf.train.AdamOptimizer(learning_rate=1e-3) 111 | .minimize(G_loss, var_list=theta_G)) 112 | 113 | sess = tf.Session() 114 | sess.run(tf.global_variables_initializer()) 115 | 116 | if not os.path.exists('out/'): 117 | os.makedirs('out/') 118 | 119 | i = 0 120 | 121 | for it in range(1000000): 122 | X_mb, _ = mnist.train.next_batch(mb_size) 123 | 124 | _, D_loss_curr = sess.run( 125 | [D_solver, D_loss], 126 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 127 | ) 128 | 129 | _, G_loss_curr = sess.run( 130 | [G_solver, G_loss], 131 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 132 | ) 133 | 134 | _, E_loss_curr = sess.run( 135 | [E_solver, E_loss], 136 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim)} 137 | ) 138 | 139 | if it % 1000 == 0: 140 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}; E_loss: {:.4}' 141 | .format(it, D_loss_curr, G_loss_curr, E_loss_curr)) 142 | 143 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 144 | 145 | fig = plot(samples) 146 | plt.savefig('out/{}.png' 147 | .format(str(i).zfill(3)), bbox_inches='tight') 148 | i += 1 149 | plt.close(fig) 150 | -------------------------------------------------------------------------------- /GAN/auxiliary_classifier_gan/ac_gan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 10 | 11 | mb_size = 32 12 | X_dim = mnist.train.images.shape[1] 13 | y_dim = mnist.train.labels.shape[1] 14 | z_dim = 10 15 | h_dim = 128 16 | eps = 1e-8 17 | lr = 1e-3 18 | d_steps = 3 19 | 20 | 21 | def plot(samples): 22 | fig = plt.figure(figsize=(4, 4)) 23 | gs = gridspec.GridSpec(4, 4) 24 | gs.update(wspace=0.05, hspace=0.05) 25 | 26 | for i, sample in enumerate(samples): 27 | ax = plt.subplot(gs[i]) 28 | plt.axis('off') 29 | ax.set_xticklabels([]) 30 | ax.set_yticklabels([]) 31 | ax.set_aspect('equal') 32 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 33 | 34 | return fig 35 | 36 | 37 | def xavier_init(size): 38 | in_dim = size[0] 39 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 40 | return tf.random_normal(shape=size, stddev=xavier_stddev) 41 | 42 | 43 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 44 | y = tf.placeholder(tf.float32, shape=[None, y_dim]) 45 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 46 | 47 | G_W1 = tf.Variable(xavier_init([z_dim + y_dim, h_dim])) 48 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 49 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 50 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 51 | 52 | 53 | def generator(z, c): 54 | inputs = tf.concat(axis=1, values=[z, c]) 55 | G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1) 56 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 57 | G_prob = tf.nn.sigmoid(G_log_prob) 58 | return G_prob 59 | 60 | 61 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 62 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 63 | D_W2_gan = tf.Variable(xavier_init([h_dim, 1])) 64 | D_b2_gan = tf.Variable(tf.zeros(shape=[1])) 65 | D_W2_aux = tf.Variable(xavier_init([h_dim, y_dim])) 66 | D_b2_aux = tf.Variable(tf.zeros(shape=[y_dim])) 67 | 68 | 69 | def discriminator(X): 70 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 71 | out_gan = tf.nn.sigmoid(tf.matmul(D_h1, D_W2_gan) + D_b2_gan) 72 | out_aux = tf.matmul(D_h1, D_W2_aux) + D_b2_aux 73 | return out_gan, out_aux 74 | 75 | 76 | theta_G = [G_W1, G_W2, G_b1, G_b2] 77 | theta_D = [D_W1, D_W2_gan, D_W2_aux, D_b1, D_b2_gan, D_b2_aux] 78 | 79 | 80 | def sample_z(m, n): 81 | return np.random.uniform(-1., 1., size=[m, n]) 82 | 83 | 84 | def cross_entropy(logit, y): 85 | return -tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(logits=logit, labels=y)) 86 | 87 | 88 | G_sample = generator(z, y) 89 | 90 | D_real, C_real = discriminator(X) 91 | D_fake, C_fake = discriminator(G_sample) 92 | 93 | # Cross entropy aux loss 94 | C_loss = cross_entropy(C_real, y) + cross_entropy(C_fake, y) 95 | 96 | # GAN D loss 97 | D_loss = tf.reduce_mean(tf.log(D_real + eps) + tf.log(1. - D_fake + eps)) 98 | DC_loss = -(D_loss + C_loss) 99 | 100 | # GAN's G loss 101 | G_loss = tf.reduce_mean(tf.log(D_fake + eps)) 102 | GC_loss = -(G_loss + C_loss) 103 | 104 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 105 | .minimize(DC_loss, var_list=theta_D)) 106 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 107 | .minimize(GC_loss, var_list=theta_G)) 108 | 109 | 110 | sess = tf.Session() 111 | sess.run(tf.global_variables_initializer()) 112 | 113 | if not os.path.exists('out/'): 114 | os.makedirs('out/') 115 | 116 | i = 0 117 | 118 | for it in range(1000000): 119 | X_mb, y_mb = mnist.train.next_batch(mb_size) 120 | z_mb = sample_z(mb_size, z_dim) 121 | 122 | _, DC_loss_curr = sess.run( 123 | [D_solver, DC_loss], 124 | feed_dict={X: X_mb, y: y_mb, z: z_mb} 125 | ) 126 | 127 | _, GC_loss_curr = sess.run( 128 | [G_solver, GC_loss], 129 | feed_dict={X: X_mb, y: y_mb, z: z_mb} 130 | ) 131 | 132 | if it % 1000 == 0: 133 | idx = np.random.randint(0, 10) 134 | c = np.zeros([16, y_dim]) 135 | c[range(16), idx] = 1 136 | 137 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim), y: c}) 138 | 139 | print('Iter: {}; DC_loss: {:.4}; GC_loss: {:.4}; Idx; {}' 140 | .format(it, DC_loss_curr, GC_loss_curr, idx)) 141 | 142 | fig = plot(samples) 143 | plt.savefig('out/{}.png' 144 | .format(str(i).zfill(3)), bbox_inches='tight') 145 | i += 1 146 | plt.close(fig) 147 | -------------------------------------------------------------------------------- /GAN/infogan/infogan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | def xavier_init(size): 10 | in_dim = size[0] 11 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 12 | return tf.random_normal(shape=size, stddev=xavier_stddev) 13 | 14 | 15 | X = tf.placeholder(tf.float32, shape=[None, 784]) 16 | 17 | D_W1 = tf.Variable(xavier_init([784, 128])) 18 | D_b1 = tf.Variable(tf.zeros(shape=[128])) 19 | 20 | D_W2 = tf.Variable(xavier_init([128, 1])) 21 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 22 | 23 | theta_D = [D_W1, D_W2, D_b1, D_b2] 24 | 25 | 26 | Z = tf.placeholder(tf.float32, shape=[None, 16]) 27 | c = tf.placeholder(tf.float32, shape=[None, 10]) 28 | 29 | G_W1 = tf.Variable(xavier_init([26, 256])) 30 | G_b1 = tf.Variable(tf.zeros(shape=[256])) 31 | 32 | G_W2 = tf.Variable(xavier_init([256, 784])) 33 | G_b2 = tf.Variable(tf.zeros(shape=[784])) 34 | 35 | theta_G = [G_W1, G_W2, G_b1, G_b2] 36 | 37 | 38 | Q_W1 = tf.Variable(xavier_init([784, 128])) 39 | Q_b1 = tf.Variable(tf.zeros(shape=[128])) 40 | 41 | Q_W2 = tf.Variable(xavier_init([128, 10])) 42 | Q_b2 = tf.Variable(tf.zeros(shape=[10])) 43 | 44 | theta_Q = [Q_W1, Q_W2, Q_b1, Q_b2] 45 | 46 | 47 | def sample_Z(m, n): 48 | return np.random.uniform(-1., 1., size=[m, n]) 49 | 50 | 51 | def sample_c(m): 52 | return np.random.multinomial(1, 10*[0.1], size=m) 53 | 54 | 55 | def generator(z, c): 56 | inputs = tf.concat(axis=1, values=[z, c]) 57 | G_h1 = tf.nn.relu(tf.matmul(inputs, G_W1) + G_b1) 58 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 59 | G_prob = tf.nn.sigmoid(G_log_prob) 60 | 61 | return G_prob 62 | 63 | 64 | def discriminator(x): 65 | D_h1 = tf.nn.relu(tf.matmul(x, D_W1) + D_b1) 66 | D_logit = tf.matmul(D_h1, D_W2) + D_b2 67 | D_prob = tf.nn.sigmoid(D_logit) 68 | 69 | return D_prob 70 | 71 | 72 | def Q(x): 73 | Q_h1 = tf.nn.relu(tf.matmul(x, Q_W1) + Q_b1) 74 | Q_prob = tf.nn.softmax(tf.matmul(Q_h1, Q_W2) + Q_b2) 75 | 76 | return Q_prob 77 | 78 | 79 | def plot(samples): 80 | fig = plt.figure(figsize=(4, 4)) 81 | gs = gridspec.GridSpec(4, 4) 82 | gs.update(wspace=0.05, hspace=0.05) 83 | 84 | for i, sample in enumerate(samples): 85 | ax = plt.subplot(gs[i]) 86 | plt.axis('off') 87 | ax.set_xticklabels([]) 88 | ax.set_yticklabels([]) 89 | ax.set_aspect('equal') 90 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 91 | 92 | return fig 93 | 94 | 95 | G_sample = generator(Z, c) 96 | D_real = discriminator(X) 97 | D_fake = discriminator(G_sample) 98 | Q_c_given_x = Q(G_sample) 99 | 100 | D_loss = -tf.reduce_mean(tf.log(D_real + 1e-8) + tf.log(1 - D_fake + 1e-8)) 101 | G_loss = -tf.reduce_mean(tf.log(D_fake + 1e-8)) 102 | 103 | cross_ent = tf.reduce_mean(-tf.reduce_sum(tf.log(Q_c_given_x + 1e-8) * c, 1)) 104 | ent = tf.reduce_mean(-tf.reduce_sum(tf.log(c + 1e-8) * c, 1)) 105 | Q_loss = cross_ent + ent 106 | 107 | D_solver = tf.train.AdamOptimizer().minimize(D_loss, var_list=theta_D) 108 | G_solver = tf.train.AdamOptimizer().minimize(G_loss, var_list=theta_G) 109 | Q_solver = tf.train.AdamOptimizer().minimize(Q_loss, var_list=theta_G + theta_Q) 110 | 111 | mb_size = 32 112 | Z_dim = 16 113 | 114 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 115 | 116 | sess = tf.Session() 117 | sess.run(tf.global_variables_initializer()) 118 | 119 | if not os.path.exists('out/'): 120 | os.makedirs('out/') 121 | 122 | i = 0 123 | 124 | for it in range(1000000): 125 | if it % 1000 == 0: 126 | Z_noise = sample_Z(16, Z_dim) 127 | 128 | idx = np.random.randint(0, 10) 129 | c_noise = np.zeros([16, 10]) 130 | c_noise[range(16), idx] = 1 131 | 132 | samples = sess.run(G_sample, 133 | feed_dict={Z: Z_noise, c: c_noise}) 134 | 135 | fig = plot(samples) 136 | plt.savefig('out/{}.png'.format(str(i).zfill(3)), bbox_inches='tight') 137 | i += 1 138 | plt.close(fig) 139 | 140 | X_mb, _ = mnist.train.next_batch(mb_size) 141 | Z_noise = sample_Z(mb_size, Z_dim) 142 | c_noise = sample_c(mb_size) 143 | 144 | _, D_loss_curr = sess.run([D_solver, D_loss], 145 | feed_dict={X: X_mb, Z: Z_noise, c: c_noise}) 146 | 147 | _, G_loss_curr = sess.run([G_solver, G_loss], 148 | feed_dict={Z: Z_noise, c: c_noise}) 149 | 150 | sess.run([Q_solver], feed_dict={Z: Z_noise, c: c_noise}) 151 | 152 | if it % 1000 == 0: 153 | print('Iter: {}'.format(it)) 154 | print('D loss: {:.4}'. format(D_loss_curr)) 155 | print('G_loss: {:.4}'.format(G_loss_curr)) 156 | print() 157 | -------------------------------------------------------------------------------- /GAN/magan/magan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | 8 | 9 | mb_size = 32 10 | X_dim = 784 11 | z_dim = 64 12 | h_dim = 128 13 | lr = 5e-4 14 | n_iter = 1000 15 | n_epoch = 1000 16 | N = n_iter * mb_size # N data per epoch 17 | 18 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 19 | 20 | 21 | def plot(samples): 22 | fig = plt.figure(figsize=(4, 4)) 23 | gs = gridspec.GridSpec(4, 4) 24 | gs.update(wspace=0.05, hspace=0.05) 25 | 26 | for i, sample in enumerate(samples): 27 | ax = plt.subplot(gs[i]) 28 | plt.axis('off') 29 | ax.set_xticklabels([]) 30 | ax.set_yticklabels([]) 31 | ax.set_aspect('equal') 32 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 33 | 34 | return fig 35 | 36 | 37 | def xavier_init(size): 38 | in_dim = size[0] 39 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 40 | return tf.random_normal(shape=size, stddev=xavier_stddev) 41 | 42 | 43 | X = tf.placeholder(tf.float32, shape=[None, X_dim]) 44 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 45 | m = tf.placeholder(tf.float32) 46 | 47 | D_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 48 | D_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 49 | D_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 50 | D_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 51 | 52 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 53 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 54 | G_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 55 | G_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 56 | 57 | theta_G = [G_W1, G_W2, G_b1, G_b2] 58 | theta_D = [D_W1, D_W2, D_b1, D_b2] 59 | 60 | 61 | def sample_z(m, n): 62 | return np.random.uniform(-1., 1., size=[m, n]) 63 | 64 | 65 | def G(z): 66 | G_h1 = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 67 | G_log_prob = tf.matmul(G_h1, G_W2) + G_b2 68 | G_prob = tf.nn.sigmoid(G_log_prob) 69 | return G_prob 70 | 71 | 72 | def D(X): 73 | D_h1 = tf.nn.relu(tf.matmul(X, D_W1) + D_b1) 74 | X_recon = tf.matmul(D_h1, D_W2) + D_b2 75 | return tf.reduce_sum((X - X_recon)**2, 1) 76 | 77 | 78 | G_sample = G(z) 79 | 80 | D_real = D(X) 81 | D_fake = D(G_sample) 82 | 83 | D_recon_loss = tf.reduce_mean(D_real) 84 | D_loss = tf.reduce_mean(D_real + tf.maximum(0., m - D_fake)) 85 | G_loss = tf.reduce_mean(D_fake) 86 | 87 | D_recon_solver = (tf.train.AdamOptimizer(learning_rate=lr) 88 | .minimize(D_recon_loss, var_list=theta_D)) 89 | D_solver = (tf.train.AdamOptimizer(learning_rate=lr) 90 | .minimize(D_loss, var_list=theta_D)) 91 | G_solver = (tf.train.AdamOptimizer(learning_rate=lr) 92 | .minimize(G_loss, var_list=theta_G)) 93 | 94 | sess = tf.Session() 95 | sess.run(tf.global_variables_initializer()) 96 | 97 | if not os.path.exists('out/'): 98 | os.makedirs('out/') 99 | 100 | 101 | # Pretrain 102 | for it in range(2*n_iter): 103 | X_mb, _ = mnist.train.next_batch(mb_size) 104 | 105 | _, D_recon_loss_curr = sess.run( 106 | [D_recon_solver, D_recon_loss], feed_dict={X: X_mb} 107 | ) 108 | 109 | if it % 1000 == 0: 110 | print('Iter-{}; Pretrained D loss: {:.4}'.format(it, D_recon_loss_curr)) 111 | 112 | 113 | i = 0 114 | # Initial margin, expected energy of real data 115 | margin = sess.run(D_recon_loss, feed_dict={X: mnist.train.images}) 116 | s_z_before = np.inf 117 | 118 | # GAN training 119 | for t in range(n_epoch): 120 | s_x, s_z = 0., 0. 121 | 122 | for it in range(n_iter): 123 | X_mb, _ = mnist.train.next_batch(mb_size) 124 | z_mb = sample_z(mb_size, z_dim) 125 | 126 | _, D_loss_curr, D_real_curr = sess.run( 127 | [D_solver, D_loss, D_real], feed_dict={X: X_mb, z: z_mb, m: margin} 128 | ) 129 | 130 | # Update real samples statistics 131 | s_x += np.sum(D_real_curr) 132 | 133 | _, G_loss_curr, D_fake_curr = sess.run( 134 | [G_solver, G_loss, D_fake], 135 | feed_dict={X: X_mb, z: sample_z(mb_size, z_dim), m: margin} 136 | ) 137 | 138 | # Update fake samples statistics 139 | s_z += np.sum(D_fake_curr) 140 | 141 | # Update margin 142 | if (s_x / N < margin) and (s_x < s_z) and (s_z_before < s_z): 143 | margin = s_x / N 144 | 145 | s_z_before = s_z 146 | 147 | # Convergence measure 148 | Ex = s_x / N 149 | Ez = s_z / N 150 | L = Ex + np.abs(Ex - Ez) 151 | 152 | # Visualize 153 | print('Epoch: {}; m: {:.4}, L: {:.4}'.format(t, margin, L)) 154 | 155 | samples = sess.run(G_sample, feed_dict={z: sample_z(16, z_dim)}) 156 | 157 | fig = plot(samples) 158 | plt.savefig('out/{}.png' 159 | .format(str(i).zfill(3)), bbox_inches='tight') 160 | i += 1 161 | plt.close(fig) 162 | -------------------------------------------------------------------------------- /GAN/disco_gan/discogan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | import scipy.ndimage.interpolation 13 | 14 | 15 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 16 | mb_size = 32 17 | z_dim = 10 18 | X_dim = mnist.train.images.shape[1] 19 | y_dim = mnist.train.labels.shape[1] 20 | h_dim = 128 21 | cnt = 0 22 | lr = 1e-3 23 | 24 | 25 | def log(x): 26 | return torch.log(x + 1e-8) 27 | 28 | 29 | def plot(samples): 30 | fig = plt.figure(figsize=(4, 4)) 31 | gs = gridspec.GridSpec(4, 4) 32 | gs.update(wspace=0.05, hspace=0.05) 33 | 34 | for i, sample in enumerate(samples): 35 | ax = plt.subplot(gs[i]) 36 | plt.axis('off') 37 | ax.set_xticklabels([]) 38 | ax.set_yticklabels([]) 39 | ax.set_aspect('equal') 40 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 41 | 42 | return fig 43 | 44 | 45 | G_AB = torch.nn.Sequential( 46 | torch.nn.Linear(X_dim, h_dim), 47 | torch.nn.ReLU(), 48 | torch.nn.Linear(h_dim, X_dim), 49 | torch.nn.Sigmoid() 50 | ) 51 | 52 | G_BA = torch.nn.Sequential( 53 | torch.nn.Linear(X_dim, h_dim), 54 | torch.nn.ReLU(), 55 | torch.nn.Linear(h_dim, X_dim), 56 | torch.nn.Sigmoid() 57 | ) 58 | 59 | D_A = torch.nn.Sequential( 60 | torch.nn.Linear(X_dim, h_dim), 61 | torch.nn.ReLU(), 62 | torch.nn.Linear(h_dim, 1), 63 | torch.nn.Sigmoid() 64 | ) 65 | 66 | D_B = torch.nn.Sequential( 67 | torch.nn.Linear(X_dim, h_dim), 68 | torch.nn.ReLU(), 69 | torch.nn.Linear(h_dim, 1), 70 | torch.nn.Sigmoid() 71 | ) 72 | 73 | nets = [G_AB, G_BA, D_A, D_B] 74 | G_params = list(G_AB.parameters()) + list(G_BA.parameters()) 75 | D_params = list(D_A.parameters()) + list(D_B.parameters()) 76 | 77 | 78 | def reset_grad(): 79 | for net in nets: 80 | net.zero_grad() 81 | 82 | 83 | G_solver = optim.Adam(G_params, lr=lr) 84 | D_solver = optim.Adam(D_params, lr=lr) 85 | 86 | if not os.path.exists('out/'): 87 | os.makedirs('out/') 88 | 89 | # Gather training data: domain1 <- real MNIST img, domain2 <- rotated MNIST img 90 | X_train = mnist.train.images 91 | half = int(X_train.shape[0] / 2) 92 | # Real image 93 | X_train1 = X_train[:half] 94 | # Rotated image 95 | X_train2 = X_train[half:].reshape(-1, 28, 28) 96 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 97 | X_train2 = X_train2.reshape(-1, 28*28) 98 | # Cleanup 99 | del X_train 100 | 101 | 102 | def sample_x(X, size): 103 | start_idx = np.random.randint(0, X.shape[0]-size) 104 | return Variable(torch.from_numpy(X[start_idx:start_idx+size])) 105 | 106 | 107 | # Training 108 | for it in range(1000000): 109 | # Sample data from both domains 110 | X_A = sample_x(X_train1, mb_size) 111 | X_B = sample_x(X_train2, mb_size) 112 | 113 | # Discriminator A 114 | X_BA = G_BA(X_B) 115 | D_A_real = D_A(X_A) 116 | D_A_fake = D_A(X_BA) 117 | 118 | L_D_A = -torch.mean(log(D_A_real) + log(1 - D_A_fake)) 119 | 120 | # Discriminator B 121 | X_AB = G_AB(X_A) 122 | D_B_real = D_B(X_B) 123 | D_B_fake = D_B(X_AB) 124 | 125 | L_D_B = -torch.mean(log(D_B_real) + log(1 - D_B_fake)) 126 | 127 | # Total discriminator loss 128 | D_loss = L_D_A + L_D_B 129 | 130 | D_loss.backward() 131 | D_solver.step() 132 | reset_grad() 133 | 134 | # Generator AB 135 | X_AB = G_AB(X_A) 136 | D_B_fake = D_B(X_AB) 137 | X_ABA = G_BA(X_AB) 138 | 139 | L_adv_B = -torch.mean(log(D_B_fake)) 140 | L_recon_A = torch.mean(torch.sum((X_A - X_ABA)**2, 1)) 141 | L_G_AB = L_adv_B + L_recon_A 142 | 143 | # Generator BA 144 | X_BA = G_BA(X_B) 145 | D_A_fake = D_A(X_BA) 146 | X_BAB = G_AB(X_BA) 147 | 148 | L_adv_A = -torch.mean(log(D_A_fake)) 149 | L_recon_B = torch.mean(torch.sum((X_B - X_BAB)**2, 1)) 150 | L_G_BA = L_adv_A + L_recon_B 151 | 152 | # Total generator loss 153 | G_loss = L_G_AB + L_G_BA 154 | 155 | G_loss.backward() 156 | G_solver.step() 157 | reset_grad() 158 | 159 | # Print and plot every now and then 160 | if it % 1000 == 0: 161 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 162 | .format(it, D_loss.data[0], G_loss.data[0])) 163 | 164 | input_A = sample_x(X_train1, size=4) 165 | input_B = sample_x(X_train2, size=4) 166 | 167 | samples_A = G_BA(input_B).data.numpy() 168 | samples_B = G_AB(input_A).data.numpy() 169 | 170 | input_A = input_A.data.numpy() 171 | input_B = input_B.data.numpy() 172 | 173 | # The resulting image sample would be in 4 rows: 174 | # row 1: real data from domain A, row 2 is its domain B translation 175 | # row 3: real data from domain B, row 4 is its domain A translation 176 | samples = np.vstack([input_A, samples_B, input_B, samples_A]) 177 | 178 | fig = plot(samples) 179 | plt.savefig('out/{}.png' 180 | .format(str(cnt).zfill(3)), bbox_inches='tight') 181 | cnt += 1 182 | plt.close(fig) 183 | -------------------------------------------------------------------------------- /GAN/infogan/infogan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn.functional as nn 3 | import torch.autograd as autograd 4 | import torch.optim as optim 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import matplotlib.gridspec as gridspec 8 | import os 9 | from torch.autograd import Variable 10 | from tensorflow.examples.tutorials.mnist import input_data 11 | 12 | 13 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 14 | mb_size = 32 15 | Z_dim = 16 16 | X_dim = mnist.train.images.shape[1] 17 | y_dim = mnist.train.labels.shape[1] 18 | h_dim = 128 19 | cnt = 0 20 | lr = 1e-3 21 | 22 | 23 | def xavier_init(size): 24 | in_dim = size[0] 25 | xavier_stddev = 1. / np.sqrt(in_dim / 2.) 26 | return Variable(torch.randn(*size) * xavier_stddev, requires_grad=True) 27 | 28 | 29 | """ ==================== GENERATOR ======================== """ 30 | 31 | Wzh = xavier_init(size=[Z_dim + 10, h_dim]) 32 | bzh = Variable(torch.zeros(h_dim), requires_grad=True) 33 | 34 | Whx = xavier_init(size=[h_dim, X_dim]) 35 | bhx = Variable(torch.zeros(X_dim), requires_grad=True) 36 | 37 | 38 | def G(z, c): 39 | inputs = torch.cat([z, c], 1) 40 | h = nn.relu(inputs @ Wzh + bzh.repeat(inputs.size(0), 1)) 41 | X = nn.sigmoid(h @ Whx + bhx.repeat(h.size(0), 1)) 42 | return X 43 | 44 | 45 | """ ==================== DISCRIMINATOR ======================== """ 46 | 47 | Wxh = xavier_init(size=[X_dim, h_dim]) 48 | bxh = Variable(torch.zeros(h_dim), requires_grad=True) 49 | 50 | Why = xavier_init(size=[h_dim, 1]) 51 | bhy = Variable(torch.zeros(1), requires_grad=True) 52 | 53 | 54 | def D(X): 55 | h = nn.relu(X @ Wxh + bxh.repeat(X.size(0), 1)) 56 | y = nn.sigmoid(h @ Why + bhy.repeat(h.size(0), 1)) 57 | return y 58 | 59 | 60 | """ ====================== Q(c|X) ========================== """ 61 | 62 | Wqxh = xavier_init(size=[X_dim, h_dim]) 63 | bqxh = Variable(torch.zeros(h_dim), requires_grad=True) 64 | 65 | Whc = xavier_init(size=[h_dim, 10]) 66 | bhc = Variable(torch.zeros(10), requires_grad=True) 67 | 68 | 69 | def Q(X): 70 | h = nn.relu(X @ Wqxh + bqxh.repeat(X.size(0), 1)) 71 | c = nn.softmax(h @ Whc + bhc.repeat(h.size(0), 1)) 72 | return c 73 | 74 | 75 | G_params = [Wzh, bzh, Whx, bhx] 76 | D_params = [Wxh, bxh, Why, bhy] 77 | Q_params = [Wqxh, bqxh, Whc, bhc] 78 | params = G_params + D_params + Q_params 79 | 80 | 81 | """ ===================== TRAINING ======================== """ 82 | 83 | 84 | def reset_grad(): 85 | for p in params: 86 | p.grad.data.zero_() 87 | 88 | 89 | G_solver = optim.Adam(G_params, lr=1e-3) 90 | D_solver = optim.Adam(D_params, lr=1e-3) 91 | Q_solver = optim.Adam(G_params + Q_params, lr=1e-3) 92 | 93 | 94 | def sample_c(size): 95 | c = np.random.multinomial(1, 10*[0.1], size=size) 96 | c = Variable(torch.from_numpy(c.astype('float32'))) 97 | return c 98 | 99 | 100 | for it in range(100000): 101 | # Sample data 102 | X, _ = mnist.train.next_batch(mb_size) 103 | X = Variable(torch.from_numpy(X)) 104 | 105 | z = Variable(torch.randn(mb_size, Z_dim)) 106 | c = sample_c(mb_size) 107 | 108 | # Dicriminator forward-loss-backward-update 109 | G_sample = G(z, c) 110 | D_real = D(X) 111 | D_fake = D(G_sample) 112 | 113 | D_loss = -torch.mean(torch.log(D_real + 1e-8) + torch.log(1 - D_fake + 1e-8)) 114 | 115 | D_loss.backward() 116 | D_solver.step() 117 | 118 | # Housekeeping - reset gradient 119 | reset_grad() 120 | 121 | # Generator forward-loss-backward-update 122 | G_sample = G(z, c) 123 | D_fake = D(G_sample) 124 | 125 | G_loss = -torch.mean(torch.log(D_fake + 1e-8)) 126 | 127 | G_loss.backward() 128 | G_solver.step() 129 | 130 | # Housekeeping - reset gradient 131 | reset_grad() 132 | 133 | # Q forward-loss-backward-update 134 | G_sample = G(z, c) 135 | Q_c_given_x = Q(G_sample) 136 | 137 | crossent_loss = torch.mean(-torch.sum(c * torch.log(Q_c_given_x + 1e-8), dim=1)) 138 | ent_loss = torch.mean(-torch.sum(c * torch.log(c + 1e-8), dim=1)) 139 | mi_loss = crossent_loss + ent_loss 140 | 141 | mi_loss.backward() 142 | Q_solver.step() 143 | 144 | # Housekeeping - reset gradient 145 | reset_grad() 146 | 147 | # Print and plot every now and then 148 | if it % 1000 == 0: 149 | idx = np.random.randint(0, 10) 150 | c = np.zeros([mb_size, 10]) 151 | c[range(mb_size), idx] = 1 152 | c = Variable(torch.from_numpy(c.astype('float32'))) 153 | samples = G(z, c).data.numpy()[:16] 154 | 155 | print('Iter-{}; D_loss: {}; G_loss: {}; Idx: {}' 156 | .format(it, D_loss.data.numpy(), G_loss.data.numpy(), idx)) 157 | 158 | fig = plt.figure(figsize=(4, 4)) 159 | gs = gridspec.GridSpec(4, 4) 160 | gs.update(wspace=0.05, hspace=0.05) 161 | 162 | for i, sample in enumerate(samples): 163 | ax = plt.subplot(gs[i]) 164 | plt.axis('off') 165 | ax.set_xticklabels([]) 166 | ax.set_yticklabels([]) 167 | ax.set_aspect('equal') 168 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 169 | 170 | if not os.path.exists('out/'): 171 | os.makedirs('out/') 172 | 173 | plt.savefig('out/{}.png' 174 | .format(str(cnt).zfill(3)), bbox_inches='tight') 175 | cnt += 1 176 | plt.close(fig) 177 | -------------------------------------------------------------------------------- /GAN/dual_gan/dualgan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | from itertools import chain 13 | import scipy.ndimage.interpolation 14 | 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | mb_size = 32 18 | z_dim = 10 19 | X_dim = mnist.train.images.shape[1] 20 | y_dim = mnist.train.labels.shape[1] 21 | h_dim = 128 22 | cnt = 0 23 | lr = 1e-4 24 | n_critics = 3 25 | lam1, lam2 = 100, 100 26 | 27 | 28 | def log(x): 29 | return torch.log(x + 1e-8) 30 | 31 | 32 | G1 = torch.nn.Sequential( 33 | torch.nn.Linear(X_dim + z_dim, h_dim), 34 | torch.nn.ReLU(), 35 | torch.nn.Linear(h_dim, X_dim), 36 | torch.nn.Sigmoid() 37 | ) 38 | 39 | G2 = torch.nn.Sequential( 40 | torch.nn.Linear(X_dim + z_dim, h_dim), 41 | torch.nn.ReLU(), 42 | torch.nn.Linear(h_dim, X_dim), 43 | torch.nn.Sigmoid() 44 | ) 45 | 46 | D1 = torch.nn.Sequential( 47 | torch.nn.Linear(X_dim, h_dim), 48 | torch.nn.ReLU(), 49 | torch.nn.Linear(h_dim, 1) 50 | ) 51 | 52 | D2 = torch.nn.Sequential( 53 | torch.nn.Linear(X_dim, h_dim), 54 | torch.nn.ReLU(), 55 | torch.nn.Linear(h_dim, 1) 56 | ) 57 | 58 | 59 | def reset_grad(): 60 | G1.zero_grad() 61 | G2.zero_grad() 62 | D1.zero_grad() 63 | D2.zero_grad() 64 | 65 | 66 | G_solver = optim.RMSprop(chain(G1.parameters(), G2.parameters()), lr=lr) 67 | D1_solver = optim.RMSprop(D1.parameters(), lr=lr) 68 | D2_solver = optim.RMSprop(D2.parameters(), lr=lr) 69 | 70 | X_train = mnist.train.images 71 | half = int(X_train.shape[0] / 2) 72 | 73 | # Real image 74 | X_train1 = X_train[:half] 75 | # Rotated image 76 | X_train2 = X_train[half:].reshape(-1, 28, 28) 77 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 78 | X_train2 = X_train2.reshape(-1, 28*28) 79 | 80 | # Cleanup 81 | del X_train 82 | 83 | 84 | def sample_x(X, size): 85 | start_idx = np.random.randint(0, X.shape[0]-size) 86 | return Variable(torch.from_numpy(X[start_idx:start_idx+size])) 87 | 88 | 89 | for it in range(1000000): 90 | for _ in range(n_critics): 91 | # Sample data 92 | z1 = Variable(torch.randn(mb_size, z_dim)) 93 | z2 = Variable(torch.randn(mb_size, z_dim)) 94 | X1 = sample_x(X_train1, mb_size) 95 | X2 = sample_x(X_train2, mb_size) 96 | 97 | # D1 98 | X2_sample = G1(torch.cat([X1, z1], 1)) # G1: X1 -> X2 99 | D1_real = D1(X2) 100 | D1_fake = D1(X2_sample) 101 | 102 | D1_loss = -(torch.mean(D1_real) - torch.mean(D1_fake)) 103 | 104 | D1_loss.backward(retain_variables=True) 105 | D1_solver.step() 106 | 107 | # Weight clipping 108 | for p in D1.parameters(): 109 | p.data.clamp_(-0.01, 0.01) 110 | 111 | reset_grad() 112 | 113 | # D2 114 | X1_sample = G2(torch.cat([X2, z2], 1)) # G2: X2 -> X1 115 | D2_real = D2(X1) 116 | D2_fake = D2(X1_sample) 117 | 118 | D2_loss = -(torch.mean(D2_real) - torch.mean(D2_fake)) 119 | 120 | D2_loss.backward() 121 | D2_solver.step() 122 | 123 | # Weight clipping 124 | for p in D2.parameters(): 125 | p.data.clamp_(-0.01, 0.01) 126 | 127 | reset_grad() 128 | 129 | # Generator 130 | z1 = Variable(torch.randn(mb_size, z_dim)) 131 | z2 = Variable(torch.randn(mb_size, z_dim)) 132 | X1 = sample_x(X_train1, mb_size) 133 | X2 = sample_x(X_train2, mb_size) 134 | 135 | X1_sample = G2(torch.cat([X2, z2], 1)) 136 | X2_sample = G1(torch.cat([X1, z1], 1)) 137 | 138 | X1_recon = G2(torch.cat([X2_sample, z2], 1)) 139 | X2_recon = G1(torch.cat([X1_sample, z1], 1)) 140 | 141 | D1_fake = D1(X1_sample) 142 | D2_fake = D2(X2_sample) 143 | 144 | G_loss = -torch.mean(D1_fake) - torch.mean(D2_fake) 145 | reg1 = lam1 * torch.mean(torch.sum(torch.abs(X1_recon - X1), 1)) 146 | reg2 = lam2 * torch.mean(torch.sum(torch.abs(X2_recon - X2), 1)) 147 | 148 | G_loss += reg1 + reg2 149 | 150 | G_loss.backward() 151 | G_solver.step() 152 | reset_grad() 153 | 154 | # Print and plot every now and then 155 | if it % 1000 == 0: 156 | print('Iter-{}; D_loss: {:.4}; G_loss: {:.4}' 157 | .format(it, D1_loss.data[0] + D2_loss.data[0], G_loss.data[0])) 158 | 159 | real1 = X1.data.numpy()[:4] 160 | real2 = X2.data.numpy()[:4] 161 | samples1 = X1_sample.data.numpy()[:4] 162 | samples2 = X2_sample.data.numpy()[:4] 163 | samples = np.vstack([real2, samples1, real1, samples2]) 164 | 165 | fig = plt.figure(figsize=(4, 4)) 166 | gs = gridspec.GridSpec(4, 4) 167 | gs.update(wspace=0.05, hspace=0.05) 168 | 169 | for i, sample in enumerate(samples): 170 | ax = plt.subplot(gs[i]) 171 | plt.axis('off') 172 | ax.set_xticklabels([]) 173 | ax.set_yticklabels([]) 174 | ax.set_aspect('equal') 175 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 176 | 177 | if not os.path.exists('out/'): 178 | os.makedirs('out/') 179 | 180 | plt.savefig('out/{}.png'.format(str(cnt).zfill(3)), bbox_inches='tight') 181 | cnt += 1 182 | plt.close(fig) 183 | -------------------------------------------------------------------------------- /GAN/coupled_gan/cogan_pytorch.py: -------------------------------------------------------------------------------- 1 | import torch 2 | import torch.nn 3 | import torch.nn.functional as nn 4 | import torch.autograd as autograd 5 | import torch.optim as optim 6 | import numpy as np 7 | import matplotlib.pyplot as plt 8 | import matplotlib.gridspec as gridspec 9 | import os 10 | from torch.autograd import Variable 11 | from tensorflow.examples.tutorials.mnist import input_data 12 | import copy 13 | import scipy.ndimage.interpolation 14 | 15 | 16 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 17 | mb_size = 32 18 | z_dim = 100 19 | X_dim = mnist.train.images.shape[1] 20 | y_dim = mnist.train.labels.shape[1] 21 | h_dim = 128 22 | cnt = 0 23 | lr = 1e-3 24 | 25 | 26 | """ Shared Generator weights """ 27 | G_shared = torch.nn.Sequential( 28 | torch.nn.Linear(z_dim, h_dim), 29 | torch.nn.ReLU(), 30 | ) 31 | 32 | """ Generator 1 """ 33 | G1_ = torch.nn.Sequential( 34 | torch.nn.Linear(h_dim, X_dim), 35 | torch.nn.Sigmoid() 36 | ) 37 | 38 | """ Generator 2 """ 39 | G2_ = torch.nn.Sequential( 40 | torch.nn.Linear(h_dim, X_dim), 41 | torch.nn.Sigmoid() 42 | ) 43 | 44 | 45 | def G1(z): 46 | h = G_shared(z) 47 | X = G1_(h) 48 | return X 49 | 50 | 51 | def G2(z): 52 | h = G_shared(z) 53 | X = G2_(h) 54 | return X 55 | 56 | 57 | """ Shared Discriminator weights """ 58 | D_shared = torch.nn.Sequential( 59 | torch.nn.Linear(h_dim, 1), 60 | torch.nn.Sigmoid() 61 | ) 62 | 63 | """ Discriminator 1 """ 64 | D1_ = torch.nn.Sequential( 65 | torch.nn.Linear(X_dim, h_dim), 66 | torch.nn.ReLU() 67 | ) 68 | 69 | """ Discriminator 2 """ 70 | D2_ = torch.nn.Sequential( 71 | torch.nn.Linear(X_dim, h_dim), 72 | torch.nn.ReLU() 73 | ) 74 | 75 | 76 | def D1(X): 77 | h = D1_(X) 78 | y = D_shared(h) 79 | return y 80 | 81 | 82 | def D2(X): 83 | h = D2_(X) 84 | y = D_shared(h) 85 | return y 86 | 87 | 88 | D_params = (list(D1_.parameters()) + list(D2_.parameters()) + 89 | list(D_shared.parameters())) 90 | G_params = (list(G1_.parameters()) + list(G2_.parameters()) + 91 | list(G_shared.parameters())) 92 | nets = [G_shared, G1_, G2_, D_shared, D1_, D2_] 93 | 94 | 95 | def reset_grad(): 96 | for net in nets: 97 | net.zero_grad() 98 | 99 | 100 | G_solver = optim.Adam(G_params, lr=lr) 101 | D_solver = optim.Adam(D_params, lr=lr) 102 | 103 | X_train = mnist.train.images 104 | half = int(X_train.shape[0] / 2) 105 | 106 | # Real image 107 | X_train1 = X_train[:half] 108 | # Rotated image 109 | X_train2 = X_train[half:].reshape(-1, 28, 28) 110 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 111 | X_train2 = X_train2.reshape(-1, 28*28) 112 | 113 | # Cleanup 114 | del X_train 115 | 116 | 117 | def sample_x(X, size): 118 | start_idx = np.random.randint(0, X.shape[0]-size) 119 | return Variable(torch.from_numpy(X[start_idx:start_idx+size])) 120 | 121 | 122 | for it in range(100000): 123 | X1 = sample_x(X_train1, mb_size) 124 | X2 = sample_x(X_train2, mb_size) 125 | z = Variable(torch.randn(mb_size, z_dim)) 126 | 127 | # Dicriminator 128 | G1_sample = G1(z) 129 | D1_real = D1(X1) 130 | D1_fake = D1(G1_sample) 131 | 132 | G2_sample = G2(z) 133 | D2_real = D2(X2) 134 | D2_fake = D2(G2_sample) 135 | 136 | D1_loss = torch.mean(-torch.log(D1_real + 1e-8) - 137 | torch.log(1. - D1_fake + 1e-8)) 138 | D2_loss = torch.mean(-torch.log(D2_real + 1e-8) - 139 | torch.log(1. - D2_fake + 1e-8)) 140 | D_loss = D1_loss + D2_loss 141 | 142 | D_loss.backward() 143 | 144 | # Average the gradients 145 | for p in D_shared.parameters(): 146 | p.grad.data = 0.5 * p.grad.data 147 | 148 | D_solver.step() 149 | reset_grad() 150 | 151 | # Generator 152 | G1_sample = G1(z) 153 | D1_fake = D1(G1_sample) 154 | 155 | G2_sample = G2(z) 156 | D2_fake = D2(G2_sample) 157 | 158 | G1_loss = torch.mean(-torch.log(D1_fake + 1e-8)) 159 | G2_loss = torch.mean(-torch.log(D2_fake + 1e-8)) 160 | G_loss = G1_loss + G2_loss 161 | 162 | G_loss.backward() 163 | 164 | # Average the gradients 165 | for p in G_shared.parameters(): 166 | p.grad.data = 0.5 * p.grad.data 167 | 168 | G_solver.step() 169 | reset_grad() 170 | 171 | # Print and plot every now and then 172 | if it % 1000 == 0: 173 | print('Iter-{}; D1_loss: {:.4}; G1_loss: {:.4}; ' 174 | 'D2_loss: {:.4}; G2_loss: {:.4}' 175 | .format( 176 | it, D1_loss.data[0], G1_loss.data[0], 177 | D2_loss.data[0], G2_loss.data[0]) 178 | ) 179 | 180 | z = Variable(torch.randn(8, z_dim)) 181 | samples1 = G1(z).data.numpy() 182 | samples2 = G2(z).data.numpy() 183 | samples = np.vstack([samples1, samples2]) 184 | 185 | fig = plt.figure(figsize=(4, 4)) 186 | gs = gridspec.GridSpec(4, 4) 187 | gs.update(wspace=0.05, hspace=0.05) 188 | 189 | for i, sample in enumerate(samples): 190 | ax = plt.subplot(gs[i]) 191 | plt.axis('off') 192 | ax.set_xticklabels([]) 193 | ax.set_yticklabels([]) 194 | ax.set_aspect('equal') 195 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 196 | 197 | if not os.path.exists('out/'): 198 | os.makedirs('out/') 199 | 200 | plt.savefig('out/{}.png' 201 | .format(str(cnt).zfill(3)), bbox_inches='tight') 202 | cnt += 1 203 | plt.close(fig) 204 | -------------------------------------------------------------------------------- /GAN/coupled_gan/cogan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | import scipy.ndimage.interpolation 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | 12 | mb_size = 32 13 | X_dim = mnist.train.images.shape[1] 14 | y_dim = mnist.train.labels.shape[1] 15 | z_dim = 10 16 | h_dim = 128 17 | eps = 1e-8 18 | lr = 1e-3 19 | d_steps = 3 20 | 21 | 22 | def plot(samples): 23 | fig = plt.figure(figsize=(4, 4)) 24 | gs = gridspec.GridSpec(4, 4) 25 | gs.update(wspace=0.05, hspace=0.05) 26 | 27 | for i, sample in enumerate(samples): 28 | ax = plt.subplot(gs[i]) 29 | plt.axis('off') 30 | ax.set_xticklabels([]) 31 | ax.set_yticklabels([]) 32 | ax.set_aspect('equal') 33 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 34 | 35 | return fig 36 | 37 | 38 | def xavier_init(size): 39 | in_dim = size[0] 40 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 41 | return tf.random_normal(shape=size, stddev=xavier_stddev) 42 | 43 | 44 | X1 = tf.placeholder(tf.float32, shape=[None, X_dim]) 45 | X2 = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 47 | 48 | G_W1 = tf.Variable(xavier_init([z_dim, h_dim])) 49 | G_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 50 | 51 | G1_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G1_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | G2_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 55 | G2_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 56 | 57 | 58 | def G(z): 59 | h = tf.nn.relu(tf.matmul(z, G_W1) + G_b1) 60 | G1 = tf.nn.sigmoid(tf.matmul(h, G1_W2) + G1_b2) 61 | G2 = tf.nn.sigmoid(tf.matmul(h, G2_W2) + G2_b2) 62 | return G1, G2 63 | 64 | 65 | D1_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 66 | D1_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 67 | 68 | D2_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 69 | D2_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 70 | 71 | D_W2 = tf.Variable(xavier_init([h_dim, 1])) 72 | D_b2 = tf.Variable(tf.zeros(shape=[1])) 73 | 74 | 75 | def D(X1, X2): 76 | h1 = tf.nn.relu(tf.matmul(X1, D1_W1) + D1_b1) 77 | h2 = tf.nn.relu(tf.matmul(X2, D2_W1) + D2_b1) 78 | D1_out = tf.nn.sigmoid(tf.matmul(h1, D_W2) + D_b2) 79 | D2_out = tf.nn.sigmoid(tf.matmul(h2, D_W2) + D_b2) 80 | return D1_out, D2_out 81 | 82 | 83 | theta_G = [G1_W2, G2_W2, G1_b2, G2_b2] 84 | theta_G_shared = [G_W1, G_b1] 85 | 86 | theta_D = [D1_W1, D2_W1, D1_b1, D2_b1] 87 | theta_D_shared = [D_W2, D_b2] 88 | 89 | # Train D 90 | G1_sample, G2_sample = G(z) 91 | D1_real, D2_real = D(X1, X2) 92 | D1_fake, D2_fake = D(G1_sample, G2_sample) 93 | 94 | D1_loss = -tf.reduce_mean(tf.log(D1_real + eps) + tf.log(1. - D1_fake + eps)) 95 | D2_loss = -tf.reduce_mean(tf.log(D2_real + eps) + tf.log(1. - D2_fake + eps)) 96 | D_loss = D1_loss + D2_loss 97 | 98 | # Train G 99 | G1_loss = -tf.reduce_mean(tf.log(D1_fake + eps)) 100 | G2_loss = -tf.reduce_mean(tf.log(D2_fake + eps)) 101 | G_loss = G1_loss + G2_loss 102 | 103 | # D optimizer 104 | D_opt = tf.train.AdamOptimizer(learning_rate=lr) 105 | # Compute the gradients for a list of variables. 106 | D_gv = D_opt.compute_gradients(D_loss, theta_D) 107 | D_shared_gv = D_opt.compute_gradients(D_loss, theta_D_shared) 108 | # Average by halfing the shared gradients 109 | D_shared_gv = [(0.5 * x[0], x[1]) for x in D_shared_gv] 110 | # Update 111 | D_solver = tf.group( 112 | D_opt.apply_gradients(D_gv), D_opt.apply_gradients(D_shared_gv) 113 | ) 114 | 115 | # G optimizer 116 | G_opt = tf.train.AdamOptimizer(learning_rate=lr) 117 | # Compute the gradients for a list of variables. 118 | G_gv = G_opt.compute_gradients(G_loss, theta_G) 119 | G_shared_gv = G_opt.compute_gradients(G_loss, theta_G_shared) 120 | # Average by halfing the shared gradients 121 | G_shared_gv = [(0.5 * x[0], x[1]) for x in G_shared_gv] 122 | # Update 123 | G_solver = tf.group( 124 | G_opt.apply_gradients(G_gv), G_opt.apply_gradients(G_shared_gv) 125 | ) 126 | 127 | sess = tf.Session() 128 | sess.run(tf.global_variables_initializer()) 129 | 130 | X_train = mnist.train.images 131 | half = int(X_train.shape[0] / 2) 132 | 133 | # Real image 134 | X_train1 = X_train[:half] 135 | # Rotated image 136 | X_train2 = X_train[half:].reshape(-1, 28, 28) 137 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 138 | X_train2 = X_train2.reshape(-1, 28*28) 139 | 140 | # Cleanup 141 | del X_train 142 | 143 | 144 | def sample_X(X, size): 145 | start_idx = np.random.randint(0, X.shape[0]-size) 146 | return X[start_idx:start_idx+size] 147 | 148 | 149 | def sample_z(m, n): 150 | return np.random.uniform(-1., 1., size=[m, n]) 151 | 152 | 153 | if not os.path.exists('out/'): 154 | os.makedirs('out/') 155 | 156 | i = 0 157 | 158 | for it in range(1000000): 159 | X1_mb, X2_mb = sample_X(X_train1, mb_size), sample_X(X_train2, mb_size) 160 | z_mb = sample_z(mb_size, z_dim) 161 | 162 | _, D_loss_curr = sess.run( 163 | [D_solver, D_loss], 164 | feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb} 165 | ) 166 | 167 | _, G_loss_curr = sess.run( 168 | [G_solver, G_loss], feed_dict={z: z_mb} 169 | ) 170 | 171 | if it % 1000 == 0: 172 | sample1, sample2 = sess.run( 173 | [G1_sample, G2_sample], feed_dict={z: sample_z(8, z_dim)} 174 | ) 175 | 176 | samples = np.vstack([sample1, sample2]) 177 | 178 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 179 | .format(it, D_loss_curr, G_loss_curr)) 180 | 181 | fig = plot(samples) 182 | plt.savefig('out/{}.png' 183 | .format(str(i).zfill(3)), bbox_inches='tight') 184 | i += 1 185 | plt.close(fig) 186 | -------------------------------------------------------------------------------- /GAN/disco_gan/discogan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | import scipy.ndimage.interpolation 8 | 9 | 10 | mb_size = 32 11 | X_dim = 784 12 | z_dim = 64 13 | h_dim = 128 14 | lr = 1e-3 15 | d_steps = 3 16 | 17 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 18 | 19 | 20 | def plot(samples): 21 | fig = plt.figure(figsize=(4, 4)) 22 | gs = gridspec.GridSpec(4, 4) 23 | gs.update(wspace=0.05, hspace=0.05) 24 | 25 | for i, sample in enumerate(samples): 26 | ax = plt.subplot(gs[i]) 27 | plt.axis('off') 28 | ax.set_xticklabels([]) 29 | ax.set_yticklabels([]) 30 | ax.set_aspect('equal') 31 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 32 | 33 | return fig 34 | 35 | 36 | def xavier_init(size): 37 | in_dim = size[0] 38 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 39 | return tf.random_normal(shape=size, stddev=xavier_stddev) 40 | 41 | 42 | def log(x): 43 | return tf.log(x + 1e-8) 44 | 45 | 46 | X_A = tf.placeholder(tf.float32, shape=[None, X_dim]) 47 | X_B = tf.placeholder(tf.float32, shape=[None, X_dim]) 48 | 49 | D_A_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 50 | D_A_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 51 | D_A_W2 = tf.Variable(xavier_init([h_dim, 1])) 52 | D_A_b2 = tf.Variable(tf.zeros(shape=[1])) 53 | 54 | D_B_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 55 | D_B_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 56 | D_B_W2 = tf.Variable(xavier_init([h_dim, 1])) 57 | D_B_b2 = tf.Variable(tf.zeros(shape=[1])) 58 | 59 | G_AB_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 60 | G_AB_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 61 | G_AB_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 62 | G_AB_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 63 | 64 | G_BA_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 65 | G_BA_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 66 | G_BA_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 67 | G_BA_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 68 | 69 | theta_D = [D_A_W1, D_A_W2, D_A_b1, D_A_b2, 70 | D_B_W1, D_B_W2, D_B_b1, D_B_b2] 71 | theta_G = [G_AB_W1, G_AB_W2, G_AB_b1, G_AB_b2, 72 | G_BA_W1, G_BA_W2, G_BA_b1, G_BA_b2] 73 | 74 | 75 | def D_A(X): 76 | h = tf.nn.relu(tf.matmul(X, D_A_W1) + D_A_b1) 77 | return tf.nn.sigmoid(tf.matmul(h, D_A_W2) + D_A_b2) 78 | 79 | 80 | def D_B(X): 81 | h = tf.nn.relu(tf.matmul(X, D_B_W1) + D_B_b1) 82 | return tf.nn.sigmoid(tf.matmul(h, D_B_W2) + D_B_b2) 83 | 84 | 85 | def G_AB(X): 86 | h = tf.nn.relu(tf.matmul(X, G_AB_W1) + G_AB_b1) 87 | return tf.nn.sigmoid(tf.matmul(h, G_AB_W2) + G_AB_b2) 88 | 89 | 90 | def G_BA(X): 91 | h = tf.nn.relu(tf.matmul(X, G_BA_W1) + G_BA_b1) 92 | return tf.nn.sigmoid(tf.matmul(h, G_BA_W2) + G_BA_b2) 93 | 94 | 95 | # Discriminator A 96 | X_BA = G_BA(X_B) 97 | D_A_real = D_A(X_A) 98 | D_A_fake = D_A(X_BA) 99 | 100 | # Discriminator B 101 | X_AB = G_AB(X_A) 102 | D_B_real = D_B(X_B) 103 | D_B_fake = D_B(X_AB) 104 | 105 | # Generator AB 106 | X_ABA = G_BA(X_AB) 107 | 108 | # Generator BA 109 | X_BAB = G_AB(X_BA) 110 | 111 | # Discriminator loss 112 | L_D_A = -tf.reduce_mean(log(D_A_real) + log(1 - D_A_fake)) 113 | L_D_B = -tf.reduce_mean(log(D_B_real) + log(1 - D_B_fake)) 114 | 115 | D_loss = L_D_A + L_D_B 116 | 117 | # Generator loss 118 | L_adv_B = -tf.reduce_mean(log(D_B_fake)) 119 | L_recon_A = tf.reduce_mean(tf.reduce_sum((X_A - X_ABA)**2, 1)) 120 | L_G_AB = L_adv_B + L_recon_A 121 | 122 | L_adv_A = -tf.reduce_mean(log(D_A_fake)) 123 | L_recon_B = tf.reduce_mean(tf.reduce_sum((X_B - X_BAB)**2, 1)) 124 | L_G_BA = L_adv_A + L_recon_B 125 | 126 | G_loss = L_G_AB + L_G_BA 127 | 128 | # Solvers 129 | solver = tf.train.AdamOptimizer(learning_rate=lr) 130 | D_solver = solver.minimize(D_loss, var_list=theta_D) 131 | G_solver = solver.minimize(G_loss, var_list=theta_G) 132 | 133 | sess = tf.Session() 134 | sess.run(tf.global_variables_initializer()) 135 | 136 | 137 | # Gather training data from 2 domains 138 | X_train = mnist.train.images 139 | half = int(X_train.shape[0] / 2) 140 | # Real image 141 | X_train1 = X_train[:half] 142 | # Rotated image 143 | X_train2 = X_train[half:].reshape(-1, 28, 28) 144 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 145 | X_train2 = X_train2.reshape(-1, 28*28) 146 | # Cleanup 147 | del X_train 148 | 149 | 150 | def sample_X(X, size): 151 | start_idx = np.random.randint(0, X.shape[0]-size) 152 | return X[start_idx:start_idx+size] 153 | 154 | 155 | if not os.path.exists('out/'): 156 | os.makedirs('out/') 157 | 158 | i = 0 159 | 160 | for it in range(1000000): 161 | # Sample data from both domains 162 | X_A_mb = sample_X(X_train1, mb_size) 163 | X_B_mb = sample_X(X_train2, mb_size) 164 | 165 | _, D_loss_curr = sess.run( 166 | [D_solver, D_loss], feed_dict={X_A: X_A_mb, X_B: X_B_mb} 167 | ) 168 | 169 | _, G_loss_curr = sess.run( 170 | [G_solver, G_loss], feed_dict={X_A: X_A_mb, X_B: X_B_mb} 171 | ) 172 | 173 | if it % 1000 == 0: 174 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 175 | .format(it, D_loss_curr, G_loss_curr)) 176 | 177 | input_A = sample_X(X_train1, size=4) 178 | input_B = sample_X(X_train2, size=4) 179 | 180 | samples_A = sess.run(X_BA, feed_dict={X_B: input_B}) 181 | samples_B = sess.run(X_AB, feed_dict={X_A: input_A}) 182 | 183 | # The resulting image sample would be in 4 rows: 184 | # row 1: real data from domain A, row 2 is its domain B translation 185 | # row 3: real data from domain B, row 4 is its domain A translation 186 | samples = np.vstack([input_A, samples_B, input_B, samples_A]) 187 | 188 | fig = plot(samples) 189 | plt.savefig('out/{}.png' 190 | .format(str(i).zfill(3)), bbox_inches='tight') 191 | i += 1 192 | plt.close(fig) 193 | -------------------------------------------------------------------------------- /GAN/dual_gan/dualgan_tensorflow.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | from tensorflow.examples.tutorials.mnist import input_data 3 | import numpy as np 4 | import matplotlib.pyplot as plt 5 | import matplotlib.gridspec as gridspec 6 | import os 7 | import scipy.ndimage.interpolation 8 | 9 | 10 | mnist = input_data.read_data_sets('../../MNIST_data', one_hot=True) 11 | 12 | mb_size = 32 13 | X_dim = mnist.train.images.shape[1] 14 | y_dim = mnist.train.labels.shape[1] 15 | z_dim = 10 16 | h_dim = 128 17 | eps = 1e-8 18 | lr = 1e-3 19 | d_steps = 3 20 | lam1, lam2 = 1000, 1000 21 | 22 | 23 | def plot(samples): 24 | fig = plt.figure(figsize=(4, 4)) 25 | gs = gridspec.GridSpec(4, 4) 26 | gs.update(wspace=0.05, hspace=0.05) 27 | 28 | for i, sample in enumerate(samples): 29 | ax = plt.subplot(gs[i]) 30 | plt.axis('off') 31 | ax.set_xticklabels([]) 32 | ax.set_yticklabels([]) 33 | ax.set_aspect('equal') 34 | plt.imshow(sample.reshape(28, 28), cmap='Greys_r') 35 | 36 | return fig 37 | 38 | 39 | def xavier_init(size): 40 | in_dim = size[0] 41 | xavier_stddev = 1. / tf.sqrt(in_dim / 2.) 42 | return tf.random_normal(shape=size, stddev=xavier_stddev) 43 | 44 | 45 | X1 = tf.placeholder(tf.float32, shape=[None, X_dim]) 46 | X2 = tf.placeholder(tf.float32, shape=[None, X_dim]) 47 | z = tf.placeholder(tf.float32, shape=[None, z_dim]) 48 | 49 | G1_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim])) 50 | G1_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 51 | G1_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 52 | G1_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 53 | 54 | G2_W1 = tf.Variable(xavier_init([X_dim + z_dim, h_dim])) 55 | G2_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 56 | G2_W2 = tf.Variable(xavier_init([h_dim, X_dim])) 57 | G2_b2 = tf.Variable(tf.zeros(shape=[X_dim])) 58 | 59 | 60 | def G1(X1, z): 61 | inputs = tf.concat([X1, z], 1) 62 | h = tf.nn.relu(tf.matmul(inputs, G1_W1) + G1_b1) 63 | return tf.nn.sigmoid(tf.matmul(h, G1_W2) + G1_b2) 64 | 65 | 66 | def G2(X2, z): 67 | inputs = tf.concat([X2, z], 1) 68 | h = tf.nn.relu(tf.matmul(inputs, G2_W1) + G2_b1) 69 | return tf.nn.sigmoid(tf.matmul(h, G2_W2) + G2_b2) 70 | 71 | 72 | D1_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 73 | D1_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 74 | D1_W2 = tf.Variable(xavier_init([h_dim, 1])) 75 | D1_b2 = tf.Variable(tf.zeros(shape=[1])) 76 | 77 | D2_W1 = tf.Variable(xavier_init([X_dim, h_dim])) 78 | D2_b1 = tf.Variable(tf.zeros(shape=[h_dim])) 79 | D2_W2 = tf.Variable(xavier_init([h_dim, 1])) 80 | D2_b2 = tf.Variable(tf.zeros(shape=[1])) 81 | 82 | 83 | def D1(X): 84 | h = tf.nn.relu(tf.matmul(X, D1_W1) + D1_b1) 85 | return tf.matmul(h, D1_W2) + D1_b2 86 | 87 | 88 | def D2(X): 89 | h = tf.nn.relu(tf.matmul(X, D1_W1) + D1_b1) 90 | return tf.matmul(h, D2_W2) + D2_b2 91 | 92 | 93 | theta_G1 = [G1_W1, G1_W2, G1_b2, G1_b2] 94 | theta_G2 = [G2_W1, G2_b1, G2_W2, G2_b2] 95 | theta_G = theta_G1 + theta_G2 96 | 97 | theta_D1 = [D1_W1, D1_W2, D1_b1, D1_b2] 98 | theta_D2 = [D2_W1, D2_b1, D2_W2, D2_b2] 99 | 100 | # D 101 | X1_sample = G2(X2, z) 102 | X2_sample = G1(X1, z) 103 | 104 | D1_real = D1(X2) 105 | D1_fake = D1(X2_sample) 106 | 107 | D2_real = D2(X1) 108 | D2_fake = D2(X1_sample) 109 | 110 | D1_G = D1(X1_sample) 111 | D2_G = D2(X2_sample) 112 | 113 | X1_recon = G2(X2_sample, z) 114 | X2_recon = G1(X1_sample, z) 115 | recon1 = tf.reduce_mean(tf.reduce_sum(tf.abs(X1 - X1_recon), 1)) 116 | recon2 = tf.reduce_mean(tf.reduce_sum(tf.abs(X2 - X2_recon), 1)) 117 | 118 | D1_loss = tf.reduce_mean(D1_fake) - tf.reduce_mean(D1_real) 119 | D2_loss = tf.reduce_mean(D2_fake) - tf.reduce_mean(D2_real) 120 | G_loss = -tf.reduce_mean(D1_G + D2_G) + lam1*recon1 + lam2*recon2 121 | 122 | D1_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4) 123 | .minimize(D1_loss, var_list=theta_D1)) 124 | D2_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4) 125 | .minimize(D2_loss, var_list=theta_D2)) 126 | G_solver = (tf.train.RMSPropOptimizer(learning_rate=1e-4) 127 | .minimize(G_loss, var_list=theta_G)) 128 | 129 | clip_D = [p.assign(tf.clip_by_value(p, -0.01, 0.01)) for p in theta_D1 + theta_D2] 130 | 131 | 132 | sess = tf.Session() 133 | sess.run(tf.global_variables_initializer()) 134 | 135 | X_train = mnist.train.images 136 | half = int(X_train.shape[0] / 2) 137 | 138 | # Real image 139 | X_train1 = X_train[:half] 140 | # Rotated image 141 | X_train2 = X_train[half:].reshape(-1, 28, 28) 142 | X_train2 = scipy.ndimage.interpolation.rotate(X_train2, 90, axes=(1, 2)) 143 | X_train2 = X_train2.reshape(-1, 28*28) 144 | 145 | # Cleanup 146 | del X_train 147 | 148 | 149 | def sample_X(X, size): 150 | start_idx = np.random.randint(0, X.shape[0]-size) 151 | return X[start_idx:start_idx+size] 152 | 153 | 154 | def sample_z(m, n): 155 | return np.random.uniform(-1., 1., size=[m, n]) 156 | 157 | 158 | if not os.path.exists('out/'): 159 | os.makedirs('out/') 160 | 161 | i = 0 162 | 163 | for it in range(1000000): 164 | for _ in range(d_steps): 165 | X1_mb, X2_mb = sample_X(X_train1, mb_size), sample_X(X_train2, mb_size) 166 | z_mb = sample_z(mb_size, z_dim) 167 | 168 | _, _, D1_loss_curr, D2_loss_curr, _ = sess.run( 169 | [D1_solver, D2_solver, D1_loss, D2_loss, clip_D], 170 | feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb} 171 | ) 172 | 173 | _, G_loss_curr = sess.run( 174 | [G_solver, G_loss], feed_dict={X1: X1_mb, X2: X2_mb, z: z_mb} 175 | ) 176 | 177 | if it % 1000 == 0: 178 | sample1, sample2 = sess.run( 179 | [X1_sample, X2_sample], 180 | feed_dict={X1: X1_mb[:4], X2: X2_mb[:4], z: sample_z(4, z_dim)} 181 | ) 182 | 183 | samples = np.vstack([X1_mb[:4], sample1, X2_mb[:4], sample2]) 184 | 185 | print('Iter: {}; D_loss: {:.4}; G_loss: {:.4}' 186 | .format(it, D1_loss_curr + D2_loss_curr, G_loss_curr)) 187 | 188 | fig = plot(samples) 189 | plt.savefig('out/{}.png' 190 | .format(str(i).zfill(3)), bbox_inches='tight') 191 | i += 1 192 | plt.close(fig) 193 | --------------------------------------------------------------------------------