├── .github └── FUNDING.yml ├── .gitignore ├── LICENCE ├── README.md ├── data ├── content-images │ ├── figures.jpg │ ├── golden_gate.jpg │ ├── golden_gate2.jpg │ ├── green_bridge.jpg │ ├── lion.jpg │ ├── taj_mahal.jpg │ └── tubingen.png ├── examples │ ├── bridge │ │ ├── content_style.jpg │ │ ├── green_bridge_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ ├── green_bridge_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg │ │ ├── green_bridge_vg_starry_night_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ └── green_bridge_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ ├── content_reconstruction │ │ ├── 0000.jpg │ │ ├── 0026.jpg │ │ ├── 0070.jpg │ │ └── 0509.jpg │ ├── figures │ │ ├── figures_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ ├── figures_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png │ │ ├── figures_vg_houses_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0.png │ │ ├── figures_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png │ │ ├── figures_vg_starry_night_o_adam_i_content_h_400_m_vgg19_cw_100000.0_sw_100000.0_tv_0.1.png │ │ ├── figures_vg_starry_night_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ ├── figures_vg_wheat_field_w_350_m_vgg19_cw_100000.0_sw_300000.0_tv_1.0_resized.jpg │ │ └── figures_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ ├── fms_gram │ │ ├── fm_vgg19_relu1_1_0005_resized.jpg │ │ ├── fm_vgg19_relu1_1_0046_resized.jpg │ │ ├── fm_vgg19_relu1_1_0058_resized.jpg │ │ └── gram_vgg19_relu2_1_0001.jpg │ ├── gatys_reconstruction │ │ ├── tubingen.jpg │ │ ├── tubingen_kandinsky_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg │ │ ├── tubingen_seated-nude_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_2000.0_tv_1.0.jpg │ │ ├── tubingen_shipwreck_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_200.0_tv_1.0_resized.jpg │ │ ├── tubingen_starry-night_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg │ │ └── tubingen_the_scream_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_300.0_tv_1.0.jpg │ ├── golden_gate │ │ ├── golden_gate2_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png │ │ └── golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png │ ├── init_methods │ │ ├── golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ ├── golden_gate_vg_la_cafe_o_lbfgs_i_random_h_500_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg │ │ └── golden_gate_vg_la_cafe_o_lbfgs_i_style_h_500_m_vgg19_cw_100000.0_sw_10.0_tv_0.1_resized.jpg │ ├── lion │ │ ├── lion_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ ├── lion_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ │ └── lion_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg │ ├── style-tradeoff │ │ ├── figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10.0_tv_1.0_resized.jpg │ │ ├── figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_100.0_tv_1.0_resized.jpg │ │ ├── figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg │ │ └── figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10000.0_tv_1.0_resized.jpg │ ├── style_reconstruction │ │ ├── 0045.jpg │ │ ├── 0129.jpg │ │ ├── 0510.jpg │ │ └── candy.jpg │ ├── taj_mahal │ │ └── taj_mahal_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg │ └── tv-tradeoff │ │ ├── figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0_resized.jpg │ │ ├── figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10000.0_resized.jpg │ │ ├── figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_100000.0_resized.jpg │ │ └── figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1000000.0_resized.jpg └── style-images │ ├── ben_giles.jpg │ ├── candy.jpg │ ├── edtaonisl.jpg │ ├── flowers_crop.jpg │ ├── giger_crop.jpg │ ├── mosaic.jpg │ ├── okeffe_red_canna.png │ ├── tahiti_guaguin.jpg │ ├── udnie.jpg │ ├── vg_houses.jpg │ ├── vg_la_cafe.jpg │ ├── vg_olive.jpg │ ├── vg_self.jpg │ ├── vg_starry_night.jpg │ ├── vg_starry_night_resized.jpg │ ├── vg_wheat_field.jpg │ ├── vg_wheat_field_cropped.jpg │ └── wave_crop.jpg ├── environment.yml ├── models └── definitions │ ├── __init__.py │ └── vgg_nets.py ├── neural_style_transfer.py ├── reconstruct_image_from_representation.py └── utils ├── __init__.py ├── utils.py └── video_utils.py /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | patreon: theaiepiphany 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | __pycache__ -------------------------------------------------------------------------------- /LICENCE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Aleksa Gordić 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Neural Style Transfer (optimization method) :computer: + :art: = :heart: 2 | This repo contains a concise PyTorch implementation of the original NST paper (:link: [Gatys et al.](https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf)). 3 | 4 | It's an accompanying repository for [this video series on YouTube](https://www.youtube.com/watch?v=S78LQebx6jo&list=PLBoQnSflObcmbfshq9oNs41vODgXG-608). 5 | 6 |

7 | NST Intro 9 |

10 | 11 | ### What is NST algorithm? 12 | The algorithm transfers style from one input image (the style image) onto another input image (the content image) using CNN nets (usually VGG-16/19) and gives a composite, stylized image out which keeps the content from the content image but takes the style from the style image. 13 | 14 |

15 | 16 | 17 |

18 | 19 | ### Why yet another NST repo? 20 | It's the **cleanest and most concise** NST repo that I know of + it's written in **PyTorch!** :heart: 21 | 22 | Most of NST repos were written in TensorFlow (before it even had L-BFGS optimizer) and torch (obsolete framework, used Lua) and are overly complicated often times including multiple functionalities (video, static image, color transfer, etc.) in 1 repo and exposing 100 parameters over command-line (out of which maybe 5 or 6 may actually be used on a regular basis). 23 | 24 | ## Examples 25 | 26 | Transfering style gives beautiful artistic results: 27 | 28 |

29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 |

37 | 38 | And here are some results coupled with their style: 39 | 40 |

41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 |

53 | 54 | *Note: all of the stylized images were produced by me (using this repo), credits for original image artists [are given bellow](#acknowledgements).* 55 | 56 | ### Content/Style tradeoff 57 | 58 | Changing style weight gives you less or more style on the final image, assuming you keep the content weight constant.
59 | I did increments of 10 here for style weight (1e1, 1e2, 1e3, 1e4), while keeping content weight at constant 1e5, and I used random image as initialization image. 60 | 61 |

62 | 63 | 64 | 65 | 66 |

67 | 68 | ### Impact of total variation (tv) loss 69 | 70 | Rarely explained, the total variation loss i.e. it's corresponding weight controls the smoothness of the image.
71 | I also did increments of 10 here (1e1, 1e4, 1e5, 1e6) and I used content image as initialization image. 72 | 73 |

74 | 75 | 76 | 77 | 78 |

79 | 80 | ### Optimization initialization 81 | 82 | Starting with different initialization images: noise (white or gaussian), content and style leads to different results.
83 | Empirically content image gives the best results as explored in [this research paper](https://arxiv.org/pdf/1602.07188.pdf) also.
84 | Here you can see results for content, random and style initialization in that order (left to right): 85 | 86 |

87 | 88 | 89 | 90 |

91 | 92 | You can also see that with style initialization we had some content from the artwork leaking directly into our output. 93 | 94 | ### Famous "Figure 3" reconstruction 95 | 96 | Finally if I haven't included this portion you couldn't say that I've successfully reproduced the [original paper]((https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf)) (laughs in Python): 97 | 98 |

99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 |

107 | 108 | I haven't give it much effort results can be much nicer. 109 | 110 | ### Content reconstruction 111 | 112 | If we only use the content (perceptual) loss and try to minimize that objective function this is what we get (starting from noise): 113 | 114 |

115 | 116 | 117 | 118 | 119 |

120 | 121 | In steps 0, 26, 70 and 509 of the L-BFGS numerical optimizer, using layer relu3_1 for content representation.
122 | Check-out [this section](#reconstruct-image-from-representation) if you want to play with this. 123 | 124 | ### Style reconstruction 125 | 126 | We can do the same thing for style (on the left is the original art image "Candy") starting from noise: 127 | 128 |

129 | 130 | 131 | 132 | 133 |

134 | 135 | In steps 45, 129 and 510 of the L-BFGS using layers relu1_1, relu2_1, relu3_1, relu4_1 and relu5_1 for style representation. 136 | 137 | ## Setup 138 | 139 | 1. Open Anaconda Prompt and navigate into project directory `cd path_to_repo` 140 | 2. Run `conda env create` (while in project directory) 141 | 3. Run `activate pytorch-nst` 142 | 143 | That's it! It should work out-of-the-box executing environment.yml file which deals with dependencies. 144 | 145 | ----- 146 | 147 | PyTorch package will pull some version of CUDA with it, but it is highly recommended that you install system-wide CUDA beforehand, mostly because of GPU drivers. I also recommend using Miniconda installer as a way to get conda on your system. 148 | 149 | Follow through points 1 and 2 of [this setup](https://github.com/Petlja/PSIML/blob/master/docs/MachineSetup.md) and use the most up-to-date versions of Miniconda (Python 3.7) and CUDA/cuDNN. 150 | (I recommend CUDA 10.1 as it is compatible with PyTorch 1.4, which is used in this repo, and newest compatible cuDNN) 151 | 152 | ## Usage 153 | 154 | 1. Copy content images to the default content image directory: `/data/content-images/` 155 | 2. Copy style images to the default style image directory: `/data/style-images/` 156 | 3. Run `python neural_style_transfer.py --content_img_name --style_img_name ` 157 | 158 | It's that easy. For more advanced usage take a look at the code it's (hopefully) self-explanatory (if you speak Python ^^). 159 | 160 | Or take a look at [this accompanying YouTube video](https://www.youtube.com/watch?v=XWMwdkaLFsI), it explains how to use this repo in greater detail. 161 | 162 | Just run it! So that you can get something like this: :heart: 163 |

164 | 165 |

166 | 167 | ### Debugging/Experimenting 168 | 169 | Q: L-BFGS can't run on my computer it takes too much GPU VRAM?
170 | A: Set Adam as your default and take a look at the code for initial style/content/tv weights you should use as a start point. 171 | 172 | Q: Output image looks too much like style image?
173 | A: Decrease style weight or take a look at the table of weights (in neural_style_transfer.py), which I've included, that works. 174 | 175 | Q: There is too much noise (image is not smooth)?
176 | A: Increase total variation (tv) weight (usually by multiples of 10, again the table is your friend here or just experiment yourself). 177 | 178 | ### Reconstruct image from representation 179 | 180 | I've also included a file that will help you better understand how the algorithm works and what the neural net sees.
181 | What it does is that it allows you to visualize content **(feature maps)** and style representations **(Gram matrices)**.
182 | It will also reconstruct either only style or content using those representations and corresponding model that produces them.
183 | 184 | Just run this:
185 | `reconstruct_image_from_representation.py --should_reconstruct_content --should_visualize_representation ` 186 |

187 | And that's it! --should_visualize_representation if set to True will visualize these for you
188 | --should_reconstruct_content picks between style and content reconstruction 189 | 190 | Here are some feature maps (relu1_1, VGG 19) as well as a Gram matrix (relu2_1, VGG 19) for Van Gogh's famous [starry night](https://en.wikipedia.org/wiki/The_Starry_Night): 191 | 192 |

193 | 194 | 195 | 196 | 197 |

198 | 199 | No more dark magic. 200 | 201 | ## Acknowledgements 202 | 203 | I found these repos useful: (while developing this one) 204 | * [fast_neural_style](https://github.com/pytorch/examples/tree/master/fast_neural_style) (PyTorch, feed-forward method) 205 | * [neural-style-tf](https://github.com/cysmith/neural-style-tf/) (TensorFlow, optimization method) 206 | * [neural-style](https://github.com/anishathalye/neural-style/) (TensorFlow, optimization method) 207 | 208 | I found some of the content/style images I was using here: 209 | * [style/artistic images](https://www.rawpixel.com/board/537381/vincent-van-gogh-free-original-public-domain-paintings?sort=curated&mode=shop&page=1) 210 | * [awesome figures pic](https://www.pexels.com/photo/action-android-device-electronics-595804/) 211 | * [awesome bridge pic](https://www.pexels.com/photo/gray-bridge-and-trees-814499/) 212 | 213 | Other images are now already classics in the NST world. 214 | 215 | ## Citation 216 | 217 | If you find this code useful for your research, please cite the following: 218 | 219 | ``` 220 | @misc{Gordić2020nst, 221 | author = {Gordić, Aleksa}, 222 | title = {pytorch-neural-style-transfer}, 223 | year = {2020}, 224 | publisher = {GitHub}, 225 | journal = {GitHub repository}, 226 | howpublished = {\url{https://github.com/gordicaleksa/pytorch-neural-style-transfer}}, 227 | } 228 | ``` 229 | 230 | ## Connect with me 231 | 232 | If you'd love to have some more AI-related content in your life :nerd_face:, consider: 233 | * Subscribing to my YouTube channel [The AI Epiphany](https://www.youtube.com/c/TheAiEpiphany) :bell: 234 | * Follow me on [LinkedIn](https://www.linkedin.com/in/aleksagordic/) and [Twitter](https://twitter.com/gordic_aleksa) :bulb: 235 | * Follow me on [Medium](https://gordicaleksa.medium.com/) :books: :heart: 236 | 237 | ## Licence 238 | 239 | [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://github.com/gordicaleksa/pytorch-neural-style-transfer/blob/master/LICENCE) -------------------------------------------------------------------------------- /data/content-images/figures.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/figures.jpg -------------------------------------------------------------------------------- /data/content-images/golden_gate.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/golden_gate.jpg -------------------------------------------------------------------------------- /data/content-images/golden_gate2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/golden_gate2.jpg -------------------------------------------------------------------------------- /data/content-images/green_bridge.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/green_bridge.jpg -------------------------------------------------------------------------------- /data/content-images/lion.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/lion.jpg -------------------------------------------------------------------------------- /data/content-images/taj_mahal.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/taj_mahal.jpg -------------------------------------------------------------------------------- /data/content-images/tubingen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/content-images/tubingen.png -------------------------------------------------------------------------------- /data/examples/bridge/content_style.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/bridge/content_style.jpg -------------------------------------------------------------------------------- /data/examples/bridge/green_bridge_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/bridge/green_bridge_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/bridge/green_bridge_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/bridge/green_bridge_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/bridge/green_bridge_vg_starry_night_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/bridge/green_bridge_vg_starry_night_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/bridge/green_bridge_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/bridge/green_bridge_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/content_reconstruction/0000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/content_reconstruction/0000.jpg -------------------------------------------------------------------------------- /data/examples/content_reconstruction/0026.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/content_reconstruction/0026.jpg -------------------------------------------------------------------------------- /data/examples/content_reconstruction/0070.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/content_reconstruction/0070.jpg -------------------------------------------------------------------------------- /data/examples/content_reconstruction/0509.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/content_reconstruction/0509.jpg -------------------------------------------------------------------------------- /data/examples/figures/figures_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/figures/figures_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png -------------------------------------------------------------------------------- /data/examples/figures/figures_vg_houses_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_vg_houses_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0.png -------------------------------------------------------------------------------- /data/examples/figures/figures_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png -------------------------------------------------------------------------------- /data/examples/figures/figures_vg_starry_night_o_adam_i_content_h_400_m_vgg19_cw_100000.0_sw_100000.0_tv_0.1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_vg_starry_night_o_adam_i_content_h_400_m_vgg19_cw_100000.0_sw_100000.0_tv_0.1.png -------------------------------------------------------------------------------- /data/examples/figures/figures_vg_starry_night_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_vg_starry_night_w_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/figures/figures_vg_wheat_field_w_350_m_vgg19_cw_100000.0_sw_300000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_vg_wheat_field_w_350_m_vgg19_cw_100000.0_sw_300000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/figures/figures_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/figures/figures_wave_crop_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/fms_gram/fm_vgg19_relu1_1_0005_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/fms_gram/fm_vgg19_relu1_1_0005_resized.jpg -------------------------------------------------------------------------------- /data/examples/fms_gram/fm_vgg19_relu1_1_0046_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/fms_gram/fm_vgg19_relu1_1_0046_resized.jpg -------------------------------------------------------------------------------- /data/examples/fms_gram/fm_vgg19_relu1_1_0058_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/fms_gram/fm_vgg19_relu1_1_0058_resized.jpg -------------------------------------------------------------------------------- /data/examples/fms_gram/gram_vgg19_relu2_1_0001.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/fms_gram/gram_vgg19_relu2_1_0001.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen_kandinsky_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen_kandinsky_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen_seated-nude_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_2000.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen_seated-nude_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_2000.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen_shipwreck_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_200.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen_shipwreck_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_200.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen_starry-night_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen_starry-night_o_lbfgs_i_content_h_400_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/gatys_reconstruction/tubingen_the_scream_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_300.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/gatys_reconstruction/tubingen_the_scream_o_lbfgs_i_random_h_400_m_vgg19_cw_100000.0_sw_300.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/golden_gate/golden_gate2_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/golden_gate/golden_gate2_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png -------------------------------------------------------------------------------- /data/examples/golden_gate/golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/golden_gate/golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.png -------------------------------------------------------------------------------- /data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_random_h_500_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_random_h_500_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_style_h_500_m_vgg19_cw_100000.0_sw_10.0_tv_0.1_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/init_methods/golden_gate_vg_la_cafe_o_lbfgs_i_style_h_500_m_vgg19_cw_100000.0_sw_10.0_tv_0.1_resized.jpg -------------------------------------------------------------------------------- /data/examples/lion/lion_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/lion/lion_candy_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/lion/lion_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/lion/lion_edtaonisl_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/lion/lion_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/lion/lion_vg_la_cafe_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_100.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_100.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_1000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10000.0_tv_1.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style-tradeoff/figures_vg_starry_night_o_lbfgs_i_random_h_352_m_vgg19_cw_100000.0_sw_10000.0_tv_1.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/style_reconstruction/0045.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style_reconstruction/0045.jpg -------------------------------------------------------------------------------- /data/examples/style_reconstruction/0129.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style_reconstruction/0129.jpg -------------------------------------------------------------------------------- /data/examples/style_reconstruction/0510.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style_reconstruction/0510.jpg -------------------------------------------------------------------------------- /data/examples/style_reconstruction/candy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/style_reconstruction/candy.jpg -------------------------------------------------------------------------------- /data/examples/taj_mahal/taj_mahal_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/taj_mahal/taj_mahal_ben_giles_o_lbfgs_i_content_h_500_m_vgg19_cw_100000.0_sw_30000.0_tv_1.0.jpg -------------------------------------------------------------------------------- /data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10000.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_10000.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_100000.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_100000.0_resized.jpg -------------------------------------------------------------------------------- /data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1000000.0_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/examples/tv-tradeoff/figures_candy_o_lbfgs_i_content_h_350_m_vgg19_cw_100000.0_sw_30000.0_tv_1000000.0_resized.jpg -------------------------------------------------------------------------------- /data/style-images/ben_giles.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/ben_giles.jpg -------------------------------------------------------------------------------- /data/style-images/candy.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/candy.jpg -------------------------------------------------------------------------------- /data/style-images/edtaonisl.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/edtaonisl.jpg -------------------------------------------------------------------------------- /data/style-images/flowers_crop.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/flowers_crop.jpg -------------------------------------------------------------------------------- /data/style-images/giger_crop.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/giger_crop.jpg -------------------------------------------------------------------------------- /data/style-images/mosaic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/mosaic.jpg -------------------------------------------------------------------------------- /data/style-images/okeffe_red_canna.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/okeffe_red_canna.png -------------------------------------------------------------------------------- /data/style-images/tahiti_guaguin.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/tahiti_guaguin.jpg -------------------------------------------------------------------------------- /data/style-images/udnie.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/udnie.jpg -------------------------------------------------------------------------------- /data/style-images/vg_houses.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_houses.jpg -------------------------------------------------------------------------------- /data/style-images/vg_la_cafe.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_la_cafe.jpg -------------------------------------------------------------------------------- /data/style-images/vg_olive.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_olive.jpg -------------------------------------------------------------------------------- /data/style-images/vg_self.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_self.jpg -------------------------------------------------------------------------------- /data/style-images/vg_starry_night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_starry_night.jpg -------------------------------------------------------------------------------- /data/style-images/vg_starry_night_resized.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_starry_night_resized.jpg -------------------------------------------------------------------------------- /data/style-images/vg_wheat_field.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_wheat_field.jpg -------------------------------------------------------------------------------- /data/style-images/vg_wheat_field_cropped.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/vg_wheat_field_cropped.jpg -------------------------------------------------------------------------------- /data/style-images/wave_crop.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/data/style-images/wave_crop.jpg -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- 1 | name: pytorch-nst 2 | channels: 3 | - defaults 4 | - pytorch 5 | dependencies: 6 | - python=3.7.6 7 | - pip=20.0.2 8 | - matplotlib=3.1.3 9 | - pytorch==1.4.0 10 | - torchvision=0.5.0 11 | - pip: 12 | - numpy==1.18.1 13 | - opencv-python==4.2.0.32 -------------------------------------------------------------------------------- /models/definitions/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/models/definitions/__init__.py -------------------------------------------------------------------------------- /models/definitions/vgg_nets.py: -------------------------------------------------------------------------------- 1 | from collections import namedtuple 2 | import torch 3 | from torchvision import models 4 | 5 | """ 6 | More detail about the VGG architecture (if you want to understand magic/hardcoded numbers) can be found here: 7 | 8 | https://github.com/pytorch/vision/blob/3c254fb7af5f8af252c24e89949c54a3461ff0be/torchvision/models/vgg.py 9 | """ 10 | 11 | 12 | class Vgg16(torch.nn.Module): 13 | """Only those layers are exposed which have already proven to work nicely.""" 14 | def __init__(self, requires_grad=False, show_progress=False): 15 | super().__init__() 16 | vgg_pretrained_features = models.vgg16(pretrained=True, progress=show_progress).features 17 | self.layer_names = ['relu1_2', 'relu2_2', 'relu3_3', 'relu4_3'] 18 | self.content_feature_maps_index = 1 # relu2_2 19 | self.style_feature_maps_indices = list(range(len(self.layer_names))) # all layers used for style representation 20 | 21 | self.slice1 = torch.nn.Sequential() 22 | self.slice2 = torch.nn.Sequential() 23 | self.slice3 = torch.nn.Sequential() 24 | self.slice4 = torch.nn.Sequential() 25 | for x in range(4): 26 | self.slice1.add_module(str(x), vgg_pretrained_features[x]) 27 | for x in range(4, 9): 28 | self.slice2.add_module(str(x), vgg_pretrained_features[x]) 29 | for x in range(9, 16): 30 | self.slice3.add_module(str(x), vgg_pretrained_features[x]) 31 | for x in range(16, 23): 32 | self.slice4.add_module(str(x), vgg_pretrained_features[x]) 33 | if not requires_grad: 34 | for param in self.parameters(): 35 | param.requires_grad = False 36 | 37 | def forward(self, x): 38 | x = self.slice1(x) 39 | relu1_2 = x 40 | x = self.slice2(x) 41 | relu2_2 = x 42 | x = self.slice3(x) 43 | relu3_3 = x 44 | x = self.slice4(x) 45 | relu4_3 = x 46 | vgg_outputs = namedtuple("VggOutputs", self.layer_names) 47 | out = vgg_outputs(relu1_2, relu2_2, relu3_3, relu4_3) 48 | return out 49 | 50 | 51 | class Vgg16Experimental(torch.nn.Module): 52 | """Everything exposed so you can play with different combinations for style and content representation""" 53 | def __init__(self, requires_grad=False, show_progress=False): 54 | super().__init__() 55 | vgg_pretrained_features = models.vgg16(pretrained=True, progress=show_progress).features 56 | self.layer_names = ['relu1_1', 'relu2_1', 'relu2_2', 'relu3_1', 'relu3_2', 'relu4_1', 'relu4_3', 'relu5_1'] 57 | self.content_feature_maps_index = 4 58 | self.style_feature_maps_indices = list(range(len(self.layer_names))) # all layers used for style representation 59 | 60 | self.conv1_1 = vgg_pretrained_features[0] 61 | self.relu1_1 = vgg_pretrained_features[1] 62 | self.conv1_2 = vgg_pretrained_features[2] 63 | self.relu1_2 = vgg_pretrained_features[3] 64 | self.max_pooling1 = vgg_pretrained_features[4] 65 | self.conv2_1 = vgg_pretrained_features[5] 66 | self.relu2_1 = vgg_pretrained_features[6] 67 | self.conv2_2 = vgg_pretrained_features[7] 68 | self.relu2_2 = vgg_pretrained_features[8] 69 | self.max_pooling2 = vgg_pretrained_features[9] 70 | self.conv3_1 = vgg_pretrained_features[10] 71 | self.relu3_1 = vgg_pretrained_features[11] 72 | self.conv3_2 = vgg_pretrained_features[12] 73 | self.relu3_2 = vgg_pretrained_features[13] 74 | self.conv3_3 = vgg_pretrained_features[14] 75 | self.relu3_3 = vgg_pretrained_features[15] 76 | self.max_pooling3 = vgg_pretrained_features[16] 77 | self.conv4_1 = vgg_pretrained_features[17] 78 | self.relu4_1 = vgg_pretrained_features[18] 79 | self.conv4_2 = vgg_pretrained_features[19] 80 | self.relu4_2 = vgg_pretrained_features[20] 81 | self.conv4_3 = vgg_pretrained_features[21] 82 | self.relu4_3 = vgg_pretrained_features[22] 83 | self.max_pooling4 = vgg_pretrained_features[23] 84 | self.conv5_1 = vgg_pretrained_features[24] 85 | self.relu5_1 = vgg_pretrained_features[25] 86 | self.conv5_2 = vgg_pretrained_features[26] 87 | self.relu5_2 = vgg_pretrained_features[27] 88 | self.conv5_3 = vgg_pretrained_features[28] 89 | self.relu5_3 = vgg_pretrained_features[29] 90 | self.max_pooling5 = vgg_pretrained_features[30] 91 | if not requires_grad: 92 | for param in self.parameters(): 93 | param.requires_grad = False 94 | 95 | def forward(self, x): 96 | x = self.conv1_1(x) 97 | conv1_1 = x 98 | x = self.relu1_1(x) 99 | relu1_1 = x 100 | x = self.conv1_2(x) 101 | conv1_2 = x 102 | x = self.relu1_2(x) 103 | relu1_2 = x 104 | x = self.max_pooling1(x) 105 | x = self.conv2_1(x) 106 | conv2_1 = x 107 | x = self.relu2_1(x) 108 | relu2_1 = x 109 | x = self.conv2_2(x) 110 | conv2_2 = x 111 | x = self.relu2_2(x) 112 | relu2_2 = x 113 | x = self.max_pooling2(x) 114 | x = self.conv3_1(x) 115 | conv3_1 = x 116 | x = self.relu3_1(x) 117 | relu3_1 = x 118 | x = self.conv3_2(x) 119 | conv3_2 = x 120 | x = self.relu3_2(x) 121 | relu3_2 = x 122 | x = self.conv3_3(x) 123 | conv3_3 = x 124 | x = self.relu3_3(x) 125 | relu3_3 = x 126 | x = self.max_pooling3(x) 127 | x = self.conv4_1(x) 128 | conv4_1 = x 129 | x = self.relu4_1(x) 130 | relu4_1 = x 131 | x = self.conv4_2(x) 132 | conv4_2 = x 133 | x = self.relu4_2(x) 134 | relu4_2 = x 135 | x = self.conv4_3(x) 136 | conv4_3 = x 137 | x = self.relu4_3(x) 138 | relu4_3 = x 139 | x = self.max_pooling4(x) 140 | x = self.conv5_1(x) 141 | conv5_1 = x 142 | x = self.relu5_1(x) 143 | relu5_1 = x 144 | x = self.conv5_2(x) 145 | conv5_2 = x 146 | x = self.relu5_2(x) 147 | relu5_2 = x 148 | x = self.conv5_3(x) 149 | conv5_3 = x 150 | x = self.relu5_3(x) 151 | relu5_3 = x 152 | x = self.max_pooling5(x) 153 | # expose only the layers that you want to experiment with here 154 | vgg_outputs = namedtuple("VggOutputs", self.layer_names) 155 | out = vgg_outputs(relu1_1, relu2_1, relu2_2, relu3_1, relu3_2, relu4_1, relu4_3, relu5_1) 156 | 157 | return out 158 | 159 | 160 | class Vgg19(torch.nn.Module): 161 | """ 162 | Used in the original NST paper, only those layers are exposed which were used in the original paper 163 | 164 | 'conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv5_1' were used for style representation 165 | 'conv4_2' was used for content representation (although they did some experiments with conv2_2 and conv5_2) 166 | """ 167 | def __init__(self, requires_grad=False, show_progress=False, use_relu=True): 168 | super().__init__() 169 | vgg_pretrained_features = models.vgg19(pretrained=True, progress=show_progress).features 170 | if use_relu: # use relu or as in original paper conv layers 171 | self.layer_names = ['relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'conv4_2', 'relu5_1'] 172 | self.offset = 1 173 | else: 174 | self.layer_names = ['conv1_1', 'conv2_1', 'conv3_1', 'conv4_1', 'conv4_2', 'conv5_1'] 175 | self.offset = 0 176 | self.content_feature_maps_index = 4 # conv4_2 177 | # all layers used for style representation except conv4_2 178 | self.style_feature_maps_indices = list(range(len(self.layer_names))) 179 | self.style_feature_maps_indices.remove(4) # conv4_2 180 | 181 | self.slice1 = torch.nn.Sequential() 182 | self.slice2 = torch.nn.Sequential() 183 | self.slice3 = torch.nn.Sequential() 184 | self.slice4 = torch.nn.Sequential() 185 | self.slice5 = torch.nn.Sequential() 186 | self.slice6 = torch.nn.Sequential() 187 | for x in range(1+self.offset): 188 | self.slice1.add_module(str(x), vgg_pretrained_features[x]) 189 | for x in range(1+self.offset, 6+self.offset): 190 | self.slice2.add_module(str(x), vgg_pretrained_features[x]) 191 | for x in range(6+self.offset, 11+self.offset): 192 | self.slice3.add_module(str(x), vgg_pretrained_features[x]) 193 | for x in range(11+self.offset, 20+self.offset): 194 | self.slice4.add_module(str(x), vgg_pretrained_features[x]) 195 | for x in range(20+self.offset, 22): 196 | self.slice5.add_module(str(x), vgg_pretrained_features[x]) 197 | for x in range(22, 29++self.offset): 198 | self.slice6.add_module(str(x), vgg_pretrained_features[x]) 199 | if not requires_grad: 200 | for param in self.parameters(): 201 | param.requires_grad = False 202 | 203 | def forward(self, x): 204 | x = self.slice1(x) 205 | layer1_1 = x 206 | x = self.slice2(x) 207 | layer2_1 = x 208 | x = self.slice3(x) 209 | layer3_1 = x 210 | x = self.slice4(x) 211 | layer4_1 = x 212 | x = self.slice5(x) 213 | conv4_2 = x 214 | x = self.slice6(x) 215 | layer5_1 = x 216 | vgg_outputs = namedtuple("VggOutputs", self.layer_names) 217 | out = vgg_outputs(layer1_1, layer2_1, layer3_1, layer4_1, conv4_2, layer5_1) 218 | return out 219 | -------------------------------------------------------------------------------- /neural_style_transfer.py: -------------------------------------------------------------------------------- 1 | import utils.utils as utils 2 | from utils.video_utils import create_video_from_intermediate_results 3 | 4 | import torch 5 | from torch.optim import Adam, LBFGS 6 | from torch.autograd import Variable 7 | import numpy as np 8 | import os 9 | import argparse 10 | 11 | 12 | def build_loss(neural_net, optimizing_img, target_representations, content_feature_maps_index, style_feature_maps_indices, config): 13 | target_content_representation = target_representations[0] 14 | target_style_representation = target_representations[1] 15 | 16 | current_set_of_feature_maps = neural_net(optimizing_img) 17 | 18 | current_content_representation = current_set_of_feature_maps[content_feature_maps_index].squeeze(axis=0) 19 | content_loss = torch.nn.MSELoss(reduction='mean')(target_content_representation, current_content_representation) 20 | 21 | style_loss = 0.0 22 | current_style_representation = [utils.gram_matrix(x) for cnt, x in enumerate(current_set_of_feature_maps) if cnt in style_feature_maps_indices] 23 | for gram_gt, gram_hat in zip(target_style_representation, current_style_representation): 24 | style_loss += torch.nn.MSELoss(reduction='sum')(gram_gt[0], gram_hat[0]) 25 | style_loss /= len(target_style_representation) 26 | 27 | tv_loss = utils.total_variation(optimizing_img) 28 | 29 | total_loss = config['content_weight'] * content_loss + config['style_weight'] * style_loss + config['tv_weight'] * tv_loss 30 | 31 | return total_loss, content_loss, style_loss, tv_loss 32 | 33 | 34 | def make_tuning_step(neural_net, optimizer, target_representations, content_feature_maps_index, style_feature_maps_indices, config): 35 | # Builds function that performs a step in the tuning loop 36 | def tuning_step(optimizing_img): 37 | total_loss, content_loss, style_loss, tv_loss = build_loss(neural_net, optimizing_img, target_representations, content_feature_maps_index, style_feature_maps_indices, config) 38 | # Computes gradients 39 | total_loss.backward() 40 | # Updates parameters and zeroes gradients 41 | optimizer.step() 42 | optimizer.zero_grad() 43 | return total_loss, content_loss, style_loss, tv_loss 44 | 45 | # Returns the function that will be called inside the tuning loop 46 | return tuning_step 47 | 48 | 49 | def neural_style_transfer(config): 50 | content_img_path = os.path.join(config['content_images_dir'], config['content_img_name']) 51 | style_img_path = os.path.join(config['style_images_dir'], config['style_img_name']) 52 | 53 | out_dir_name = 'combined_' + os.path.split(content_img_path)[1].split('.')[0] + '_' + os.path.split(style_img_path)[1].split('.')[0] 54 | dump_path = os.path.join(config['output_img_dir'], out_dir_name) 55 | os.makedirs(dump_path, exist_ok=True) 56 | 57 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 58 | 59 | content_img = utils.prepare_img(content_img_path, config['height'], device) 60 | style_img = utils.prepare_img(style_img_path, config['height'], device) 61 | 62 | if config['init_method'] == 'random': 63 | # white_noise_img = np.random.uniform(-90., 90., content_img.shape).astype(np.float32) 64 | gaussian_noise_img = np.random.normal(loc=0, scale=90., size=content_img.shape).astype(np.float32) 65 | init_img = torch.from_numpy(gaussian_noise_img).float().to(device) 66 | elif config['init_method'] == 'content': 67 | init_img = content_img 68 | else: 69 | # init image has same dimension as content image - this is a hard constraint 70 | # feature maps need to be of same size for content image and init image 71 | style_img_resized = utils.prepare_img(style_img_path, np.asarray(content_img.shape[2:]), device) 72 | init_img = style_img_resized 73 | 74 | # we are tuning optimizing_img's pixels! (that's why requires_grad=True) 75 | optimizing_img = Variable(init_img, requires_grad=True) 76 | 77 | neural_net, content_feature_maps_index_name, style_feature_maps_indices_names = utils.prepare_model(config['model'], device) 78 | print(f'Using {config["model"]} in the optimization procedure.') 79 | 80 | content_img_set_of_feature_maps = neural_net(content_img) 81 | style_img_set_of_feature_maps = neural_net(style_img) 82 | 83 | target_content_representation = content_img_set_of_feature_maps[content_feature_maps_index_name[0]].squeeze(axis=0) 84 | target_style_representation = [utils.gram_matrix(x) for cnt, x in enumerate(style_img_set_of_feature_maps) if cnt in style_feature_maps_indices_names[0]] 85 | target_representations = [target_content_representation, target_style_representation] 86 | 87 | # magic numbers in general are a big no no - some things in this code are left like this by design to avoid clutter 88 | num_of_iterations = { 89 | "lbfgs": 1000, 90 | "adam": 3000, 91 | } 92 | 93 | # 94 | # Start of optimization procedure 95 | # 96 | if config['optimizer'] == 'adam': 97 | optimizer = Adam((optimizing_img,), lr=1e1) 98 | tuning_step = make_tuning_step(neural_net, optimizer, target_representations, content_feature_maps_index_name[0], style_feature_maps_indices_names[0], config) 99 | for cnt in range(num_of_iterations[config['optimizer']]): 100 | total_loss, content_loss, style_loss, tv_loss = tuning_step(optimizing_img) 101 | with torch.no_grad(): 102 | print(f'Adam | iteration: {cnt:03}, total loss={total_loss.item():12.4f}, content_loss={config["content_weight"] * content_loss.item():12.4f}, style loss={config["style_weight"] * style_loss.item():12.4f}, tv loss={config["tv_weight"] * tv_loss.item():12.4f}') 103 | utils.save_and_maybe_display(optimizing_img, dump_path, config, cnt, num_of_iterations[config['optimizer']], should_display=False) 104 | elif config['optimizer'] == 'lbfgs': 105 | # line_search_fn does not seem to have significant impact on result 106 | optimizer = LBFGS((optimizing_img,), max_iter=num_of_iterations['lbfgs'], line_search_fn='strong_wolfe') 107 | cnt = 0 108 | 109 | def closure(): 110 | nonlocal cnt 111 | if torch.is_grad_enabled(): 112 | optimizer.zero_grad() 113 | total_loss, content_loss, style_loss, tv_loss = build_loss(neural_net, optimizing_img, target_representations, content_feature_maps_index_name[0], style_feature_maps_indices_names[0], config) 114 | if total_loss.requires_grad: 115 | total_loss.backward() 116 | with torch.no_grad(): 117 | print(f'L-BFGS | iteration: {cnt:03}, total loss={total_loss.item():12.4f}, content_loss={config["content_weight"] * content_loss.item():12.4f}, style loss={config["style_weight"] * style_loss.item():12.4f}, tv loss={config["tv_weight"] * tv_loss.item():12.4f}') 118 | utils.save_and_maybe_display(optimizing_img, dump_path, config, cnt, num_of_iterations[config['optimizer']], should_display=False) 119 | 120 | cnt += 1 121 | return total_loss 122 | 123 | optimizer.step(closure) 124 | 125 | return dump_path 126 | 127 | 128 | if __name__ == "__main__": 129 | # 130 | # fixed args - don't change these unless you have a good reason 131 | # 132 | default_resource_dir = os.path.join(os.path.dirname(__file__), 'data') 133 | content_images_dir = os.path.join(default_resource_dir, 'content-images') 134 | style_images_dir = os.path.join(default_resource_dir, 'style-images') 135 | output_img_dir = os.path.join(default_resource_dir, 'output-images') 136 | img_format = (4, '.jpg') # saves images in the format: %04d.jpg 137 | 138 | # 139 | # modifiable args - feel free to play with these (only small subset is exposed by design to avoid cluttering) 140 | # sorted so that the ones on the top are more likely to be changed than the ones on the bottom 141 | # 142 | parser = argparse.ArgumentParser() 143 | parser.add_argument("--content_img_name", type=str, help="content image name", default='figures.jpg') 144 | parser.add_argument("--style_img_name", type=str, help="style image name", default='vg_starry_night.jpg') 145 | parser.add_argument("--height", type=int, help="height of content and style images", default=400) 146 | 147 | parser.add_argument("--content_weight", type=float, help="weight factor for content loss", default=1e5) 148 | parser.add_argument("--style_weight", type=float, help="weight factor for style loss", default=3e4) 149 | parser.add_argument("--tv_weight", type=float, help="weight factor for total variation loss", default=1e0) 150 | 151 | parser.add_argument("--optimizer", type=str, choices=['lbfgs', 'adam'], default='lbfgs') 152 | parser.add_argument("--model", type=str, choices=['vgg16', 'vgg19'], default='vgg19') 153 | parser.add_argument("--init_method", type=str, choices=['random', 'content', 'style'], default='content') 154 | parser.add_argument("--saving_freq", type=int, help="saving frequency for intermediate images (-1 means only final)", default=-1) 155 | args = parser.parse_args() 156 | 157 | # some values of weights that worked for figures.jpg, vg_starry_night.jpg (starting point for finding good images) 158 | # once you understand what each one does it gets really easy -> also see README.md 159 | 160 | # lbfgs, content init -> (cw, sw, tv) = (1e5, 3e4, 1e0) 161 | # lbfgs, style init -> (cw, sw, tv) = (1e5, 1e1, 1e-1) 162 | # lbfgs, random init -> (cw, sw, tv) = (1e5, 1e3, 1e0) 163 | 164 | # adam, content init -> (cw, sw, tv, lr) = (1e5, 1e5, 1e-1, 1e1) 165 | # adam, style init -> (cw, sw, tv, lr) = (1e5, 1e2, 1e-1, 1e1) 166 | # adam, random init -> (cw, sw, tv, lr) = (1e5, 1e2, 1e-1, 1e1) 167 | 168 | # just wrapping settings into a dictionary 169 | optimization_config = dict() 170 | for arg in vars(args): 171 | optimization_config[arg] = getattr(args, arg) 172 | optimization_config['content_images_dir'] = content_images_dir 173 | optimization_config['style_images_dir'] = style_images_dir 174 | optimization_config['output_img_dir'] = output_img_dir 175 | optimization_config['img_format'] = img_format 176 | 177 | # original NST (Neural Style Transfer) algorithm (Gatys et al.) 178 | results_path = neural_style_transfer(optimization_config) 179 | 180 | # uncomment this if you want to create a video from images dumped during the optimization procedure 181 | # create_video_from_intermediate_results(results_path, img_format) 182 | -------------------------------------------------------------------------------- /reconstruct_image_from_representation.py: -------------------------------------------------------------------------------- 1 | import utils.utils as utils 2 | from utils.video_utils import create_video_from_intermediate_results 3 | 4 | import os 5 | import argparse 6 | import torch 7 | from torch.autograd import Variable 8 | from torch.optim import Adam, LBFGS 9 | import numpy as np 10 | import matplotlib.pyplot as plt 11 | 12 | 13 | def make_tuning_step(model, optimizer, target_representation, should_reconstruct_content, content_feature_maps_index, style_feature_maps_indices): 14 | # Builds function that performs a step in the tuning loop 15 | def tuning_step(optimizing_img): 16 | # Finds the current representation 17 | set_of_feature_maps = model(optimizing_img) 18 | if should_reconstruct_content: 19 | current_representation = set_of_feature_maps[content_feature_maps_index].squeeze(axis=0) 20 | else: 21 | current_representation = [utils.gram_matrix(fmaps) for i, fmaps in enumerate(set_of_feature_maps) if i in style_feature_maps_indices] 22 | 23 | # Computes the loss between current and target representations 24 | loss = 0.0 25 | if should_reconstruct_content: 26 | loss = torch.nn.MSELoss(reduction='mean')(target_representation, current_representation) 27 | else: 28 | for gram_gt, gram_hat in zip(target_representation, current_representation): 29 | loss += (1 / len(target_representation)) * torch.nn.MSELoss(reduction='sum')(gram_gt[0], gram_hat[0]) 30 | 31 | # Computes gradients 32 | loss.backward() 33 | # Updates parameters and zeroes gradients 34 | optimizer.step() 35 | optimizer.zero_grad() 36 | # Returns the loss 37 | return loss.item(), current_representation 38 | 39 | # Returns the function that will be called inside the tuning loop 40 | return tuning_step 41 | 42 | 43 | def reconstruct_image_from_representation(config): 44 | should_reconstruct_content = config['should_reconstruct_content'] 45 | should_visualize_representation = config['should_visualize_representation'] 46 | dump_path = os.path.join(config['output_img_dir'], ('c' if should_reconstruct_content else 's') + '_reconstruction_' + config['optimizer']) 47 | dump_path = os.path.join(dump_path, os.path.basename(config['content_img_name']).split('.')[0] if should_reconstruct_content else os.path.basename(config['style_img_name']).split('.')[0]) 48 | os.makedirs(dump_path, exist_ok=True) 49 | 50 | content_img_path = os.path.join(config['content_images_dir'], config['content_img_name']) 51 | style_img_path = os.path.join(config['style_images_dir'], config['style_img_name']) 52 | img_path = content_img_path if should_reconstruct_content else style_img_path 53 | 54 | device = torch.device("cuda" if torch.cuda.is_available() else "cpu") 55 | 56 | img = utils.prepare_img(img_path, config['height'], device) 57 | 58 | gaussian_noise_img = np.random.normal(loc=0, scale=90., size=img.shape).astype(np.float32) 59 | white_noise_img = np.random.uniform(-90., 90., img.shape).astype(np.float32) 60 | init_img = torch.from_numpy(white_noise_img).float().to(device) 61 | optimizing_img = Variable(init_img, requires_grad=True) 62 | 63 | # indices pick relevant feature maps (say conv4_1, relu1_1, etc.) 64 | neural_net, content_feature_maps_index_name, style_feature_maps_indices_names = utils.prepare_model(config['model'], device) 65 | 66 | # don't want to expose everything that's not crucial so some things are hardcoded 67 | num_of_iterations = {'adam': 3000, 'lbfgs': 350} 68 | 69 | set_of_feature_maps = neural_net(img) 70 | 71 | # 72 | # Visualize feature maps and Gram matrices (depending whether you're reconstructing content or style img) 73 | # 74 | if should_reconstruct_content: 75 | target_content_representation = set_of_feature_maps[content_feature_maps_index_name[0]].squeeze(axis=0) 76 | if should_visualize_representation: 77 | num_of_feature_maps = target_content_representation.size()[0] 78 | print(f'Number of feature maps: {num_of_feature_maps}') 79 | for i in range(num_of_feature_maps): 80 | feature_map = target_content_representation[i].to('cpu').numpy() 81 | feature_map = np.uint8(utils.get_uint8_range(feature_map)) 82 | plt.imshow(feature_map) 83 | plt.title(f'Feature map {i+1}/{num_of_feature_maps} from layer {content_feature_maps_index_name[1]} (model={config["model"]}) for {config["content_img_name"]} image.') 84 | plt.show() 85 | filename = f'fm_{config["model"]}_{content_feature_maps_index_name[1]}_{str(i).zfill(config["img_format"][0])}{config["img_format"][1]}' 86 | utils.save_image(feature_map, os.path.join(dump_path, filename)) 87 | else: 88 | target_style_representation = [utils.gram_matrix(fmaps) for i, fmaps in enumerate(set_of_feature_maps) if i in style_feature_maps_indices_names[0]] 89 | if should_visualize_representation: 90 | num_of_gram_matrices = len(target_style_representation) 91 | print(f'Number of Gram matrices: {num_of_gram_matrices}') 92 | for i in range(num_of_gram_matrices): 93 | Gram_matrix = target_style_representation[i].squeeze(axis=0).to('cpu').numpy() 94 | Gram_matrix = np.uint8(utils.get_uint8_range(Gram_matrix)) 95 | plt.imshow(Gram_matrix) 96 | plt.title(f'Gram matrix from layer {style_feature_maps_indices_names[1][i]} (model={config["model"]}) for {config["style_img_name"]} image.') 97 | plt.show() 98 | filename = f'gram_{config["model"]}_{style_feature_maps_indices_names[1][i]}_{str(i).zfill(config["img_format"][0])}{config["img_format"][1]}' 99 | utils.save_image(Gram_matrix, os.path.join(dump_path, filename)) 100 | 101 | # 102 | # Start of optimization procedure 103 | # 104 | if config['optimizer'] == 'adam': 105 | optimizer = Adam((optimizing_img,)) 106 | target_representation = target_content_representation if should_reconstruct_content else target_style_representation 107 | tuning_step = make_tuning_step(neural_net, optimizer, target_representation, should_reconstruct_content, content_feature_maps_index_name[0], style_feature_maps_indices_names[0]) 108 | for it in range(num_of_iterations[config['optimizer']]): 109 | loss, _ = tuning_step(optimizing_img) 110 | with torch.no_grad(): 111 | print(f'Iteration: {it}, current {"content" if should_reconstruct_content else "style"} loss={loss:10.8f}') 112 | utils.save_and_maybe_display(optimizing_img, dump_path, config, it, num_of_iterations[config['optimizer']], should_display=False) 113 | elif config['optimizer'] == 'lbfgs': 114 | cnt = 0 115 | 116 | # closure is a function required by L-BFGS optimizer 117 | def closure(): 118 | nonlocal cnt 119 | optimizer.zero_grad() 120 | loss = 0.0 121 | if should_reconstruct_content: 122 | loss = torch.nn.MSELoss(reduction='mean')(target_content_representation, neural_net(optimizing_img)[content_feature_maps_index_name[0]].squeeze(axis=0)) 123 | else: 124 | current_set_of_feature_maps = neural_net(optimizing_img) 125 | current_style_representation = [utils.gram_matrix(fmaps) for i, fmaps in enumerate(current_set_of_feature_maps) if i in style_feature_maps_indices_names[0]] 126 | for gram_gt, gram_hat in zip(target_style_representation, current_style_representation): 127 | loss += (1 / len(target_style_representation)) * torch.nn.MSELoss(reduction='sum')(gram_gt[0], gram_hat[0]) 128 | loss.backward() 129 | with torch.no_grad(): 130 | print(f'Iteration: {cnt}, current {"content" if should_reconstruct_content else "style"} loss={loss.item()}') 131 | utils.save_and_maybe_display(optimizing_img, dump_path, config, cnt, num_of_iterations[config['optimizer']], should_display=False) 132 | cnt += 1 133 | return loss 134 | 135 | optimizer = torch.optim.LBFGS((optimizing_img,), max_iter=num_of_iterations[config['optimizer']], line_search_fn='strong_wolfe') 136 | optimizer.step(closure) 137 | 138 | return dump_path 139 | 140 | 141 | if __name__ == "__main__": 142 | # 143 | # fixed args - don't change these unless you have a good reason (default img locations and img dump format) 144 | # 145 | default_resource_dir = os.path.join(os.path.dirname(__file__), 'data') 146 | content_images_dir = os.path.join(default_resource_dir, 'content-images') 147 | style_images_dir = os.path.join(default_resource_dir, 'style-images') 148 | output_img_dir = os.path.join(default_resource_dir, 'output-images') 149 | img_format = (4, '.jpg') # saves images in the format: %04d.jpg 150 | 151 | # 152 | # modifiable args - feel free to play with these (only small subset is exposed by design to avoid cluttering) 153 | # 154 | parser = argparse.ArgumentParser() 155 | parser.add_argument("--should_reconstruct_content", type=bool, help="pick between content or style image reconstruction", default=True) 156 | parser.add_argument("--should_visualize_representation", type=bool, help="visualize feature maps or Gram matrices", default=False) 157 | 158 | parser.add_argument("--content_img_name", type=str, help="content image name", default='lion.jpg') 159 | parser.add_argument("--style_img_name", type=str, help="style image name", default='ben_giles.jpg') 160 | parser.add_argument("--height", type=int, help="width of content and style images (-1 keep original)", default=500) 161 | 162 | parser.add_argument("--saving_freq", type=int, help="saving frequency for intermediate images (-1 means only final)", default=1) 163 | parser.add_argument("--model", type=str, choices=['vgg16', 'vgg19'], default='vgg19') 164 | parser.add_argument("--optimizer", type=str, choices=['lbfgs', 'adam'], default='lbfgs') 165 | parser.add_argument("--reconstruct_script", type=str, help='dummy param - used in saving func', default=True) 166 | args = parser.parse_args() 167 | 168 | # just wrapping settings into a dictionary 169 | optimization_config = dict() 170 | for arg in vars(args): 171 | optimization_config[arg] = getattr(args, arg) 172 | optimization_config['content_images_dir'] = content_images_dir 173 | optimization_config['style_images_dir'] = style_images_dir 174 | optimization_config['output_img_dir'] = output_img_dir 175 | optimization_config['img_format'] = img_format 176 | 177 | # reconstruct style or content image purely from their representation 178 | results_path = reconstruct_image_from_representation(optimization_config) 179 | 180 | # create_video_from_intermediate_results(results_path, img_format) 181 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gordicaleksa/pytorch-neural-style-transfer/a244dc61797f920a62146d98de92384352a4d3a2/utils/__init__.py -------------------------------------------------------------------------------- /utils/utils.py: -------------------------------------------------------------------------------- 1 | import cv2 as cv 2 | import numpy as np 3 | import torch 4 | from torchvision import transforms 5 | import os 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | from models.definitions.vgg_nets import Vgg16, Vgg19, Vgg16Experimental 10 | 11 | 12 | IMAGENET_MEAN_255 = [123.675, 116.28, 103.53] 13 | IMAGENET_STD_NEUTRAL = [1, 1, 1] 14 | 15 | 16 | # 17 | # Image manipulation util functions 18 | # 19 | 20 | def load_image(img_path, target_shape=None): 21 | if not os.path.exists(img_path): 22 | raise Exception(f'Path does not exist: {img_path}') 23 | img = cv.imread(img_path)[:, :, ::-1] # [:, :, ::-1] converts BGR (opencv format...) into RGB 24 | 25 | if target_shape is not None: # resize section 26 | if isinstance(target_shape, int) and target_shape != -1: # scalar -> implicitly setting the height 27 | current_height, current_width = img.shape[:2] 28 | new_height = target_shape 29 | new_width = int(current_width * (new_height / current_height)) 30 | img = cv.resize(img, (new_width, new_height), interpolation=cv.INTER_CUBIC) 31 | else: # set both dimensions to target shape 32 | img = cv.resize(img, (target_shape[1], target_shape[0]), interpolation=cv.INTER_CUBIC) 33 | 34 | # this need to go after resizing - otherwise cv.resize will push values outside of [0,1] range 35 | img = img.astype(np.float32) # convert from uint8 to float32 36 | img /= 255.0 # get to [0, 1] range 37 | return img 38 | 39 | 40 | def prepare_img(img_path, target_shape, device): 41 | img = load_image(img_path, target_shape=target_shape) 42 | 43 | # normalize using ImageNet's mean 44 | # [0, 255] range worked much better for me than [0, 1] range (even though PyTorch models were trained on latter) 45 | transform = transforms.Compose([ 46 | transforms.ToTensor(), 47 | transforms.Lambda(lambda x: x.mul(255)), 48 | transforms.Normalize(mean=IMAGENET_MEAN_255, std=IMAGENET_STD_NEUTRAL) 49 | ]) 50 | 51 | img = transform(img).to(device).unsqueeze(0) 52 | 53 | return img 54 | 55 | 56 | def save_image(img, img_path): 57 | if len(img.shape) == 2: 58 | img = np.stack((img,) * 3, axis=-1) 59 | cv.imwrite(img_path, img[:, :, ::-1]) # [:, :, ::-1] converts rgb into bgr (opencv contraint...) 60 | 61 | 62 | def generate_out_img_name(config): 63 | prefix = os.path.basename(config['content_img_name']).split('.')[0] + '_' + os.path.basename(config['style_img_name']).split('.')[0] 64 | # called from the reconstruction script 65 | if 'reconstruct_script' in config: 66 | suffix = f'_o_{config["optimizer"]}_h_{str(config["height"])}_m_{config["model"]}{config["img_format"][1]}' 67 | else: 68 | suffix = f'_o_{config["optimizer"]}_i_{config["init_method"]}_h_{str(config["height"])}_m_{config["model"]}_cw_{config["content_weight"]}_sw_{config["style_weight"]}_tv_{config["tv_weight"]}{config["img_format"][1]}' 69 | return prefix + suffix 70 | 71 | 72 | def save_and_maybe_display(optimizing_img, dump_path, config, img_id, num_of_iterations, should_display=False): 73 | saving_freq = config['saving_freq'] 74 | out_img = optimizing_img.squeeze(axis=0).to('cpu').detach().numpy() 75 | out_img = np.moveaxis(out_img, 0, 2) # swap channel from 1st to 3rd position: ch, _, _ -> _, _, chr 76 | 77 | # for saving_freq == -1 save only the final result (otherwise save with frequency saving_freq and save the last pic) 78 | if img_id == num_of_iterations-1 or (saving_freq > 0 and img_id % saving_freq == 0): 79 | img_format = config['img_format'] 80 | out_img_name = str(img_id).zfill(img_format[0]) + img_format[1] if saving_freq != -1 else generate_out_img_name(config) 81 | dump_img = np.copy(out_img) 82 | dump_img += np.array(IMAGENET_MEAN_255).reshape((1, 1, 3)) 83 | dump_img = np.clip(dump_img, 0, 255).astype('uint8') 84 | cv.imwrite(os.path.join(dump_path, out_img_name), dump_img[:, :, ::-1]) 85 | 86 | if should_display: 87 | plt.imshow(np.uint8(get_uint8_range(out_img))) 88 | plt.show() 89 | 90 | 91 | def get_uint8_range(x): 92 | if isinstance(x, np.ndarray): 93 | x -= np.min(x) 94 | x /= np.max(x) 95 | x *= 255 96 | return x 97 | else: 98 | raise ValueError(f'Expected numpy array got {type(x)}') 99 | 100 | 101 | # 102 | # End of image manipulation util functions 103 | # 104 | 105 | 106 | # initially it takes some time for PyTorch to download the models into local cache 107 | def prepare_model(model, device): 108 | # we are not tuning model weights -> we are only tuning optimizing_img's pixels! (that's why requires_grad=False) 109 | experimental = False 110 | if model == 'vgg16': 111 | if experimental: 112 | # much more flexible for experimenting with different style representations 113 | model = Vgg16Experimental(requires_grad=False, show_progress=True) 114 | else: 115 | model = Vgg16(requires_grad=False, show_progress=True) 116 | elif model == 'vgg19': 117 | model = Vgg19(requires_grad=False, show_progress=True) 118 | else: 119 | raise ValueError(f'{model} not supported.') 120 | 121 | content_feature_maps_index = model.content_feature_maps_index 122 | style_feature_maps_indices = model.style_feature_maps_indices 123 | layer_names = model.layer_names 124 | 125 | content_fms_index_name = (content_feature_maps_index, layer_names[content_feature_maps_index]) 126 | style_fms_indices_names = (style_feature_maps_indices, layer_names) 127 | return model.to(device).eval(), content_fms_index_name, style_fms_indices_names 128 | 129 | 130 | def gram_matrix(x, should_normalize=True): 131 | (b, ch, h, w) = x.size() 132 | features = x.view(b, ch, w * h) 133 | features_t = features.transpose(1, 2) 134 | gram = features.bmm(features_t) 135 | if should_normalize: 136 | gram /= ch * h * w 137 | return gram 138 | 139 | 140 | def total_variation(y): 141 | return torch.sum(torch.abs(y[:, :, :, :-1] - y[:, :, :, 1:])) + \ 142 | torch.sum(torch.abs(y[:, :, :-1, :] - y[:, :, 1:, :])) 143 | -------------------------------------------------------------------------------- /utils/video_utils.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | 5 | def create_video_from_intermediate_results(results_path, img_format): 6 | import shutil 7 | # 8 | # change this depending on what you want to accomplish (modify out video name, change fps and trim video) 9 | # 10 | out_file_name = 'out.mp4' 11 | fps = 30 12 | first_frame = 0 13 | number_of_frames_to_process = len(os.listdir(results_path)) # default don't trim take process every frame 14 | 15 | ffmpeg = 'ffmpeg' 16 | if shutil.which(ffmpeg): # if ffmpeg is in system path 17 | img_name_format = '%' + str(img_format[0]) + 'd' + img_format[1] # example: '%4d.png' for (4, '.png') 18 | pattern = os.path.join(results_path, img_name_format) 19 | out_video_path = os.path.join(results_path, out_file_name) 20 | 21 | trim_video_command = ['-start_number', str(first_frame), '-vframes', str(number_of_frames_to_process)] 22 | input_options = ['-r', str(fps), '-i', pattern] 23 | encoding_options = ['-c:v', 'libx264', '-crf', '25', '-pix_fmt', 'yuv420p'] 24 | subprocess.call([ffmpeg, *input_options, *trim_video_command, *encoding_options, out_video_path]) 25 | else: 26 | print(f'{ffmpeg} not found in the system path, aborting.') --------------------------------------------------------------------------------