├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── content-layer.ipynb ├── content-reconstruction.ipynb ├── frames_to_video.py ├── img ├── content-layer │ ├── block1_conv1.jpg │ ├── block2_conv1.jpg │ ├── block3_conv1.jpg │ ├── block4_conv1.jpg │ └── block5_conv1.jpg ├── content-reconstruction │ ├── block1_conv1.jpg │ ├── block2_conv1.jpg │ ├── block3_conv1.jpg │ ├── block4_conv1.jpg │ ├── block4_conv2.jpg │ └── block5_conv1.jpg ├── content │ ├── piedmont-park-night.jpg │ ├── piedmont-park.jpg │ └── tubingen.jpg ├── optimizers │ ├── Adam.jpg │ ├── GradientDescent.jpg │ ├── L_BFGS.jpg │ └── plot.png ├── photo-realistic-style-transfer │ ├── content.jpg │ ├── piedmont-park-Adam.jpg │ ├── piedmont-park-L_BFGS.jpg │ ├── piedmont-park-SGD.jpg │ └── style.jpg ├── style-layer │ ├── block1_conv1.jpg │ ├── block2_conv1.jpg │ ├── block3_conv1.jpg │ ├── block4_conv1.jpg │ └── block5_conv1.jpg ├── style-reconstruction │ ├── block1_conv1.jpg │ ├── block2_conv1.jpg │ ├── block3_conv1.jpg │ ├── block4_conv1.jpg │ └── block5_conv1.jpg ├── style-transfer.drawio.png ├── style-transfer │ ├── composition-vii.jpg │ ├── houses-of-parliament.jpg │ ├── seated-nude.jpg │ ├── the-scream.jpg │ ├── the-shipwreck-of-the-minotaur.jpg │ └── the-starry-night.jpg ├── styles-web │ ├── composition-vii.jpg │ ├── houses-of-parliament.jpg │ ├── seated-nude.jpg │ ├── the-scream.jpg │ ├── the-shipwreck-of-the-minotaur.jpg │ └── the-starry-night.jpg ├── styles │ ├── composition-vii.jpg │ ├── houses-of-parliament.jpg │ ├── seated-nude.jpg │ ├── the-scream.jpg │ ├── the-shipwreck-of-the-minotaur.jpg │ └── the-starry-night.jpg └── tv │ ├── 0.jpg │ ├── 1.jpg │ ├── 10.jpg │ ├── 100.jpg │ └── 1000.jpg ├── optimizers.ipynb ├── photo-realistic-style-transfer.ipynb ├── requirements.txt ├── src ├── __init__.py ├── callback.py ├── loss_functions.py ├── normalize.py ├── optimizers │ ├── __init__.py │ ├── adam.py │ ├── gd.py │ └── l_bfgs.py ├── reconstruct_content.py ├── reconstruct_style.py └── transfer_style.py ├── style-layers.ipynb ├── style-reconstruction.ipynb ├── style-transfer.ipynb └── tv-loss.ipynb /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | # Project files 3 | # 4 | build/ 5 | 6 | # 7 | # MacOS 8 | # 9 | # General 10 | *.DS_Store 11 | .AppleDouble 12 | .LSOverride 13 | 14 | # Icon must end with two \r 15 | Icon 16 | 17 | 18 | # Thumbnails 19 | ._* 20 | 21 | # Files that might appear in the root of a volume 22 | .DocumentRevisions-V100 23 | .fseventsd 24 | .Spotlight-V100 25 | .TemporaryItems 26 | .Trashes 27 | .VolumeIcon.icns 28 | .com.apple.timemachine.donotpresent 29 | 30 | # Directories potentially created on remote AFP share 31 | .AppleDB 32 | .AppleDesktop 33 | Network Trash Folder 34 | Temporary Items 35 | .apdisk 36 | 37 | 38 | # 39 | # Python 40 | # 41 | # Byte-compiled / optimized / DLL files 42 | __pycache__/ 43 | *.py[cod] 44 | *$py.class 45 | 46 | # C extensions 47 | *.so 48 | 49 | # Distribution / packaging 50 | MANIFEST 51 | .Python 52 | build/ 53 | develop-eggs/ 54 | dist/ 55 | downloads/ 56 | eggs/ 57 | .eggs/ 58 | lib/ 59 | lib64/ 60 | parts/ 61 | sdist/ 62 | var/ 63 | wheels/ 64 | *.egg-info/ 65 | .installed.cfg 66 | *.egg 67 | 68 | # PyInstaller 69 | # Usually these files are written by a python script from a template 70 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 71 | *.manifest 72 | *.spec 73 | 74 | # Installer logs 75 | pip-log.txt 76 | pip-delete-this-directory.txt 77 | 78 | # Unit test / coverage reports 79 | htmlcov/ 80 | .tox/ 81 | .coverage 82 | .coverage.* 83 | .cache 84 | nosetests.xml 85 | coverage.xml 86 | *.cover 87 | .hypothesis/ 88 | 89 | # Translations 90 | *.mo 91 | *.pot 92 | 93 | # Django stuff: 94 | *.log 95 | local_settings.py 96 | 97 | # Flask stuff: 98 | instance/ 99 | .webassets-cache 100 | 101 | # Scrapy stuff: 102 | .scrapy 103 | 104 | # Sphinx documentation 105 | docs/_build/ 106 | 107 | # PyBuilder 108 | target/ 109 | 110 | # Jupyter Notebook 111 | .ipynb_checkpoints 112 | 113 | # pyenv 114 | .python-version 115 | 116 | # celery beat schedule file 117 | celerybeat-schedule 118 | 119 | # SageMath parsed files 120 | *.sage.py 121 | 122 | # Environments 123 | .env 124 | .venv 125 | env/ 126 | venv/ 127 | ENV/ 128 | 129 | # Spyder project settings 130 | .spyderproject 131 | .spyproject 132 | 133 | # Rope project settings 134 | .ropeproject 135 | 136 | # mkdocs documentation 137 | /site 138 | 139 | # mypy 140 | .mypy_cache/ 141 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Christian Kauten 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # the executable for Python 2 | PYTHON=python3 3 | # the build directory 4 | BUILD=build 5 | # the number of frames of interpolation to use in videos 6 | INTER_FRAMES=3 7 | 8 | all: style_transfer_videos layer_videos reconstruction_videos optimizer_videos tv_loss_videos photo_realistic_style_transfer_videos 9 | 10 | # install Python dependencies in the requirements.txt 11 | install: 12 | ${PYTHON} -m pip install -r requirements.txt 13 | 14 | # make a build directory 15 | build: 16 | mkdir -p ${BUILD} 17 | 18 | # Convert the frames in a given directory to a video of the directories name 19 | # in its parent directory. 20 | # Args: 21 | # 1: the name of the directory in the build directory to find frames in 22 | define frames_to_video 23 | ${PYTHON} frames_to_video.py build/$(1) build/$(1).mp4 ${INTER_FRAMES} 24 | endef 25 | 26 | style_transfer_videos: build 27 | $(call frames_to_video,style-transfer/seated-nude) 28 | $(call frames_to_video,style-transfer/the-starry-night) 29 | $(call frames_to_video,style-transfer/the-scream) 30 | $(call frames_to_video,style-transfer/the-shipwreck-of-the-minotaur) 31 | $(call frames_to_video,style-transfer/composition-vii) 32 | $(call frames_to_video,style-transfer/houses-of-parliament) 33 | 34 | content_layer_videos: build 35 | $(call frames_to_video,content-layer/block1_conv1) 36 | $(call frames_to_video,content-layer/block2_conv1) 37 | $(call frames_to_video,content-layer/block3_conv1) 38 | $(call frames_to_video,content-layer/block4_conv1) 39 | $(call frames_to_video,content-layer/block5_conv1) 40 | 41 | style_layer_videos: build 42 | $(call frames_to_video,style-layer/block1_conv1) 43 | $(call frames_to_video,style-layer/block2_conv1) 44 | $(call frames_to_video,style-layer/block3_conv1) 45 | $(call frames_to_video,style-layer/block4_conv1) 46 | $(call frames_to_video,style-layer/block5_conv1) 47 | 48 | layer_videos: content_layer_videos style_layer_videos 49 | 50 | content_reconstruction_videos: build 51 | $(call frames_to_video,content-reconstruction/block1_conv1) 52 | $(call frames_to_video,content-reconstruction/block2_conv1) 53 | $(call frames_to_video,content-reconstruction/block3_conv1) 54 | $(call frames_to_video,content-reconstruction/block4_conv1) 55 | $(call frames_to_video,content-reconstruction/block4_conv2) 56 | $(call frames_to_video,content-reconstruction/block5_conv1) 57 | 58 | style_reconstruction_videos: build 59 | $(call frames_to_video,style-reconstruction/block1_conv1) 60 | $(call frames_to_video,style-reconstruction/block2_conv1) 61 | $(call frames_to_video,style-reconstruction/block3_conv1) 62 | $(call frames_to_video,style-reconstruction/block4_conv1) 63 | $(call frames_to_video,style-reconstruction/block5_conv1) 64 | 65 | reconstruction_videos: content_reconstruction_videos style_reconstruction_videos 66 | 67 | optimizer_videos: build 68 | $(call frames_to_video,optimizers/Adam) 69 | $(call frames_to_video,optimizers/GradientDescent) 70 | $(call frames_to_video,optimizers/L_BFGS) 71 | 72 | tv_loss_videos: build 73 | $(call frames_to_video,tv/0) 74 | $(call frames_to_video,tv/1) 75 | $(call frames_to_video,tv/10) 76 | $(call frames_to_video,tv/100) 77 | $(call frames_to_video,tv/1000) 78 | 79 | photo_realistic_style_transfer_videos: build 80 | $(call frames_to_video,photo-realistic-style-transfer/Adam) 81 | $(call frames_to_video,photo-realistic-style-transfer/GradientDescent) 82 | $(call frames_to_video,photo-realistic-style-transfer/L_BFGS) 83 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A Neural Algorithm of Artistic Style (Keras Implementation) 2 | 3 | An **implementation** of the arXiv preprint 4 | [_A Neural Algorithm of Artistic Style [1]_](#references) 5 | & paper 6 | [_Image Style Transfer Using Convolutional Neural Networks [2]_](#references). 7 | 8 | _Supports TensorFlow 2.4.1._ 9 | 10 |
11 | 12 |
13 | 14 | ## Style Transfer 15 | 16 | [style-transfer.ipynb](style-transfer.ipynb) describes the style transfer 17 | process between a white noise image **x**, a content image **p**, and a style 18 | representation **a**. Performing gradient descent of the content loss and 19 | style loss with respect to **x** impressions the content of **p** into **x**, 20 | bearing local styles, and colors from **a**. 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 |
Original Photograph Tubingen, Germany
Claude Monet Houses of Parliament
Pablo Picasso Seated Nude
Edvard Munch The Scream
Vincent van Gogh The Starry Night
William Turner The Shipwreck of The Minotaur
Wassily Kandinsky Composition VII
59 | 60 | ## Content Reconstruction 61 | 62 | [content-reconstruction.ipynb](content-reconstruction.ipynb) describes the 63 | content reconstruction process from white noise. Performing gradient descent 64 | of the content loss on a white noise input **x** for a given content **p** 65 | yields a representation of the networks activation for a given layer _l_. 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 |
LayerResult
block1_conv1
block2_conv1
block3_conv1
block4_conv1
block4_conv2
block5_conv1
97 | 98 | ## Style Reconstruction 99 | 100 | [style-reconstruction.ipynb](style-reconstruction.ipynb) describes the style 101 | reconstruction process on Wassily Kandinsky's Composition VII from white 102 | noise. Performing gradient descent of the style loss on a white noise input 103 | **x** for a given artwork **a** yields a representation of the networks 104 | activation for a given set of layers _L_. 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 |
LayerResult
block1_conv1
block1_conv1, block2_conv1
block1_conv1, block2_conv1, block3_conv1
block1_conv1, block2_conv1, block3_conv1, block4_conv1
block1_conv1, block2_conv1, block3_conv1, block4_conv1, block5_conv1
132 | 133 | ## Content Layer 134 | 135 | [content-layer.ipynb](content-layer.ipynb) visualizes how the style transfer 136 | is affected by using different layers for content loss. 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 |
LayerResult
block1_conv1
block2_conv1
block3_conv1
block4_conv1
block5_conv1
164 | 165 | ## Style Layers 166 | 167 | [style-layers.ipynb](style-layers.ipynb) visualizes how the style transfer is 168 | affected by using different sets of layers for style loss. 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 |
LayersResult
block1_conv1
block1_conv1, block2_conv1
block1_conv1, block2_conv1, block3_conv1
block1_conv1, block2_conv1, block3_conv1, block4_conv1
block1_conv1, block2_conv1, block3_conv1, block4_conv1, block5_conv1
196 | 197 | ## Optimizers 198 | 199 | [optimizers.ipynb](optimizers.ipynb) employs _gradient descent_, _adam_, and 200 | _L-BFGS_ to understand the effect of different black-box optimizers. Gatys et. 201 | al use L-BFGS, but Adam appears to produce comparable results without as much 202 | overhead. 203 | 204 | 205 | 206 | 207 | 208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 |
Gradient DescentAdamL-BFGS
216 | 217 |

218 | 219 |

220 | 221 | ## TV Loss 222 | 223 | [tv-loss.ipynb](tv-loss.ipynb) introduces total-variation loss to reduce 224 | impulse noise in the images. 225 | 226 | 227 | 228 | 229 | 230 | 231 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 |
TV Loss Scale FactorResult
0
1
10
100
1000
252 | 253 | ## Photo-Realistic Style Transfer 254 | 255 | [photo-realistic-style-transfer.ipynb](photo-realistic-style-transfer.ipynb) 256 | describes the photo-realistic style transfer process. Opposed to transferring 257 | style from an artwork, this notebook explores transferring a nighttime style 258 | from a picture of Piedmont Park at night to a daytime picture of Piedmont Park. 259 | 260 | 261 | 262 | 263 | 264 | 265 | 266 | 267 | 268 | 269 | 270 | 271 |
ContentStyleResult
272 | 273 | ## References 274 | 275 | [_[1] L. A. Gatys, A. S. Ecker, and M. Bethge. A neural algorithm of artistic style. arXiv preprint 276 | arXiv:1508.06576, 2015._][ref1] 277 | 278 | [ref1]: https://arxiv.org/abs/1508.06576 279 | 280 | [_[2] L. A. Gatys, A. S. Ecker, and M. Bethge. Image style transfer using convolutional neural networks. In 281 | Computer Vision and Pattern Recognition (CVPR), 2016 IEEE Conference on, pages 2414–2423. 282 | IEEE, 2016._][ref2] 283 | 284 | [ref2]: https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf 285 | -------------------------------------------------------------------------------- /frames_to_video.py: -------------------------------------------------------------------------------- 1 | """This file turns a directory of png files into video file.""" 2 | import cv2 3 | import re 4 | from sys import argv 5 | from os import listdir 6 | 7 | 8 | def pairs(iterable): 9 | """ 10 | Return a new iterable over sequential pairs in the given iterable. 11 | i.e. (0,1), (1,2), ..., (n-2,n-1) 12 | Args: 13 | iterable: the iterable to iterate over the pairs of 14 | Returns: a new iterator over the pairs of the given iterator 15 | """ 16 | # lazily import tee from `itertools` 17 | from itertools import tee 18 | # split the iterator into 2 identical iterators 19 | a, b = tee(iterable) 20 | # retrieve the next item from the b iterator 21 | next(b, None) 22 | # zip up the iterators of current and next items 23 | return zip(a, b) 24 | 25 | 26 | # the directory of png files to make into a video 27 | directory = argv[1] 28 | # the name of the video file to write 29 | video_name = argv[2] 30 | # the number of frames to use for interpolation 31 | interpolation_frames = int(argv[3]) 32 | 33 | # get the frames from the given directory in sorted order 34 | frames = [filename for filename in listdir(directory) if '.jpg' in filename] 35 | # use the integer filename value to account for the lack of 0 padding 36 | frames = sorted(frames, key=lambda x: int(re.sub(r'[^0-9]+', "", x))) 37 | # open the frames as images 38 | frames = [cv2.imread('{}/{}'.format(directory, frame)) for frame in frames] 39 | # unpack the dimensions of the images 40 | height, width, dims = frames[0].shape 41 | 42 | # setup a video output stream 43 | fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Be sure to use lower case 44 | out = cv2.VideoWriter(video_name, fourcc, 20.0, (width, height)) 45 | 46 | # write the frames to the video file 47 | for frame, next_frame in pairs(frames): 48 | # write the current frame 49 | out.write(frame) 50 | # iterate over the number of interpolation frames 51 | for i in range(1, interpolation_frames + 1): 52 | # calculate a frame between current and next frames 53 | w = i / interpolation_frames 54 | mid_frame = cv2.addWeighted(frame, 1 - w, next_frame, w, 0) 55 | # write this mid frame to the video 56 | out.write(mid_frame) 57 | 58 | # the last frame still needs written 59 | out.write(frames[-1]) 60 | 61 | # cleanup and release the video 62 | out.release() 63 | cv2.destroyAllWindows() 64 | -------------------------------------------------------------------------------- /img/content-layer/block1_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block1_conv1.jpg -------------------------------------------------------------------------------- /img/content-layer/block2_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block2_conv1.jpg -------------------------------------------------------------------------------- /img/content-layer/block3_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block3_conv1.jpg -------------------------------------------------------------------------------- /img/content-layer/block4_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block4_conv1.jpg -------------------------------------------------------------------------------- /img/content-layer/block5_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block5_conv1.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block1_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block1_conv1.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block2_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block2_conv1.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block3_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block3_conv1.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block4_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block4_conv1.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block4_conv2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block4_conv2.jpg -------------------------------------------------------------------------------- /img/content-reconstruction/block5_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block5_conv1.jpg -------------------------------------------------------------------------------- /img/content/piedmont-park-night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/piedmont-park-night.jpg -------------------------------------------------------------------------------- /img/content/piedmont-park.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/piedmont-park.jpg -------------------------------------------------------------------------------- /img/content/tubingen.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/tubingen.jpg -------------------------------------------------------------------------------- /img/optimizers/Adam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/Adam.jpg -------------------------------------------------------------------------------- /img/optimizers/GradientDescent.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/GradientDescent.jpg -------------------------------------------------------------------------------- /img/optimizers/L_BFGS.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/L_BFGS.jpg -------------------------------------------------------------------------------- /img/optimizers/plot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/plot.png -------------------------------------------------------------------------------- /img/photo-realistic-style-transfer/content.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/content.jpg -------------------------------------------------------------------------------- /img/photo-realistic-style-transfer/piedmont-park-Adam.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-Adam.jpg -------------------------------------------------------------------------------- /img/photo-realistic-style-transfer/piedmont-park-L_BFGS.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-L_BFGS.jpg -------------------------------------------------------------------------------- /img/photo-realistic-style-transfer/piedmont-park-SGD.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-SGD.jpg -------------------------------------------------------------------------------- /img/photo-realistic-style-transfer/style.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/style.jpg -------------------------------------------------------------------------------- /img/style-layer/block1_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block1_conv1.jpg -------------------------------------------------------------------------------- /img/style-layer/block2_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block2_conv1.jpg -------------------------------------------------------------------------------- /img/style-layer/block3_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block3_conv1.jpg -------------------------------------------------------------------------------- /img/style-layer/block4_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block4_conv1.jpg -------------------------------------------------------------------------------- /img/style-layer/block5_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block5_conv1.jpg -------------------------------------------------------------------------------- /img/style-reconstruction/block1_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block1_conv1.jpg -------------------------------------------------------------------------------- /img/style-reconstruction/block2_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block2_conv1.jpg -------------------------------------------------------------------------------- /img/style-reconstruction/block3_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block3_conv1.jpg -------------------------------------------------------------------------------- /img/style-reconstruction/block4_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block4_conv1.jpg -------------------------------------------------------------------------------- /img/style-reconstruction/block5_conv1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block5_conv1.jpg -------------------------------------------------------------------------------- /img/style-transfer.drawio.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer.drawio.png -------------------------------------------------------------------------------- /img/style-transfer/composition-vii.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/composition-vii.jpg -------------------------------------------------------------------------------- /img/style-transfer/houses-of-parliament.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/houses-of-parliament.jpg -------------------------------------------------------------------------------- /img/style-transfer/seated-nude.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/seated-nude.jpg -------------------------------------------------------------------------------- /img/style-transfer/the-scream.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-scream.jpg -------------------------------------------------------------------------------- /img/style-transfer/the-shipwreck-of-the-minotaur.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-shipwreck-of-the-minotaur.jpg -------------------------------------------------------------------------------- /img/style-transfer/the-starry-night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-starry-night.jpg -------------------------------------------------------------------------------- /img/styles-web/composition-vii.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/composition-vii.jpg -------------------------------------------------------------------------------- /img/styles-web/houses-of-parliament.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/houses-of-parliament.jpg -------------------------------------------------------------------------------- /img/styles-web/seated-nude.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/seated-nude.jpg -------------------------------------------------------------------------------- /img/styles-web/the-scream.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-scream.jpg -------------------------------------------------------------------------------- /img/styles-web/the-shipwreck-of-the-minotaur.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-shipwreck-of-the-minotaur.jpg -------------------------------------------------------------------------------- /img/styles-web/the-starry-night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-starry-night.jpg -------------------------------------------------------------------------------- /img/styles/composition-vii.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/composition-vii.jpg -------------------------------------------------------------------------------- /img/styles/houses-of-parliament.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/houses-of-parliament.jpg -------------------------------------------------------------------------------- /img/styles/seated-nude.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/seated-nude.jpg -------------------------------------------------------------------------------- /img/styles/the-scream.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-scream.jpg -------------------------------------------------------------------------------- /img/styles/the-shipwreck-of-the-minotaur.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-shipwreck-of-the-minotaur.jpg -------------------------------------------------------------------------------- /img/styles/the-starry-night.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-starry-night.jpg -------------------------------------------------------------------------------- /img/tv/0.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/0.jpg -------------------------------------------------------------------------------- /img/tv/1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/1.jpg -------------------------------------------------------------------------------- /img/tv/10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/10.jpg -------------------------------------------------------------------------------- /img/tv/100.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/100.jpg -------------------------------------------------------------------------------- /img/tv/1000.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/1000.jpg -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | numpy==1.19.5 2 | matplotlib==3.3.3 3 | pandas==1.1.4 4 | tensorflow==2.4.1 5 | jupyter==1.0.0 6 | tqdm==4.54.0 7 | scipy==1.6.0 8 | scikit-image==0.17.2 9 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- 1 | """An implementation of the work in _A Neural Algorithm of Artistic Style_.""" 2 | -------------------------------------------------------------------------------- /src/callback.py: -------------------------------------------------------------------------------- 1 | """A method for building a rich callback for optimizers.""" 2 | import os 3 | from glob import glob 4 | from IPython import display 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | import skimage.io as io 8 | from .normalize import denormalize 9 | 10 | 11 | class Callback: 12 | """A callback for displaying and saving images.""" 13 | 14 | def __init__(self, 15 | out_dir: str=None, 16 | extension='.png', 17 | plot=True, 18 | clear_display=True, 19 | ): 20 | """ 21 | Initialize a new callback. 22 | 23 | Args: 24 | out_dir: the name of the artwork directory to store frames in 25 | extension: the file-type to save the images as 26 | plot: whether to plot the images 27 | clear_display: whether to clear the IPython display for new plots 28 | 29 | Returns: 30 | None 31 | 32 | """ 33 | self.out_dir = out_dir 34 | self.extension = extension 35 | self.plot = plot 36 | self.clear_display = clear_display 37 | # make the directory if it doesn't exist 38 | if not os.path.exists(self.out_dir): 39 | os.makedirs(self.out_dir) 40 | # delete contents of the directory in case 41 | # it already existed with stuff in it 42 | for file in glob(f'{self.out_dir}/{self.extension}'): 43 | os.remove(file) 44 | 45 | def __call__(self, image, i): 46 | """ 47 | De-normalize an iteration of optimization to display. 48 | 49 | Args: 50 | image: the image to de-normalize and display 51 | i: the iteration of optimization 52 | 53 | Returns: 54 | None 55 | 56 | """ 57 | image = np.clip(denormalize(image[0]), 0, 255).astype('uint8') 58 | if self.out_dir is not None: 59 | io.imsave('{}/{}.jpg'.format(self.out_dir, i), image) 60 | if self.plot: 61 | if self.clear_display: 62 | display.clear_output(wait=True) 63 | ax = plt.imshow(image) 64 | ax.axes.xaxis.set_major_locator(plt.NullLocator()) 65 | ax.axes.yaxis.set_major_locator(plt.NullLocator()) 66 | plt.show() 67 | 68 | 69 | # explicitly define the outward facing API of this module 70 | __all__ = [Callback.__name__] 71 | -------------------------------------------------------------------------------- /src/loss_functions.py: -------------------------------------------------------------------------------- 1 | """Methods representing various loss functions.""" 2 | from tensorflow.keras import backend as K 3 | 4 | 5 | def content_loss(content, combination): 6 | """ 7 | Return the content loss between the content and combinations tensors. 8 | 9 | Args: 10 | content: the output of a layer for the content image 11 | combination: the output of a layer for the combination image 12 | 13 | Returns: 14 | the loss between `content` and `combination` 15 | 16 | """ 17 | # squared euclidean distance, exactly how it is in the paper 18 | return 0.5 * K.sum(K.square(combination - content)) 19 | 20 | 21 | def gram(x): 22 | """ 23 | Return a gram matrix for the given input matrix. 24 | 25 | Args: 26 | x: the matrix to calculate the gram matrix of 27 | 28 | Returns: 29 | the gram matrix of x 30 | 31 | """ 32 | # use the keras function to access shape opposed to the instance member. 33 | # this allows backward compatibility with TF1.2.1 (the version in conda) 34 | shape = K.shape(x) 35 | # flatten the 3D tensor by converting each filter's 2D matrix of points 36 | # to a vector. thus we have the matrix: 37 | # [filter_width x filter_height, num_filters] 38 | F = K.reshape(x, (shape[0] * shape[1], shape[2])) 39 | # take inner product over all the vectors to produce the Gram matrix over 40 | # the number of filters 41 | return K.dot(K.transpose(F), F) 42 | 43 | 44 | def style_loss(style, combination): 45 | """ 46 | Return the style loss between the style and combinations tensors. 47 | 48 | Args: 49 | style: the output of a layer for the style image 50 | combination: the output of a layer for the combination image 51 | 52 | Returns: 53 | the loss between `style` and `combination` 54 | 55 | """ 56 | # M_l is the width times the height of the current layer 57 | Ml2 = int(style.shape[0] * style.shape[1])**2 58 | # N_l is the number of distinct filters in the layer 59 | Nl2 = int(style.shape[2])**2 60 | 61 | # take the squared euclidean distance between the gram matrices of both 62 | # the style and combination image. divide by the constant scaling factor 63 | # based on parameterized sizes 64 | return K.sum(K.square(gram(style) - gram(combination))) / (4 * Nl2 * Ml2) 65 | 66 | 67 | def total_variation_loss(x, kind='isotropic'): 68 | """ 69 | Return the total variation loss for the image x. 70 | 71 | Args: 72 | x: the image tensor to return the variation loss of 73 | kind: the kind of total variation loss to use (default 'anisotropic') 74 | 75 | Returns: 76 | the total variation loss of the image x 77 | 78 | """ 79 | # store the dimensions for indexing from the image x 80 | h, w = x.shape[1], x.shape[2] 81 | if kind == 'anisotropic': 82 | # take the absolute value between this image, and the image one pixel 83 | # down, and one pixel to the right. take the absolute value as 84 | # specified by anisotropic loss 85 | a = K.abs(x[:, :h-1, :w-1, :] - x[:, 1:, :w-1, :]) 86 | b = K.abs(x[:, :h-1, :w-1, :] - x[:, :h-1, 1:, :]) 87 | # add up all the differences 88 | return K.sum(a + b) 89 | elif kind == 'isotropic': 90 | # take the absolute value between this image, and the image one pixel 91 | # down, and one pixel to the right. take the square root as specified 92 | # by isotropic loss 93 | a = K.square(x[:, :h-1, :w-1, :] - x[:, 1:, :w-1, :]) 94 | b = K.square(x[:, :h-1, :w-1, :] - x[:, :h-1, 1:, :]) 95 | # take the vector square root of all the pixel differences, then sum 96 | # them all up 97 | return K.sum(K.pow(a + b, 2)) 98 | else: 99 | # kind can only be two values, raise an error on unexpected kind value 100 | raise ValueError("`kind` should be 'anisotropic' or 'isotropic'") 101 | 102 | 103 | # explicitly define the outward facing API of this module 104 | __all__ = [ 105 | content_loss.__name__, 106 | style_loss.__name__, 107 | total_variation_loss.__name__, 108 | ] 109 | -------------------------------------------------------------------------------- /src/normalize.py: -------------------------------------------------------------------------------- 1 | """Image utility methods for the project.""" 2 | import numpy as np 3 | 4 | 5 | # the BGR means from the ImageNet database 6 | IMAGENET_MEANS = np.array([103.939, 116.779, 123.68]) 7 | 8 | 9 | def normalize(image: np.ndarray, inplace: bool=False) -> np.ndarray: 10 | """ 11 | Normalize an image by a set of means. 12 | 13 | Args: 14 | image: the image to normalize assuming shape [1, H, W, channels] 15 | inplace: whether to perform the operation of the array in-place 16 | 17 | Returns: 18 | image after flipping from RGB to BGR and subtracting ImageNet means 19 | 20 | """ 21 | # validate the shape of the image 22 | assert image.shape[3] == IMAGENET_MEANS.shape[0] 23 | # if in-place is enabled, copy the array 24 | if not inplace: 25 | image = image.copy() 26 | # flip image from RGB, to BGR 27 | image = image[:, :, :, ::-1] 28 | # vector subtract the means from ImageNet to the image 29 | image[:, :, :, np.arange(IMAGENET_MEANS.shape[0])] -= IMAGENET_MEANS 30 | 31 | return image 32 | 33 | 34 | def denormalize(image: np.ndarray, inplace: bool=False) -> np.ndarray: 35 | """ 36 | De-normalize an image by a set of means. 37 | 38 | Args: 39 | image: the image to normalize (assuming standard image shape in BGR) 40 | 41 | Returns: 42 | image after flipping from BGR to RGB and adding ImageNet means 43 | 44 | """ 45 | # validate the shape of the image 46 | assert image.shape[2] == IMAGENET_MEANS.shape[0] 47 | # if in-place is enabled, copy the array 48 | if not inplace: 49 | image = image.copy() 50 | # vector add the means from ImageNet to the image 51 | image[:, :, np.arange(IMAGENET_MEANS.shape[0])] += IMAGENET_MEANS 52 | # flip image from BGR, to RGB 53 | image = image[:, :, ::-1] 54 | 55 | return image 56 | 57 | 58 | # explicitly specify the public API of the module 59 | __all__ = [normalize.__name__, denormalize.__name__] 60 | -------------------------------------------------------------------------------- /src/optimizers/__init__.py: -------------------------------------------------------------------------------- 1 | """Black Box optimization methods.""" 2 | from .gd import GradientDescent 3 | from .l_bfgs import L_BFGS 4 | from .adam import Adam 5 | 6 | 7 | # export the public API for this package 8 | __all__ = [ 9 | 'GradientDescent', 10 | 'L_BFGS', 11 | 'Adam' 12 | ] 13 | -------------------------------------------------------------------------------- /src/optimizers/adam.py: -------------------------------------------------------------------------------- 1 | """An implementation of a the Adam optimizer.""" 2 | from typing import Callable 3 | from tqdm import tqdm 4 | import numpy as np 5 | 6 | 7 | class Adam(object): 8 | """The Adam optimizer.""" 9 | 10 | def __init__(self, 11 | learning_rate: float=1e-4, 12 | beta1: float=0.9, 13 | beta2: float=0.999, 14 | epsilon: float=1e-6) -> None: 15 | """ 16 | Initialize a new Adam optimizer. 17 | 18 | Args: 19 | learning_rate: how fast to adjust the parameters (dW) 20 | 21 | Returns: 22 | None 23 | 24 | """ 25 | self.learning_rate = learning_rate 26 | self.beta1 = beta1 27 | self.beta2 = beta2 28 | self.epsilon = epsilon 29 | # set the history of loss evaluations to empty list 30 | self.loss_history = [] 31 | 32 | def __repr__(self) -> str: 33 | """Return an executable string representation of this object.""" 34 | return '{}(learning_rate={}, beta1={}, beta2={})'.format(*[ 35 | self.__class__.__name__, 36 | self.learning_rate, 37 | self.beta1, 38 | self.beta2 39 | ]) 40 | 41 | def __call__(self, 42 | X: np.ndarray, 43 | shape: tuple, 44 | loss_grads: Callable, 45 | iterations: int=1000, 46 | callback: Callable=None): 47 | """ 48 | Reduce the loss generated by X by moving it based on its gradient. 49 | 50 | Args: 51 | X: the input value to adjust to minimize loss 52 | shape: the shape to coerce X to 53 | loss_grads: a callable method that returns loss and gradients 54 | given some input 55 | iterations: the number of iterations of optimization to perform 56 | callback: an optional callback method to receive image updates 57 | 58 | Returns: 59 | an optimized X about the loss and gradients given 60 | 61 | """ 62 | # setup the local data structures for the optimization 63 | moment1 = np.zeros_like(X) 64 | moment2 = np.zeros_like(X) 65 | 66 | # reset the history of loss evaluations to empty list 67 | self.loss_history = [] 68 | for i in tqdm(range(iterations)): 69 | # pass the input through the loss function and generate gradients 70 | loss_i, grads_i = loss_grads([X]) 71 | # calculate the new values of the first and second moment 72 | moment1 = self.beta1 * moment1 + (1 - self.beta1) * grads_i 73 | moment2 = self.beta2 * moment2 + (1 - self.beta1) * grads_i**2 74 | # debias the moments to avoid bias towards 0 at the beginning 75 | debias1 = moment1 / (1 - self.beta1**(i+1)) 76 | debias2 = moment2 / (1 - self.beta2**(i+1)) 77 | # move the input based on the gradients and learning rate 78 | X -= self.learning_rate * debias1 / (debias2**0.5 + self.epsilon) 79 | # update the loss history with this loss value 80 | self.loss_history.append(loss_i) 81 | # pass the values to the callback if any 82 | if callable(callback): 83 | callback(X, i) 84 | 85 | return X 86 | 87 | 88 | # explicitly define the outward facing API of this module 89 | __all__ = [Adam.__name__] 90 | -------------------------------------------------------------------------------- /src/optimizers/gd.py: -------------------------------------------------------------------------------- 1 | """An implementation of a basic gradient descent algorithm.""" 2 | from typing import Callable 3 | from tqdm import tqdm 4 | import numpy as np 5 | 6 | 7 | class GradientDescent(object): 8 | """A basic gradient descent optimizer.""" 9 | 10 | def __init__(self, learning_rate: float=1e-4) -> None: 11 | """ 12 | Initialize a new Stochastic Gradient Descent optimizer. 13 | 14 | Args: 15 | learning_rate: how fast to adjust the parameters (dW) 16 | 17 | Returns: 18 | None 19 | 20 | """ 21 | self.learning_rate = learning_rate 22 | # set the history of loss evaluations to empty list 23 | self.loss_history = [] 24 | 25 | def __repr__(self) -> str: 26 | """Return an executable string representation of this object.""" 27 | return '{}(learning_rate={})'.format(*[ 28 | self.__class__.__name__, 29 | self.learning_rate 30 | ]) 31 | 32 | def __call__(self, 33 | X: np.ndarray, 34 | shape: tuple, 35 | loss_grads: Callable, 36 | iterations: int=1000, 37 | callback: Callable=None): 38 | """ 39 | Reduce the loss generated by X by moving it based on its gradient. 40 | 41 | Args: 42 | X: the input value to adjust to minimize loss 43 | shape: the shape to coerce X to 44 | loss_grads: a callable method that returns loss and gradients 45 | given some input 46 | iterations: the number of iterations of optimization to perform 47 | callback: an optional callback method to receive image updates 48 | 49 | Returns: 50 | an optimized X about the loss and gradients given 51 | 52 | """ 53 | # reset the history of loss evaluations to empty list 54 | self.loss_history = [] 55 | for i in tqdm(range(iterations)): 56 | # pass the input through the loss function and generate gradients 57 | loss_i, grads_i = loss_grads([X]) 58 | # move the input based on the gradients and learning rate 59 | X -= self.learning_rate * grads_i 60 | # update the loss history with this loss value 61 | self.loss_history.append(loss_i) 62 | # pass the values to the callback if any 63 | if callable(callback): 64 | callback(X, i) 65 | 66 | return X 67 | 68 | 69 | # explicitly define the outward facing API of this module 70 | __all__ = [GradientDescent.__name__] 71 | -------------------------------------------------------------------------------- /src/optimizers/l_bfgs.py: -------------------------------------------------------------------------------- 1 | """An interface to the L-BFGS Algorithm.""" 2 | import numpy as np 3 | from typing import Callable 4 | from tqdm import tqdm 5 | from scipy.optimize import fmin_l_bfgs_b 6 | 7 | 8 | class L_BFGS(object): 9 | """L-BFGS Optimization Algorithm.""" 10 | 11 | def __init__(self, max_evaluations: int=20) -> None: 12 | """ 13 | Initialize a new L-BFGS Hill climbing optimizer. 14 | 15 | Args: 16 | max_evaluations: how fast to adjust the parameters (dW) 17 | 18 | Returns: 19 | None 20 | 21 | """ 22 | self.max_evaluations = max_evaluations 23 | self._loss = None 24 | self._gradients = None 25 | # set the history of loss evaluations to empty list 26 | self.loss_history = [] 27 | 28 | def __repr__(self) -> str: 29 | """Return an executable string representation of this object.""" 30 | return '{}(max_evaluations={})'.format(*[ 31 | self.__class__.__name__, 32 | self.max_evaluations 33 | ]) 34 | 35 | def loss(self, X) -> np.ndarray: 36 | """Calculate the loss given some X.""" 37 | # make sure this is called _only after gradients_ 38 | assert self._loss is None 39 | # calculate and store both the loss and the gradients 40 | loss, gradients = self.loss_and_gradients(X) 41 | # update the cache for this pass 42 | self._loss = loss 43 | self._gradients = gradients 44 | 45 | return self._loss 46 | 47 | def gradients(self, X) -> np.ndarray: 48 | """Calculate the gradients (lazily) given some X.""" 49 | # make sure this is called _only after loss_ 50 | assert self._loss is not None 51 | # copy the gradients 52 | gradients = np.copy(self._gradients) 53 | # nullify the cache for this pass 54 | self._loss = None 55 | self._gradients = None 56 | 57 | return gradients 58 | 59 | def __call__(self, 60 | X: np.ndarray, 61 | shape: tuple, 62 | loss_grads: Callable, 63 | iterations: int=1000, 64 | callback: Callable=None) -> np.ndarray: 65 | """ 66 | Reduce the loss generated by X. 67 | 68 | Args: 69 | X: the input value to adjust to minimize loss 70 | shape: the shape to coerce X to 71 | loss_grads: a callable method that returns loss and gradients 72 | given some input 73 | iterations: the number of iterations of optimization to perform 74 | callback: an optional callback method to receive image updates 75 | 76 | Returns: 77 | an optimized X about the loss and gradients given 78 | 79 | """ 80 | # reset the history of loss evaluations to empty list 81 | self.loss_history = [] 82 | # create a loss and gradients evaluation method for SciPy 83 | def loss_and_gradients(X): 84 | """Calculate the loss and gradients with appropriate reshaping.""" 85 | loss, gradients = loss_grads([X.reshape(shape)]) 86 | return loss, gradients.flatten().astype('float64') 87 | 88 | # assign the custom method to self for loss / gradient calculation 89 | self.loss_and_gradients = loss_and_gradients 90 | 91 | for i in tqdm(range(iterations)): 92 | # pass X through an iteration of LBFGS 93 | X, min_val, info = fmin_l_bfgs_b(self.loss, X, 94 | fprime=self.gradients, 95 | maxfun=self.max_evaluations) 96 | # update the loss history with this loss value 97 | self.loss_history += info['funcalls'] * [min_val] 98 | # pass the values to the callback if any 99 | if callable(callback): 100 | callback(X.reshape(shape), i) 101 | 102 | return X 103 | 104 | 105 | # explicitly define the outward facing API of this module 106 | __all__ = [L_BFGS.__name__] 107 | -------------------------------------------------------------------------------- /src/reconstruct_content.py: -------------------------------------------------------------------------------- 1 | """A functional decomposition of the content reconstruction algorithm.""" 2 | from typing import Callable 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.keras import backend as K 6 | from tensorflow.keras.applications.vgg19 import VGG19 7 | from .normalize import normalize, denormalize 8 | from .loss_functions import content_loss 9 | from .optimizers.l_bfgs import L_BFGS 10 | 11 | 12 | def reconstruct_content(content: np.ndarray, 13 | layer_name: str='block4_conv2', 14 | optimize: Callable=L_BFGS(), 15 | iterations: int=10, 16 | noise_range: tuple=(0, 1), 17 | callback: Callable=None 18 | ): 19 | """ 20 | Reconstruct the given content image at the given VGG19 layer. 21 | 22 | Args: 23 | content: the content image to reconstruct 24 | layer_name: the layer to reconstruct the content from 25 | optimize: the optimization method for minimizing the content loss 26 | iterations: the number of iterations to run the optimizer 27 | noise_range: the range of values for initializing random noise 28 | callback: the callback for iterations of gradient descent 29 | 30 | Returns: 31 | the reconstructed content image based on the VGG19 response 32 | at the given layer name 33 | 34 | """ 35 | # disable eager mode for this operation 36 | tf.compat.v1.disable_eager_execution() 37 | 38 | # normalize the image's RGB values about the RGB channel means for the 39 | # ImageNet dataset 40 | content = normalize(content[None, ...].astype('float')) 41 | # load the content image into Keras as a constant, it never changes 42 | content = K.constant(content, name='Content') 43 | # create a placeholder for the trained image, this variable trains 44 | canvas = K.placeholder(content.shape, name='Canvas') 45 | # combine the content and canvas tensors along the frame axis (0) into a 46 | # 4D tensor of shape [2, height, width, channels] 47 | input_tensor = K.concatenate([content, canvas], axis=0) 48 | # build the model with the 4D input tensor of content and canvas 49 | model = VGG19(include_top=False, input_tensor=input_tensor, pooling='avg') 50 | 51 | # extract the layer's out that we have interest in for reconstruction 52 | layer = model.get_layer(layer_name) 53 | 54 | # calculate the loss between the output of the layer on the content (0) 55 | # and the canvas (1) 56 | loss = content_loss(layer.output[0], layer.output[1]) 57 | # calculate the gradients 58 | grads = K.gradients(loss, canvas)[0] 59 | # generate the iteration function for gradient descent optimization 60 | step = K.function([canvas], [loss, grads]) 61 | 62 | # generate random noise with the given noise range 63 | noise = np.random.uniform(*noise_range, size=content.shape) 64 | 65 | # optimize the white noise to reconstruct the content 66 | image = optimize(noise, canvas.shape, step, iterations, callback) 67 | 68 | # clear the Keras session 69 | K.clear_session() 70 | 71 | # de-normalize the image (from ImageNet means) and convert back to binary 72 | return np.clip(denormalize(image.reshape(canvas.shape)[0]), 0, 255).astype('uint8') 73 | 74 | 75 | # explicitly define the outward facing API of this module 76 | __all__ = [reconstruct_content.__name__] 77 | -------------------------------------------------------------------------------- /src/reconstruct_style.py: -------------------------------------------------------------------------------- 1 | """A functional decomposition of the style reconstruction algorithm.""" 2 | from typing import Callable 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.keras import backend as K 6 | from tensorflow.keras.applications.vgg19 import VGG19 7 | from .normalize import normalize, denormalize 8 | from .loss_functions import style_loss 9 | from .optimizers.l_bfgs import L_BFGS 10 | 11 | 12 | def reconstruct_style(style: np.ndarray, 13 | layer_names: list = [ 14 | 'block1_conv1', 15 | 'block2_conv1', 16 | 'block3_conv1', 17 | 'block4_conv1', 18 | 'block5_conv1' 19 | ], 20 | optimize: Callable = L_BFGS(), 21 | iterations: int = 10, 22 | noise_range: tuple = (0, 1), 23 | callback: Callable = None 24 | ): 25 | """ 26 | Reconstruct the given content image at the given VGG19 layer. 27 | 28 | Args: 29 | style: the style to reconstruct 30 | layer_name: the layer to reconstruct the content from 31 | optimizer: the optimizer for minimizing the content loss 32 | iterations: the number of iterations to run the optimizer 33 | noise_range: the range of values for initializing random noise 34 | callback: the callback for iterations of gradient descent 35 | 36 | Returns: 37 | the reconstructed content image based on the VGG19 response at the 38 | given layer name 39 | 40 | """ 41 | # disable eager mode for this operation 42 | tf.compat.v1.disable_eager_execution() 43 | 44 | # normalize the image's RGB values about the RGB channel means for the 45 | # ImageNet dataset 46 | style = normalize(style[None, ...].astype('float')) 47 | 48 | # load the style image into Keras as a constant, it never changes 49 | style = K.constant(style, name='Style') 50 | # create a placeholder for the trained image, this variable trains 51 | canvas = K.placeholder(style.shape, name='Canvas') 52 | # combine the style and canvas tensors along the frame axis (0) into a 53 | # 4D tensor of shape [2, height, width, channels] 54 | input_tensor = K.concatenate([style, canvas], axis=0) 55 | # build the model with the 4D input tensor of style and canvas 56 | model = VGG19(include_top=False, input_tensor=input_tensor, pooling='avg') 57 | 58 | # initialize the loss to accumulate iteratively over the layers 59 | loss = K.variable(0.0) 60 | 61 | # iterate over the list of all the layers that we want to include 62 | for layer_name in layer_names: 63 | # extract the layer's out that we have interest in for reconstruction 64 | layer = model.get_layer(layer_name) 65 | # calculate the loss between the output of the layer on the style (0) 66 | # and the canvas (1). The style loss needs to know the size of the 67 | # image as well by width (shape[2]) and height (shape[1]) 68 | loss = loss + style_loss(layer.output[0], layer.output[1]) 69 | 70 | # Gatys et al. use a w_l of 1/5 for their example with the 5 layers. As 71 | # such, we'll simply and say for any length of layers, just take the 72 | # average. (mirroring what they did) 73 | loss /= len(layer_names) 74 | 75 | # calculate the gradients 76 | grads = K.gradients(loss, canvas)[0] 77 | # generate the iteration function for gradient descent optimization 78 | step = K.function([canvas], [loss, grads]) 79 | 80 | # generate random noise as the starting point 81 | noise = np.random.uniform(*noise_range, size=canvas.shape) 82 | 83 | # optimize the white noise to reconstruct the content 84 | image = optimize(noise, canvas.shape, step, iterations, callback) 85 | 86 | # clear the Keras session 87 | K.clear_session() 88 | 89 | # de-normalize the image (from ImageNet means) and convert back to binary 90 | return np.clip(denormalize(image.reshape(canvas.shape)[0]), 0, 255).astype('uint8') 91 | 92 | 93 | # explicitly define the outward facing API of this module 94 | __all__ = [reconstruct_style.__name__] 95 | -------------------------------------------------------------------------------- /src/transfer_style.py: -------------------------------------------------------------------------------- 1 | """A mechanism for transferring style of art to content.""" 2 | from typing import Callable, List 3 | import numpy as np 4 | import tensorflow as tf 5 | from tensorflow.keras import backend as K 6 | from tensorflow.keras.applications.vgg19 import VGG19 7 | from .normalize import normalize, denormalize 8 | from .loss_functions import content_loss, style_loss, total_variation_loss 9 | 10 | 11 | # the template for the class's __repr__ method 12 | TEMPLATE = """{}( 13 | content_layer_name={}, 14 | content_weight={}, 15 | style_layer_names={}, 16 | style_layer_weights={}, 17 | style_weight={}, 18 | total_variation_weight={} 19 | )""".lstrip() 20 | 21 | 22 | class Stylizer(object): 23 | """An algorithm for stylizing images based on artwork.""" 24 | 25 | def __init__(self, 26 | content_layer_name: str='block4_conv2', 27 | content_weight: float=1.0, 28 | style_layer_names: List[str]=[ 29 | 'block1_conv1', 30 | 'block2_conv1', 31 | 'block3_conv1', 32 | 'block4_conv1', 33 | 'block5_conv1' 34 | ], 35 | style_layer_weights: List[float]=None, 36 | style_weight: float=10000.0, 37 | total_variation_weight: float=0.0 38 | ) -> None: 39 | """ 40 | Initialize a new neural stylization algorithm. 41 | 42 | Args: 43 | content_layer_name: the name of the layer to extract content from 44 | content_weight: the weight, alpha, to attribute to content loss 45 | style_layer_names: the names of the layers to extract style from 46 | style_weight: the weight, beta, to attribute to style loss 47 | style_layer_weights: the set of weights to apply to the individual 48 | losses from each style layer. If None, the default is to take 49 | the average, i.e. divide each by len(style_layer_names). 50 | total_variation_weight: the amount of total variation de-noising 51 | to apply to the synthetic images 52 | 53 | Returns: 54 | None 55 | 56 | """ 57 | # get the names of the layers from the model to error check 58 | layer_names = [l.name for l in VGG19(include_top=False).layers] 59 | 60 | # type and value check: content_layer_name 61 | if not isinstance(content_layer_name, str): 62 | raise TypeError('`content_layer_name` must be of type: str') 63 | if content_layer_name not in layer_names: 64 | raise ValueError( 65 | '`content_layer_name` must be a layer name in VGG19' 66 | ) 67 | self.content_layer_name = content_layer_name 68 | 69 | # type and value check: content_weight 70 | if not isinstance(content_weight, (int, float)): 71 | raise TypeError('`content_weight` must be of type: int or float') 72 | if content_weight < 0: 73 | raise ValueError('`content_weight` must be >= 0') 74 | self.content_weight = content_weight 75 | 76 | # type and value check: content_layer_name 77 | if not isinstance(style_layer_names, list): 78 | raise TypeError('`style_layer_names` must be of type: list') 79 | if not all(layer in layer_names for layer in style_layer_names): 80 | raise ValueError( 81 | '`style_layer_names` must be a list of layer names in VGG19' 82 | ) 83 | self.style_layer_names = style_layer_names 84 | 85 | # type and value check: style_layer_weights 86 | if style_layer_weights is None: 87 | # initialize style layer weights as an average between them. 88 | total = len(style_layer_names) 89 | style_layer_weights = total * [1.0 / total] 90 | else: 91 | if not isinstance(style_layer_weights, list): 92 | raise TypeError( 93 | '`style_layer_weights` must be of type: None or list' 94 | ) 95 | if not all(isinstance(w, (float, int)) for w in style_layer_weights): 96 | raise ValueError( 97 | '`style_layer_weights` must be a list of numbers or None' 98 | ) 99 | self.style_layer_weights = style_layer_weights 100 | 101 | # type and value check: style_weight 102 | if not isinstance(style_weight, (int, float)): 103 | raise TypeError('`style_weight` must be of type: int or float') 104 | if style_weight < 0: 105 | raise ValueError('`style_weight` must be >= 0') 106 | self.style_weight = style_weight 107 | 108 | # type and value check: total_variation_weight 109 | if not isinstance(total_variation_weight, (int, float)): 110 | raise TypeError( 111 | '`total_variation_weight` must be of type: int or float' 112 | ) 113 | if total_variation_weight < 0: 114 | raise ValueError('`total_variation_weight` must be >= 0') 115 | self.total_variation_weight = total_variation_weight 116 | 117 | # disable eager mode for this operation 118 | tf.compat.v1.disable_eager_execution() 119 | 120 | def __repr__(self) -> str: 121 | """Return an executable string representation of this object.""" 122 | return TEMPLATE.format(*[ 123 | self.__class__.__name__, 124 | repr(self.content_layer_name), 125 | self.content_weight, 126 | self.style_layer_names, 127 | self.style_layer_weights, 128 | self.style_weight, 129 | self.total_variation_weight 130 | ]) 131 | 132 | @property 133 | def content_style_ratio(self) -> float: 134 | """Return the ratio of content weight to style weight.""" 135 | return self.content_weight / self.style_weight 136 | 137 | def _build_model(self, content: np.ndarray, style: np.ndarray) -> tuple: 138 | """ 139 | Build a synthesis model with the given content and style. 140 | 141 | Args: 142 | content: the content to fuse the artwork into 143 | style: the artwork to get the style from 144 | 145 | Returns: 146 | a tuple of: 147 | - the constructed VGG19 model from the input images 148 | - the canvas tensor for the synthesized image 149 | 150 | """ 151 | # load the content image into Keras as a constant, it never changes 152 | content_tensor = K.constant(content, name='Content') 153 | # load the style image into Keras as a constant, it never changes 154 | style_tensor = K.constant(style, name='Style') 155 | # create a placeholder for the trained image, this variable changes 156 | canvas = K.placeholder(content.shape, name='Canvas') 157 | # combine the content, style, and canvas tensors along the frame 158 | # axis (0) into a 4D tensor of shape [3, height, width, channels] 159 | tensor = K.concatenate([content_tensor, style_tensor, canvas], axis=0) 160 | # build the model with the input tensor of content, style, and canvas 161 | model = VGG19(include_top=False, input_tensor=tensor, pooling='avg') 162 | 163 | return model, canvas 164 | 165 | def _build_loss_grads(self, model, canvas: 'Tensor') -> Callable: 166 | """ 167 | Build the optimization methods for stylizing the image from a model. 168 | 169 | Args: 170 | model: the model to extract layers from 171 | canvas: the input to the model thats being mutated 172 | 173 | Returns: 174 | a function to calculate loss and gradients from an input X 175 | 176 | """ 177 | # initialize a variable to store the loss into 178 | loss = K.variable(0.0, name='Loss') 179 | 180 | # CONTENT LOSS 181 | # extract the content layer tensor for optimizing content loss 182 | content_layer_output = model.get_layer(self.content_layer_name).output 183 | # calculate the loss between the output of the layer on the 184 | # content (0) and the canvas (2) 185 | cl = content_loss(content_layer_output[0], 186 | content_layer_output[2]) 187 | loss = loss + self.content_weight * cl 188 | 189 | # STYLE LOSS 190 | sl = K.variable(0.0) 191 | # iterate over the list of all the layers that we want to include 192 | for style_layer_name, layer_weight in zip(self.style_layer_names, 193 | self.style_layer_weights): 194 | # extract the layer out that we have interest in 195 | style_layer_output = model.get_layer(style_layer_name).output 196 | # calculate the loss between the output of the layer on the 197 | # style (1) and the canvas (2). 198 | sl = sl + layer_weight * style_loss(style_layer_output[1], 199 | style_layer_output[2]) 200 | # apply the style weight to style loss and add it to the total loss 201 | loss = loss + self.style_weight * sl 202 | 203 | # TOTAL VARIATION LOSS 204 | # Gatys et al. don't use the total variation de-noising in their paper 205 | # (or at least they never mention it) so the weight is 0.0 by 206 | # default, but can be applied if desired 207 | loss = loss + self.total_variation_weight * total_variation_loss(canvas) 208 | 209 | # GRADIENTS 210 | # calculate the gradients of the input image with respect to 211 | # the loss. i.e. back-propagate the loss through the network 212 | # to the input layer (only the canvas though) 213 | grads = K.gradients(loss, canvas)[0] 214 | 215 | # generate the iteration function for gradient descent optimization 216 | # Args: 217 | # noise: the input to the noise placeholder in the model 218 | # this effectively takes a the white noise image being 219 | # optimized and passes it forward and backward through 220 | # the model collecting the loss and gradient along the 221 | # way 222 | # 223 | # Returns: 224 | # a tuple of (loss, gradients) 225 | # - loss: the content loss between the content, style image 226 | # and the white noise 227 | # - gradients: the gradients of the inputs with respect 228 | # to the loss 229 | return K.function([canvas], [loss, grads]) 230 | 231 | def __call__(self, content: np.ndarray, style: np.ndarray, optimize: Callable, 232 | iterations: int = 10, 233 | initialization_strat: str = 'noise', 234 | noise_range: tuple = (0, 1), 235 | callback: Callable = None 236 | ): 237 | """ 238 | Stylize the given content image with the give style image. 239 | 240 | Args: 241 | content: the content image to use 242 | style: the style image to use 243 | optimize: the black-box optimizer to use. This is a callable 244 | method conforming to the API for optimizers 245 | iterations: the number of optimization iterations to perform 246 | image_size: the custom size to load images with, if any. When 247 | set to None, the size of the content image will be used 248 | initialization_strat: the way to initialize the canvas for the 249 | style transfer. Can be one of: 250 | - 'noise': initialize the canvas as random noise with the 251 | given noise range for sampling pixels 252 | - 'content': initialize the canvas as the content image 253 | - 'style': initialize the canvas as the style image 254 | noise_range: the custom range for initializing random noise. This 255 | option is only used when `initialization_strat` is 'noise'. 256 | callback: the optional callback method for optimizer iterations 257 | 258 | Returns: 259 | the image as a result of blending content with style 260 | 261 | """ 262 | # normalize the input data 263 | content = normalize(content[None, ...].astype('float')) 264 | style = normalize(style[None, ...].astype('float')) 265 | 266 | # disable eager mode for this operation 267 | tf.compat.v1.disable_eager_execution() 268 | # build the inputs tensor from the images 269 | model, canvas = self._build_model(content, style) 270 | # build the iteration function 271 | loss_grads = self._build_loss_grads(model, canvas) 272 | 273 | # setup the initial image for the optimizer 274 | if initialization_strat == 'noise': 275 | # generate white noise in the shape of the canvas 276 | initial = np.random.uniform(*noise_range, size=canvas.shape) 277 | elif initialization_strat == 'content': 278 | # copy the content as the initial image 279 | initial = content.copy() 280 | elif initialization_strat == 'style': 281 | # copy the style as the initial image 282 | initial = style.copy() 283 | else: 284 | raise ValueError( 285 | "`initialization_strat` must be one of: ", 286 | " 'noise', 'content', 'style' " 287 | ) 288 | 289 | # optimize the initial image into a synthetic painting. Name all args 290 | # by keyword to help catch erroneous optimize callables. 291 | image = optimize( 292 | X=initial, 293 | shape=canvas.shape, 294 | loss_grads=loss_grads, 295 | iterations=iterations, 296 | callback=callback 297 | ) 298 | 299 | # clear the Keras session (this removes the variables from memory). 300 | K.clear_session() 301 | 302 | # reshape the image in case the optimizer did something funky with the 303 | # shape. `denormalize` expects a vector of shape [h, w, c], but 304 | # canvas has an additional dimension for frame, [frame, h, w, c]. Ss 305 | # such, take the first item along the frame axis 306 | image = image.reshape(canvas.shape)[0] 307 | # denormalize the image about the ImageNet means. this will invert the 308 | # channel dimension turning image from BGR to RGB 309 | return np.clip(denormalize(image), 0, 255).astype('uint8') 310 | 311 | 312 | # explicitly define the outward facing API of this module 313 | __all__ = [Stylizer.__name__] 314 | --------------------------------------------------------------------------------