├── .gitignore
├── LICENSE
├── Makefile
├── README.md
├── content-layer.ipynb
├── content-reconstruction.ipynb
├── frames_to_video.py
├── img
├── content-layer
│ ├── block1_conv1.jpg
│ ├── block2_conv1.jpg
│ ├── block3_conv1.jpg
│ ├── block4_conv1.jpg
│ └── block5_conv1.jpg
├── content-reconstruction
│ ├── block1_conv1.jpg
│ ├── block2_conv1.jpg
│ ├── block3_conv1.jpg
│ ├── block4_conv1.jpg
│ ├── block4_conv2.jpg
│ └── block5_conv1.jpg
├── content
│ ├── piedmont-park-night.jpg
│ ├── piedmont-park.jpg
│ └── tubingen.jpg
├── optimizers
│ ├── Adam.jpg
│ ├── GradientDescent.jpg
│ ├── L_BFGS.jpg
│ └── plot.png
├── photo-realistic-style-transfer
│ ├── content.jpg
│ ├── piedmont-park-Adam.jpg
│ ├── piedmont-park-L_BFGS.jpg
│ ├── piedmont-park-SGD.jpg
│ └── style.jpg
├── style-layer
│ ├── block1_conv1.jpg
│ ├── block2_conv1.jpg
│ ├── block3_conv1.jpg
│ ├── block4_conv1.jpg
│ └── block5_conv1.jpg
├── style-reconstruction
│ ├── block1_conv1.jpg
│ ├── block2_conv1.jpg
│ ├── block3_conv1.jpg
│ ├── block4_conv1.jpg
│ └── block5_conv1.jpg
├── style-transfer.drawio.png
├── style-transfer
│ ├── composition-vii.jpg
│ ├── houses-of-parliament.jpg
│ ├── seated-nude.jpg
│ ├── the-scream.jpg
│ ├── the-shipwreck-of-the-minotaur.jpg
│ └── the-starry-night.jpg
├── styles-web
│ ├── composition-vii.jpg
│ ├── houses-of-parliament.jpg
│ ├── seated-nude.jpg
│ ├── the-scream.jpg
│ ├── the-shipwreck-of-the-minotaur.jpg
│ └── the-starry-night.jpg
├── styles
│ ├── composition-vii.jpg
│ ├── houses-of-parliament.jpg
│ ├── seated-nude.jpg
│ ├── the-scream.jpg
│ ├── the-shipwreck-of-the-minotaur.jpg
│ └── the-starry-night.jpg
└── tv
│ ├── 0.jpg
│ ├── 1.jpg
│ ├── 10.jpg
│ ├── 100.jpg
│ └── 1000.jpg
├── optimizers.ipynb
├── photo-realistic-style-transfer.ipynb
├── requirements.txt
├── src
├── __init__.py
├── callback.py
├── loss_functions.py
├── normalize.py
├── optimizers
│ ├── __init__.py
│ ├── adam.py
│ ├── gd.py
│ └── l_bfgs.py
├── reconstruct_content.py
├── reconstruct_style.py
└── transfer_style.py
├── style-layers.ipynb
├── style-reconstruction.ipynb
├── style-transfer.ipynb
└── tv-loss.ipynb
/.gitignore:
--------------------------------------------------------------------------------
1 | #
2 | # Project files
3 | #
4 | build/
5 |
6 | #
7 | # MacOS
8 | #
9 | # General
10 | *.DS_Store
11 | .AppleDouble
12 | .LSOverride
13 |
14 | # Icon must end with two \r
15 | Icon
16 |
17 |
18 | # Thumbnails
19 | ._*
20 |
21 | # Files that might appear in the root of a volume
22 | .DocumentRevisions-V100
23 | .fseventsd
24 | .Spotlight-V100
25 | .TemporaryItems
26 | .Trashes
27 | .VolumeIcon.icns
28 | .com.apple.timemachine.donotpresent
29 |
30 | # Directories potentially created on remote AFP share
31 | .AppleDB
32 | .AppleDesktop
33 | Network Trash Folder
34 | Temporary Items
35 | .apdisk
36 |
37 |
38 | #
39 | # Python
40 | #
41 | # Byte-compiled / optimized / DLL files
42 | __pycache__/
43 | *.py[cod]
44 | *$py.class
45 |
46 | # C extensions
47 | *.so
48 |
49 | # Distribution / packaging
50 | MANIFEST
51 | .Python
52 | build/
53 | develop-eggs/
54 | dist/
55 | downloads/
56 | eggs/
57 | .eggs/
58 | lib/
59 | lib64/
60 | parts/
61 | sdist/
62 | var/
63 | wheels/
64 | *.egg-info/
65 | .installed.cfg
66 | *.egg
67 |
68 | # PyInstaller
69 | # Usually these files are written by a python script from a template
70 | # before PyInstaller builds the exe, so as to inject date/other infos into it.
71 | *.manifest
72 | *.spec
73 |
74 | # Installer logs
75 | pip-log.txt
76 | pip-delete-this-directory.txt
77 |
78 | # Unit test / coverage reports
79 | htmlcov/
80 | .tox/
81 | .coverage
82 | .coverage.*
83 | .cache
84 | nosetests.xml
85 | coverage.xml
86 | *.cover
87 | .hypothesis/
88 |
89 | # Translations
90 | *.mo
91 | *.pot
92 |
93 | # Django stuff:
94 | *.log
95 | local_settings.py
96 |
97 | # Flask stuff:
98 | instance/
99 | .webassets-cache
100 |
101 | # Scrapy stuff:
102 | .scrapy
103 |
104 | # Sphinx documentation
105 | docs/_build/
106 |
107 | # PyBuilder
108 | target/
109 |
110 | # Jupyter Notebook
111 | .ipynb_checkpoints
112 |
113 | # pyenv
114 | .python-version
115 |
116 | # celery beat schedule file
117 | celerybeat-schedule
118 |
119 | # SageMath parsed files
120 | *.sage.py
121 |
122 | # Environments
123 | .env
124 | .venv
125 | env/
126 | venv/
127 | ENV/
128 |
129 | # Spyder project settings
130 | .spyderproject
131 | .spyproject
132 |
133 | # Rope project settings
134 | .ropeproject
135 |
136 | # mkdocs documentation
137 | /site
138 |
139 | # mypy
140 | .mypy_cache/
141 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2021 Christian Kauten
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
1 | # the executable for Python
2 | PYTHON=python3
3 | # the build directory
4 | BUILD=build
5 | # the number of frames of interpolation to use in videos
6 | INTER_FRAMES=3
7 |
8 | all: style_transfer_videos layer_videos reconstruction_videos optimizer_videos tv_loss_videos photo_realistic_style_transfer_videos
9 |
10 | # install Python dependencies in the requirements.txt
11 | install:
12 | ${PYTHON} -m pip install -r requirements.txt
13 |
14 | # make a build directory
15 | build:
16 | mkdir -p ${BUILD}
17 |
18 | # Convert the frames in a given directory to a video of the directories name
19 | # in its parent directory.
20 | # Args:
21 | # 1: the name of the directory in the build directory to find frames in
22 | define frames_to_video
23 | ${PYTHON} frames_to_video.py build/$(1) build/$(1).mp4 ${INTER_FRAMES}
24 | endef
25 |
26 | style_transfer_videos: build
27 | $(call frames_to_video,style-transfer/seated-nude)
28 | $(call frames_to_video,style-transfer/the-starry-night)
29 | $(call frames_to_video,style-transfer/the-scream)
30 | $(call frames_to_video,style-transfer/the-shipwreck-of-the-minotaur)
31 | $(call frames_to_video,style-transfer/composition-vii)
32 | $(call frames_to_video,style-transfer/houses-of-parliament)
33 |
34 | content_layer_videos: build
35 | $(call frames_to_video,content-layer/block1_conv1)
36 | $(call frames_to_video,content-layer/block2_conv1)
37 | $(call frames_to_video,content-layer/block3_conv1)
38 | $(call frames_to_video,content-layer/block4_conv1)
39 | $(call frames_to_video,content-layer/block5_conv1)
40 |
41 | style_layer_videos: build
42 | $(call frames_to_video,style-layer/block1_conv1)
43 | $(call frames_to_video,style-layer/block2_conv1)
44 | $(call frames_to_video,style-layer/block3_conv1)
45 | $(call frames_to_video,style-layer/block4_conv1)
46 | $(call frames_to_video,style-layer/block5_conv1)
47 |
48 | layer_videos: content_layer_videos style_layer_videos
49 |
50 | content_reconstruction_videos: build
51 | $(call frames_to_video,content-reconstruction/block1_conv1)
52 | $(call frames_to_video,content-reconstruction/block2_conv1)
53 | $(call frames_to_video,content-reconstruction/block3_conv1)
54 | $(call frames_to_video,content-reconstruction/block4_conv1)
55 | $(call frames_to_video,content-reconstruction/block4_conv2)
56 | $(call frames_to_video,content-reconstruction/block5_conv1)
57 |
58 | style_reconstruction_videos: build
59 | $(call frames_to_video,style-reconstruction/block1_conv1)
60 | $(call frames_to_video,style-reconstruction/block2_conv1)
61 | $(call frames_to_video,style-reconstruction/block3_conv1)
62 | $(call frames_to_video,style-reconstruction/block4_conv1)
63 | $(call frames_to_video,style-reconstruction/block5_conv1)
64 |
65 | reconstruction_videos: content_reconstruction_videos style_reconstruction_videos
66 |
67 | optimizer_videos: build
68 | $(call frames_to_video,optimizers/Adam)
69 | $(call frames_to_video,optimizers/GradientDescent)
70 | $(call frames_to_video,optimizers/L_BFGS)
71 |
72 | tv_loss_videos: build
73 | $(call frames_to_video,tv/0)
74 | $(call frames_to_video,tv/1)
75 | $(call frames_to_video,tv/10)
76 | $(call frames_to_video,tv/100)
77 | $(call frames_to_video,tv/1000)
78 |
79 | photo_realistic_style_transfer_videos: build
80 | $(call frames_to_video,photo-realistic-style-transfer/Adam)
81 | $(call frames_to_video,photo-realistic-style-transfer/GradientDescent)
82 | $(call frames_to_video,photo-realistic-style-transfer/L_BFGS)
83 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # A Neural Algorithm of Artistic Style (Keras Implementation)
2 |
3 | An **implementation** of the arXiv preprint
4 | [_A Neural Algorithm of Artistic Style [1]_](#references)
5 | & paper
6 | [_Image Style Transfer Using Convolutional Neural Networks [2]_](#references).
7 |
8 | _Supports TensorFlow 2.4.1._
9 |
10 |
11 |

12 |
13 |
14 | ## Style Transfer
15 |
16 | [style-transfer.ipynb](style-transfer.ipynb) describes the style transfer
17 | process between a white noise image **x**, a content image **p**, and a style
18 | representation **a**. Performing gradient descent of the content loss and
19 | style loss with respect to **x** impressions the content of **p** into **x**,
20 | bearing local styles, and colors from **a**.
21 |
22 |
23 |
24 | Original Photograph Tubingen, Germany |
25 | |
26 |  |
27 |
28 |
29 | Claude Monet Houses of Parliament |
30 |  |
31 |  |
32 |
33 |
34 | Pablo Picasso Seated Nude |
35 |  |
36 |  |
37 |
38 |
39 | Edvard Munch The Scream |
40 |  |
41 |  |
42 |
43 |
44 | Vincent van Gogh The Starry Night |
45 |  |
46 |  |
47 |
48 |
49 | William Turner The Shipwreck of The Minotaur |
50 |  |
51 |  |
52 |
53 |
54 | Wassily Kandinsky Composition VII |
55 |  |
56 |  |
57 |
58 |
59 |
60 | ## Content Reconstruction
61 |
62 | [content-reconstruction.ipynb](content-reconstruction.ipynb) describes the
63 | content reconstruction process from white noise. Performing gradient descent
64 | of the content loss on a white noise input **x** for a given content **p**
65 | yields a representation of the networks activation for a given layer _l_.
66 |
67 |
68 |
69 | Layer |
70 | Result |
71 |
72 |
73 | block1_conv1 |
74 |  |
75 |
76 |
77 | block2_conv1 |
78 |  |
79 |
80 |
81 | block3_conv1 |
82 |  |
83 |
84 |
85 | block4_conv1 |
86 |  |
87 |
88 |
89 | block4_conv2 |
90 |  |
91 |
92 |
93 | block5_conv1 |
94 |  |
95 |
96 |
97 |
98 | ## Style Reconstruction
99 |
100 | [style-reconstruction.ipynb](style-reconstruction.ipynb) describes the style
101 | reconstruction process on Wassily Kandinsky's Composition VII from white
102 | noise. Performing gradient descent of the style loss on a white noise input
103 | **x** for a given artwork **a** yields a representation of the networks
104 | activation for a given set of layers _L_.
105 |
106 |
107 |
108 | Layer |
109 | Result |
110 |
111 |
112 | block1_conv1 |
113 |  |
114 |
115 |
116 | block1_conv1 , block2_conv1 |
117 |  |
118 |
119 |
120 | block1_conv1 , block2_conv1 , block3_conv1 |
121 |  |
122 |
123 |
124 | block1_conv1 , block2_conv1 , block3_conv1 , block4_conv1 |
125 |  |
126 |
127 |
128 | block1_conv1 , block2_conv1 , block3_conv1 , block4_conv1 , block5_conv1 |
129 |  |
130 |
131 |
132 |
133 | ## Content Layer
134 |
135 | [content-layer.ipynb](content-layer.ipynb) visualizes how the style transfer
136 | is affected by using different layers for content loss.
137 |
138 |
139 |
140 | Layer |
141 | Result |
142 |
143 |
144 | block1_conv1 |
145 |  |
146 |
147 |
148 | block2_conv1 |
149 |  |
150 |
151 |
152 | block3_conv1 |
153 |  |
154 |
155 |
156 | block4_conv1 |
157 |  |
158 |
159 |
160 | block5_conv1 |
161 |  |
162 |
163 |
164 |
165 | ## Style Layers
166 |
167 | [style-layers.ipynb](style-layers.ipynb) visualizes how the style transfer is
168 | affected by using different sets of layers for style loss.
169 |
170 |
171 |
172 | Layers |
173 | Result |
174 |
175 |
176 | block1_conv1 |
177 |  |
178 |
179 |
180 | block1_conv1 , block2_conv1 |
181 |  |
182 |
183 |
184 | block1_conv1 , block2_conv1 , block3_conv1 |
185 |  |
186 |
187 |
188 | block1_conv1 , block2_conv1 , block3_conv1 , block4_conv1 |
189 |  |
190 |
191 |
192 | block1_conv1 , block2_conv1 , block3_conv1 , block4_conv1 , block5_conv1 |
193 |  |
194 |
195 |
196 |
197 | ## Optimizers
198 |
199 | [optimizers.ipynb](optimizers.ipynb) employs _gradient descent_, _adam_, and
200 | _L-BFGS_ to understand the effect of different black-box optimizers. Gatys et.
201 | al use L-BFGS, but Adam appears to produce comparable results without as much
202 | overhead.
203 |
204 |
205 |
206 | Gradient Descent |
207 | Adam |
208 | L-BFGS |
209 |
210 |
211 |  |
212 |  |
213 |  |
214 |
215 |
216 |
217 |
218 |
219 |
220 |
221 | ## TV Loss
222 |
223 | [tv-loss.ipynb](tv-loss.ipynb) introduces total-variation loss to reduce
224 | impulse noise in the images.
225 |
226 |
227 |
228 | TV Loss Scale Factor |
229 | Result |
230 |
231 |
232 | 0 |
233 |  |
234 |
235 |
236 | 1 |
237 |  |
238 |
239 |
240 | 10 |
241 |  |
242 |
243 |
244 | 100 |
245 |  |
246 |
247 |
248 | 1000 |
249 |  |
250 |
251 |
252 |
253 | ## Photo-Realistic Style Transfer
254 |
255 | [photo-realistic-style-transfer.ipynb](photo-realistic-style-transfer.ipynb)
256 | describes the photo-realistic style transfer process. Opposed to transferring
257 | style from an artwork, this notebook explores transferring a nighttime style
258 | from a picture of Piedmont Park at night to a daytime picture of Piedmont Park.
259 |
260 |
261 |
262 | Content |
263 | Style |
264 | Result |
265 |
266 |
267 |  |
268 |  |
269 |  |
270 |
271 |
272 |
273 | ## References
274 |
275 | [_[1] L. A. Gatys, A. S. Ecker, and M. Bethge. A neural algorithm of artistic style. arXiv preprint
276 | arXiv:1508.06576, 2015._][ref1]
277 |
278 | [ref1]: https://arxiv.org/abs/1508.06576
279 |
280 | [_[2] L. A. Gatys, A. S. Ecker, and M. Bethge. Image style transfer using convolutional neural networks. In
281 | Computer Vision and Pattern Recognition (CVPR), 2016 IEEE Conference on, pages 2414–2423.
282 | IEEE, 2016._][ref2]
283 |
284 | [ref2]: https://www.cv-foundation.org/openaccess/content_cvpr_2016/papers/Gatys_Image_Style_Transfer_CVPR_2016_paper.pdf
285 |
--------------------------------------------------------------------------------
/frames_to_video.py:
--------------------------------------------------------------------------------
1 | """This file turns a directory of png files into video file."""
2 | import cv2
3 | import re
4 | from sys import argv
5 | from os import listdir
6 |
7 |
8 | def pairs(iterable):
9 | """
10 | Return a new iterable over sequential pairs in the given iterable.
11 | i.e. (0,1), (1,2), ..., (n-2,n-1)
12 | Args:
13 | iterable: the iterable to iterate over the pairs of
14 | Returns: a new iterator over the pairs of the given iterator
15 | """
16 | # lazily import tee from `itertools`
17 | from itertools import tee
18 | # split the iterator into 2 identical iterators
19 | a, b = tee(iterable)
20 | # retrieve the next item from the b iterator
21 | next(b, None)
22 | # zip up the iterators of current and next items
23 | return zip(a, b)
24 |
25 |
26 | # the directory of png files to make into a video
27 | directory = argv[1]
28 | # the name of the video file to write
29 | video_name = argv[2]
30 | # the number of frames to use for interpolation
31 | interpolation_frames = int(argv[3])
32 |
33 | # get the frames from the given directory in sorted order
34 | frames = [filename for filename in listdir(directory) if '.jpg' in filename]
35 | # use the integer filename value to account for the lack of 0 padding
36 | frames = sorted(frames, key=lambda x: int(re.sub(r'[^0-9]+', "", x)))
37 | # open the frames as images
38 | frames = [cv2.imread('{}/{}'.format(directory, frame)) for frame in frames]
39 | # unpack the dimensions of the images
40 | height, width, dims = frames[0].shape
41 |
42 | # setup a video output stream
43 | fourcc = cv2.VideoWriter_fourcc(*'mp4v') # Be sure to use lower case
44 | out = cv2.VideoWriter(video_name, fourcc, 20.0, (width, height))
45 |
46 | # write the frames to the video file
47 | for frame, next_frame in pairs(frames):
48 | # write the current frame
49 | out.write(frame)
50 | # iterate over the number of interpolation frames
51 | for i in range(1, interpolation_frames + 1):
52 | # calculate a frame between current and next frames
53 | w = i / interpolation_frames
54 | mid_frame = cv2.addWeighted(frame, 1 - w, next_frame, w, 0)
55 | # write this mid frame to the video
56 | out.write(mid_frame)
57 |
58 | # the last frame still needs written
59 | out.write(frames[-1])
60 |
61 | # cleanup and release the video
62 | out.release()
63 | cv2.destroyAllWindows()
64 |
--------------------------------------------------------------------------------
/img/content-layer/block1_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block1_conv1.jpg
--------------------------------------------------------------------------------
/img/content-layer/block2_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block2_conv1.jpg
--------------------------------------------------------------------------------
/img/content-layer/block3_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block3_conv1.jpg
--------------------------------------------------------------------------------
/img/content-layer/block4_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block4_conv1.jpg
--------------------------------------------------------------------------------
/img/content-layer/block5_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-layer/block5_conv1.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block1_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block1_conv1.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block2_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block2_conv1.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block3_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block3_conv1.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block4_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block4_conv1.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block4_conv2.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block4_conv2.jpg
--------------------------------------------------------------------------------
/img/content-reconstruction/block5_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content-reconstruction/block5_conv1.jpg
--------------------------------------------------------------------------------
/img/content/piedmont-park-night.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/piedmont-park-night.jpg
--------------------------------------------------------------------------------
/img/content/piedmont-park.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/piedmont-park.jpg
--------------------------------------------------------------------------------
/img/content/tubingen.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/content/tubingen.jpg
--------------------------------------------------------------------------------
/img/optimizers/Adam.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/Adam.jpg
--------------------------------------------------------------------------------
/img/optimizers/GradientDescent.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/GradientDescent.jpg
--------------------------------------------------------------------------------
/img/optimizers/L_BFGS.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/L_BFGS.jpg
--------------------------------------------------------------------------------
/img/optimizers/plot.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/optimizers/plot.png
--------------------------------------------------------------------------------
/img/photo-realistic-style-transfer/content.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/content.jpg
--------------------------------------------------------------------------------
/img/photo-realistic-style-transfer/piedmont-park-Adam.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-Adam.jpg
--------------------------------------------------------------------------------
/img/photo-realistic-style-transfer/piedmont-park-L_BFGS.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-L_BFGS.jpg
--------------------------------------------------------------------------------
/img/photo-realistic-style-transfer/piedmont-park-SGD.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/piedmont-park-SGD.jpg
--------------------------------------------------------------------------------
/img/photo-realistic-style-transfer/style.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/photo-realistic-style-transfer/style.jpg
--------------------------------------------------------------------------------
/img/style-layer/block1_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block1_conv1.jpg
--------------------------------------------------------------------------------
/img/style-layer/block2_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block2_conv1.jpg
--------------------------------------------------------------------------------
/img/style-layer/block3_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block3_conv1.jpg
--------------------------------------------------------------------------------
/img/style-layer/block4_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block4_conv1.jpg
--------------------------------------------------------------------------------
/img/style-layer/block5_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-layer/block5_conv1.jpg
--------------------------------------------------------------------------------
/img/style-reconstruction/block1_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block1_conv1.jpg
--------------------------------------------------------------------------------
/img/style-reconstruction/block2_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block2_conv1.jpg
--------------------------------------------------------------------------------
/img/style-reconstruction/block3_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block3_conv1.jpg
--------------------------------------------------------------------------------
/img/style-reconstruction/block4_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block4_conv1.jpg
--------------------------------------------------------------------------------
/img/style-reconstruction/block5_conv1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-reconstruction/block5_conv1.jpg
--------------------------------------------------------------------------------
/img/style-transfer.drawio.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer.drawio.png
--------------------------------------------------------------------------------
/img/style-transfer/composition-vii.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/composition-vii.jpg
--------------------------------------------------------------------------------
/img/style-transfer/houses-of-parliament.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/houses-of-parliament.jpg
--------------------------------------------------------------------------------
/img/style-transfer/seated-nude.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/seated-nude.jpg
--------------------------------------------------------------------------------
/img/style-transfer/the-scream.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-scream.jpg
--------------------------------------------------------------------------------
/img/style-transfer/the-shipwreck-of-the-minotaur.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-shipwreck-of-the-minotaur.jpg
--------------------------------------------------------------------------------
/img/style-transfer/the-starry-night.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/style-transfer/the-starry-night.jpg
--------------------------------------------------------------------------------
/img/styles-web/composition-vii.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/composition-vii.jpg
--------------------------------------------------------------------------------
/img/styles-web/houses-of-parliament.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/houses-of-parliament.jpg
--------------------------------------------------------------------------------
/img/styles-web/seated-nude.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/seated-nude.jpg
--------------------------------------------------------------------------------
/img/styles-web/the-scream.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-scream.jpg
--------------------------------------------------------------------------------
/img/styles-web/the-shipwreck-of-the-minotaur.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-shipwreck-of-the-minotaur.jpg
--------------------------------------------------------------------------------
/img/styles-web/the-starry-night.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles-web/the-starry-night.jpg
--------------------------------------------------------------------------------
/img/styles/composition-vii.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/composition-vii.jpg
--------------------------------------------------------------------------------
/img/styles/houses-of-parliament.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/houses-of-parliament.jpg
--------------------------------------------------------------------------------
/img/styles/seated-nude.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/seated-nude.jpg
--------------------------------------------------------------------------------
/img/styles/the-scream.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-scream.jpg
--------------------------------------------------------------------------------
/img/styles/the-shipwreck-of-the-minotaur.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-shipwreck-of-the-minotaur.jpg
--------------------------------------------------------------------------------
/img/styles/the-starry-night.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/styles/the-starry-night.jpg
--------------------------------------------------------------------------------
/img/tv/0.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/0.jpg
--------------------------------------------------------------------------------
/img/tv/1.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/1.jpg
--------------------------------------------------------------------------------
/img/tv/10.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/10.jpg
--------------------------------------------------------------------------------
/img/tv/100.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/100.jpg
--------------------------------------------------------------------------------
/img/tv/1000.jpg:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/Kautenja/a-neural-algorithm-of-artistic-style/f86601c5dc94ed9f5ad31a0132e844544ec3141f/img/tv/1000.jpg
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | numpy==1.19.5
2 | matplotlib==3.3.3
3 | pandas==1.1.4
4 | tensorflow==2.4.1
5 | jupyter==1.0.0
6 | tqdm==4.54.0
7 | scipy==1.6.0
8 | scikit-image==0.17.2
9 |
--------------------------------------------------------------------------------
/src/__init__.py:
--------------------------------------------------------------------------------
1 | """An implementation of the work in _A Neural Algorithm of Artistic Style_."""
2 |
--------------------------------------------------------------------------------
/src/callback.py:
--------------------------------------------------------------------------------
1 | """A method for building a rich callback for optimizers."""
2 | import os
3 | from glob import glob
4 | from IPython import display
5 | import numpy as np
6 | import matplotlib.pyplot as plt
7 | import skimage.io as io
8 | from .normalize import denormalize
9 |
10 |
11 | class Callback:
12 | """A callback for displaying and saving images."""
13 |
14 | def __init__(self,
15 | out_dir: str=None,
16 | extension='.png',
17 | plot=True,
18 | clear_display=True,
19 | ):
20 | """
21 | Initialize a new callback.
22 |
23 | Args:
24 | out_dir: the name of the artwork directory to store frames in
25 | extension: the file-type to save the images as
26 | plot: whether to plot the images
27 | clear_display: whether to clear the IPython display for new plots
28 |
29 | Returns:
30 | None
31 |
32 | """
33 | self.out_dir = out_dir
34 | self.extension = extension
35 | self.plot = plot
36 | self.clear_display = clear_display
37 | # make the directory if it doesn't exist
38 | if not os.path.exists(self.out_dir):
39 | os.makedirs(self.out_dir)
40 | # delete contents of the directory in case
41 | # it already existed with stuff in it
42 | for file in glob(f'{self.out_dir}/{self.extension}'):
43 | os.remove(file)
44 |
45 | def __call__(self, image, i):
46 | """
47 | De-normalize an iteration of optimization to display.
48 |
49 | Args:
50 | image: the image to de-normalize and display
51 | i: the iteration of optimization
52 |
53 | Returns:
54 | None
55 |
56 | """
57 | image = np.clip(denormalize(image[0]), 0, 255).astype('uint8')
58 | if self.out_dir is not None:
59 | io.imsave('{}/{}.jpg'.format(self.out_dir, i), image)
60 | if self.plot:
61 | if self.clear_display:
62 | display.clear_output(wait=True)
63 | ax = plt.imshow(image)
64 | ax.axes.xaxis.set_major_locator(plt.NullLocator())
65 | ax.axes.yaxis.set_major_locator(plt.NullLocator())
66 | plt.show()
67 |
68 |
69 | # explicitly define the outward facing API of this module
70 | __all__ = [Callback.__name__]
71 |
--------------------------------------------------------------------------------
/src/loss_functions.py:
--------------------------------------------------------------------------------
1 | """Methods representing various loss functions."""
2 | from tensorflow.keras import backend as K
3 |
4 |
5 | def content_loss(content, combination):
6 | """
7 | Return the content loss between the content and combinations tensors.
8 |
9 | Args:
10 | content: the output of a layer for the content image
11 | combination: the output of a layer for the combination image
12 |
13 | Returns:
14 | the loss between `content` and `combination`
15 |
16 | """
17 | # squared euclidean distance, exactly how it is in the paper
18 | return 0.5 * K.sum(K.square(combination - content))
19 |
20 |
21 | def gram(x):
22 | """
23 | Return a gram matrix for the given input matrix.
24 |
25 | Args:
26 | x: the matrix to calculate the gram matrix of
27 |
28 | Returns:
29 | the gram matrix of x
30 |
31 | """
32 | # use the keras function to access shape opposed to the instance member.
33 | # this allows backward compatibility with TF1.2.1 (the version in conda)
34 | shape = K.shape(x)
35 | # flatten the 3D tensor by converting each filter's 2D matrix of points
36 | # to a vector. thus we have the matrix:
37 | # [filter_width x filter_height, num_filters]
38 | F = K.reshape(x, (shape[0] * shape[1], shape[2]))
39 | # take inner product over all the vectors to produce the Gram matrix over
40 | # the number of filters
41 | return K.dot(K.transpose(F), F)
42 |
43 |
44 | def style_loss(style, combination):
45 | """
46 | Return the style loss between the style and combinations tensors.
47 |
48 | Args:
49 | style: the output of a layer for the style image
50 | combination: the output of a layer for the combination image
51 |
52 | Returns:
53 | the loss between `style` and `combination`
54 |
55 | """
56 | # M_l is the width times the height of the current layer
57 | Ml2 = int(style.shape[0] * style.shape[1])**2
58 | # N_l is the number of distinct filters in the layer
59 | Nl2 = int(style.shape[2])**2
60 |
61 | # take the squared euclidean distance between the gram matrices of both
62 | # the style and combination image. divide by the constant scaling factor
63 | # based on parameterized sizes
64 | return K.sum(K.square(gram(style) - gram(combination))) / (4 * Nl2 * Ml2)
65 |
66 |
67 | def total_variation_loss(x, kind='isotropic'):
68 | """
69 | Return the total variation loss for the image x.
70 |
71 | Args:
72 | x: the image tensor to return the variation loss of
73 | kind: the kind of total variation loss to use (default 'anisotropic')
74 |
75 | Returns:
76 | the total variation loss of the image x
77 |
78 | """
79 | # store the dimensions for indexing from the image x
80 | h, w = x.shape[1], x.shape[2]
81 | if kind == 'anisotropic':
82 | # take the absolute value between this image, and the image one pixel
83 | # down, and one pixel to the right. take the absolute value as
84 | # specified by anisotropic loss
85 | a = K.abs(x[:, :h-1, :w-1, :] - x[:, 1:, :w-1, :])
86 | b = K.abs(x[:, :h-1, :w-1, :] - x[:, :h-1, 1:, :])
87 | # add up all the differences
88 | return K.sum(a + b)
89 | elif kind == 'isotropic':
90 | # take the absolute value between this image, and the image one pixel
91 | # down, and one pixel to the right. take the square root as specified
92 | # by isotropic loss
93 | a = K.square(x[:, :h-1, :w-1, :] - x[:, 1:, :w-1, :])
94 | b = K.square(x[:, :h-1, :w-1, :] - x[:, :h-1, 1:, :])
95 | # take the vector square root of all the pixel differences, then sum
96 | # them all up
97 | return K.sum(K.pow(a + b, 2))
98 | else:
99 | # kind can only be two values, raise an error on unexpected kind value
100 | raise ValueError("`kind` should be 'anisotropic' or 'isotropic'")
101 |
102 |
103 | # explicitly define the outward facing API of this module
104 | __all__ = [
105 | content_loss.__name__,
106 | style_loss.__name__,
107 | total_variation_loss.__name__,
108 | ]
109 |
--------------------------------------------------------------------------------
/src/normalize.py:
--------------------------------------------------------------------------------
1 | """Image utility methods for the project."""
2 | import numpy as np
3 |
4 |
5 | # the BGR means from the ImageNet database
6 | IMAGENET_MEANS = np.array([103.939, 116.779, 123.68])
7 |
8 |
9 | def normalize(image: np.ndarray, inplace: bool=False) -> np.ndarray:
10 | """
11 | Normalize an image by a set of means.
12 |
13 | Args:
14 | image: the image to normalize assuming shape [1, H, W, channels]
15 | inplace: whether to perform the operation of the array in-place
16 |
17 | Returns:
18 | image after flipping from RGB to BGR and subtracting ImageNet means
19 |
20 | """
21 | # validate the shape of the image
22 | assert image.shape[3] == IMAGENET_MEANS.shape[0]
23 | # if in-place is enabled, copy the array
24 | if not inplace:
25 | image = image.copy()
26 | # flip image from RGB, to BGR
27 | image = image[:, :, :, ::-1]
28 | # vector subtract the means from ImageNet to the image
29 | image[:, :, :, np.arange(IMAGENET_MEANS.shape[0])] -= IMAGENET_MEANS
30 |
31 | return image
32 |
33 |
34 | def denormalize(image: np.ndarray, inplace: bool=False) -> np.ndarray:
35 | """
36 | De-normalize an image by a set of means.
37 |
38 | Args:
39 | image: the image to normalize (assuming standard image shape in BGR)
40 |
41 | Returns:
42 | image after flipping from BGR to RGB and adding ImageNet means
43 |
44 | """
45 | # validate the shape of the image
46 | assert image.shape[2] == IMAGENET_MEANS.shape[0]
47 | # if in-place is enabled, copy the array
48 | if not inplace:
49 | image = image.copy()
50 | # vector add the means from ImageNet to the image
51 | image[:, :, np.arange(IMAGENET_MEANS.shape[0])] += IMAGENET_MEANS
52 | # flip image from BGR, to RGB
53 | image = image[:, :, ::-1]
54 |
55 | return image
56 |
57 |
58 | # explicitly specify the public API of the module
59 | __all__ = [normalize.__name__, denormalize.__name__]
60 |
--------------------------------------------------------------------------------
/src/optimizers/__init__.py:
--------------------------------------------------------------------------------
1 | """Black Box optimization methods."""
2 | from .gd import GradientDescent
3 | from .l_bfgs import L_BFGS
4 | from .adam import Adam
5 |
6 |
7 | # export the public API for this package
8 | __all__ = [
9 | 'GradientDescent',
10 | 'L_BFGS',
11 | 'Adam'
12 | ]
13 |
--------------------------------------------------------------------------------
/src/optimizers/adam.py:
--------------------------------------------------------------------------------
1 | """An implementation of a the Adam optimizer."""
2 | from typing import Callable
3 | from tqdm import tqdm
4 | import numpy as np
5 |
6 |
7 | class Adam(object):
8 | """The Adam optimizer."""
9 |
10 | def __init__(self,
11 | learning_rate: float=1e-4,
12 | beta1: float=0.9,
13 | beta2: float=0.999,
14 | epsilon: float=1e-6) -> None:
15 | """
16 | Initialize a new Adam optimizer.
17 |
18 | Args:
19 | learning_rate: how fast to adjust the parameters (dW)
20 |
21 | Returns:
22 | None
23 |
24 | """
25 | self.learning_rate = learning_rate
26 | self.beta1 = beta1
27 | self.beta2 = beta2
28 | self.epsilon = epsilon
29 | # set the history of loss evaluations to empty list
30 | self.loss_history = []
31 |
32 | def __repr__(self) -> str:
33 | """Return an executable string representation of this object."""
34 | return '{}(learning_rate={}, beta1={}, beta2={})'.format(*[
35 | self.__class__.__name__,
36 | self.learning_rate,
37 | self.beta1,
38 | self.beta2
39 | ])
40 |
41 | def __call__(self,
42 | X: np.ndarray,
43 | shape: tuple,
44 | loss_grads: Callable,
45 | iterations: int=1000,
46 | callback: Callable=None):
47 | """
48 | Reduce the loss generated by X by moving it based on its gradient.
49 |
50 | Args:
51 | X: the input value to adjust to minimize loss
52 | shape: the shape to coerce X to
53 | loss_grads: a callable method that returns loss and gradients
54 | given some input
55 | iterations: the number of iterations of optimization to perform
56 | callback: an optional callback method to receive image updates
57 |
58 | Returns:
59 | an optimized X about the loss and gradients given
60 |
61 | """
62 | # setup the local data structures for the optimization
63 | moment1 = np.zeros_like(X)
64 | moment2 = np.zeros_like(X)
65 |
66 | # reset the history of loss evaluations to empty list
67 | self.loss_history = []
68 | for i in tqdm(range(iterations)):
69 | # pass the input through the loss function and generate gradients
70 | loss_i, grads_i = loss_grads([X])
71 | # calculate the new values of the first and second moment
72 | moment1 = self.beta1 * moment1 + (1 - self.beta1) * grads_i
73 | moment2 = self.beta2 * moment2 + (1 - self.beta1) * grads_i**2
74 | # debias the moments to avoid bias towards 0 at the beginning
75 | debias1 = moment1 / (1 - self.beta1**(i+1))
76 | debias2 = moment2 / (1 - self.beta2**(i+1))
77 | # move the input based on the gradients and learning rate
78 | X -= self.learning_rate * debias1 / (debias2**0.5 + self.epsilon)
79 | # update the loss history with this loss value
80 | self.loss_history.append(loss_i)
81 | # pass the values to the callback if any
82 | if callable(callback):
83 | callback(X, i)
84 |
85 | return X
86 |
87 |
88 | # explicitly define the outward facing API of this module
89 | __all__ = [Adam.__name__]
90 |
--------------------------------------------------------------------------------
/src/optimizers/gd.py:
--------------------------------------------------------------------------------
1 | """An implementation of a basic gradient descent algorithm."""
2 | from typing import Callable
3 | from tqdm import tqdm
4 | import numpy as np
5 |
6 |
7 | class GradientDescent(object):
8 | """A basic gradient descent optimizer."""
9 |
10 | def __init__(self, learning_rate: float=1e-4) -> None:
11 | """
12 | Initialize a new Stochastic Gradient Descent optimizer.
13 |
14 | Args:
15 | learning_rate: how fast to adjust the parameters (dW)
16 |
17 | Returns:
18 | None
19 |
20 | """
21 | self.learning_rate = learning_rate
22 | # set the history of loss evaluations to empty list
23 | self.loss_history = []
24 |
25 | def __repr__(self) -> str:
26 | """Return an executable string representation of this object."""
27 | return '{}(learning_rate={})'.format(*[
28 | self.__class__.__name__,
29 | self.learning_rate
30 | ])
31 |
32 | def __call__(self,
33 | X: np.ndarray,
34 | shape: tuple,
35 | loss_grads: Callable,
36 | iterations: int=1000,
37 | callback: Callable=None):
38 | """
39 | Reduce the loss generated by X by moving it based on its gradient.
40 |
41 | Args:
42 | X: the input value to adjust to minimize loss
43 | shape: the shape to coerce X to
44 | loss_grads: a callable method that returns loss and gradients
45 | given some input
46 | iterations: the number of iterations of optimization to perform
47 | callback: an optional callback method to receive image updates
48 |
49 | Returns:
50 | an optimized X about the loss and gradients given
51 |
52 | """
53 | # reset the history of loss evaluations to empty list
54 | self.loss_history = []
55 | for i in tqdm(range(iterations)):
56 | # pass the input through the loss function and generate gradients
57 | loss_i, grads_i = loss_grads([X])
58 | # move the input based on the gradients and learning rate
59 | X -= self.learning_rate * grads_i
60 | # update the loss history with this loss value
61 | self.loss_history.append(loss_i)
62 | # pass the values to the callback if any
63 | if callable(callback):
64 | callback(X, i)
65 |
66 | return X
67 |
68 |
69 | # explicitly define the outward facing API of this module
70 | __all__ = [GradientDescent.__name__]
71 |
--------------------------------------------------------------------------------
/src/optimizers/l_bfgs.py:
--------------------------------------------------------------------------------
1 | """An interface to the L-BFGS Algorithm."""
2 | import numpy as np
3 | from typing import Callable
4 | from tqdm import tqdm
5 | from scipy.optimize import fmin_l_bfgs_b
6 |
7 |
8 | class L_BFGS(object):
9 | """L-BFGS Optimization Algorithm."""
10 |
11 | def __init__(self, max_evaluations: int=20) -> None:
12 | """
13 | Initialize a new L-BFGS Hill climbing optimizer.
14 |
15 | Args:
16 | max_evaluations: how fast to adjust the parameters (dW)
17 |
18 | Returns:
19 | None
20 |
21 | """
22 | self.max_evaluations = max_evaluations
23 | self._loss = None
24 | self._gradients = None
25 | # set the history of loss evaluations to empty list
26 | self.loss_history = []
27 |
28 | def __repr__(self) -> str:
29 | """Return an executable string representation of this object."""
30 | return '{}(max_evaluations={})'.format(*[
31 | self.__class__.__name__,
32 | self.max_evaluations
33 | ])
34 |
35 | def loss(self, X) -> np.ndarray:
36 | """Calculate the loss given some X."""
37 | # make sure this is called _only after gradients_
38 | assert self._loss is None
39 | # calculate and store both the loss and the gradients
40 | loss, gradients = self.loss_and_gradients(X)
41 | # update the cache for this pass
42 | self._loss = loss
43 | self._gradients = gradients
44 |
45 | return self._loss
46 |
47 | def gradients(self, X) -> np.ndarray:
48 | """Calculate the gradients (lazily) given some X."""
49 | # make sure this is called _only after loss_
50 | assert self._loss is not None
51 | # copy the gradients
52 | gradients = np.copy(self._gradients)
53 | # nullify the cache for this pass
54 | self._loss = None
55 | self._gradients = None
56 |
57 | return gradients
58 |
59 | def __call__(self,
60 | X: np.ndarray,
61 | shape: tuple,
62 | loss_grads: Callable,
63 | iterations: int=1000,
64 | callback: Callable=None) -> np.ndarray:
65 | """
66 | Reduce the loss generated by X.
67 |
68 | Args:
69 | X: the input value to adjust to minimize loss
70 | shape: the shape to coerce X to
71 | loss_grads: a callable method that returns loss and gradients
72 | given some input
73 | iterations: the number of iterations of optimization to perform
74 | callback: an optional callback method to receive image updates
75 |
76 | Returns:
77 | an optimized X about the loss and gradients given
78 |
79 | """
80 | # reset the history of loss evaluations to empty list
81 | self.loss_history = []
82 | # create a loss and gradients evaluation method for SciPy
83 | def loss_and_gradients(X):
84 | """Calculate the loss and gradients with appropriate reshaping."""
85 | loss, gradients = loss_grads([X.reshape(shape)])
86 | return loss, gradients.flatten().astype('float64')
87 |
88 | # assign the custom method to self for loss / gradient calculation
89 | self.loss_and_gradients = loss_and_gradients
90 |
91 | for i in tqdm(range(iterations)):
92 | # pass X through an iteration of LBFGS
93 | X, min_val, info = fmin_l_bfgs_b(self.loss, X,
94 | fprime=self.gradients,
95 | maxfun=self.max_evaluations)
96 | # update the loss history with this loss value
97 | self.loss_history += info['funcalls'] * [min_val]
98 | # pass the values to the callback if any
99 | if callable(callback):
100 | callback(X.reshape(shape), i)
101 |
102 | return X
103 |
104 |
105 | # explicitly define the outward facing API of this module
106 | __all__ = [L_BFGS.__name__]
107 |
--------------------------------------------------------------------------------
/src/reconstruct_content.py:
--------------------------------------------------------------------------------
1 | """A functional decomposition of the content reconstruction algorithm."""
2 | from typing import Callable
3 | import numpy as np
4 | import tensorflow as tf
5 | from tensorflow.keras import backend as K
6 | from tensorflow.keras.applications.vgg19 import VGG19
7 | from .normalize import normalize, denormalize
8 | from .loss_functions import content_loss
9 | from .optimizers.l_bfgs import L_BFGS
10 |
11 |
12 | def reconstruct_content(content: np.ndarray,
13 | layer_name: str='block4_conv2',
14 | optimize: Callable=L_BFGS(),
15 | iterations: int=10,
16 | noise_range: tuple=(0, 1),
17 | callback: Callable=None
18 | ):
19 | """
20 | Reconstruct the given content image at the given VGG19 layer.
21 |
22 | Args:
23 | content: the content image to reconstruct
24 | layer_name: the layer to reconstruct the content from
25 | optimize: the optimization method for minimizing the content loss
26 | iterations: the number of iterations to run the optimizer
27 | noise_range: the range of values for initializing random noise
28 | callback: the callback for iterations of gradient descent
29 |
30 | Returns:
31 | the reconstructed content image based on the VGG19 response
32 | at the given layer name
33 |
34 | """
35 | # disable eager mode for this operation
36 | tf.compat.v1.disable_eager_execution()
37 |
38 | # normalize the image's RGB values about the RGB channel means for the
39 | # ImageNet dataset
40 | content = normalize(content[None, ...].astype('float'))
41 | # load the content image into Keras as a constant, it never changes
42 | content = K.constant(content, name='Content')
43 | # create a placeholder for the trained image, this variable trains
44 | canvas = K.placeholder(content.shape, name='Canvas')
45 | # combine the content and canvas tensors along the frame axis (0) into a
46 | # 4D tensor of shape [2, height, width, channels]
47 | input_tensor = K.concatenate([content, canvas], axis=0)
48 | # build the model with the 4D input tensor of content and canvas
49 | model = VGG19(include_top=False, input_tensor=input_tensor, pooling='avg')
50 |
51 | # extract the layer's out that we have interest in for reconstruction
52 | layer = model.get_layer(layer_name)
53 |
54 | # calculate the loss between the output of the layer on the content (0)
55 | # and the canvas (1)
56 | loss = content_loss(layer.output[0], layer.output[1])
57 | # calculate the gradients
58 | grads = K.gradients(loss, canvas)[0]
59 | # generate the iteration function for gradient descent optimization
60 | step = K.function([canvas], [loss, grads])
61 |
62 | # generate random noise with the given noise range
63 | noise = np.random.uniform(*noise_range, size=content.shape)
64 |
65 | # optimize the white noise to reconstruct the content
66 | image = optimize(noise, canvas.shape, step, iterations, callback)
67 |
68 | # clear the Keras session
69 | K.clear_session()
70 |
71 | # de-normalize the image (from ImageNet means) and convert back to binary
72 | return np.clip(denormalize(image.reshape(canvas.shape)[0]), 0, 255).astype('uint8')
73 |
74 |
75 | # explicitly define the outward facing API of this module
76 | __all__ = [reconstruct_content.__name__]
77 |
--------------------------------------------------------------------------------
/src/reconstruct_style.py:
--------------------------------------------------------------------------------
1 | """A functional decomposition of the style reconstruction algorithm."""
2 | from typing import Callable
3 | import numpy as np
4 | import tensorflow as tf
5 | from tensorflow.keras import backend as K
6 | from tensorflow.keras.applications.vgg19 import VGG19
7 | from .normalize import normalize, denormalize
8 | from .loss_functions import style_loss
9 | from .optimizers.l_bfgs import L_BFGS
10 |
11 |
12 | def reconstruct_style(style: np.ndarray,
13 | layer_names: list = [
14 | 'block1_conv1',
15 | 'block2_conv1',
16 | 'block3_conv1',
17 | 'block4_conv1',
18 | 'block5_conv1'
19 | ],
20 | optimize: Callable = L_BFGS(),
21 | iterations: int = 10,
22 | noise_range: tuple = (0, 1),
23 | callback: Callable = None
24 | ):
25 | """
26 | Reconstruct the given content image at the given VGG19 layer.
27 |
28 | Args:
29 | style: the style to reconstruct
30 | layer_name: the layer to reconstruct the content from
31 | optimizer: the optimizer for minimizing the content loss
32 | iterations: the number of iterations to run the optimizer
33 | noise_range: the range of values for initializing random noise
34 | callback: the callback for iterations of gradient descent
35 |
36 | Returns:
37 | the reconstructed content image based on the VGG19 response at the
38 | given layer name
39 |
40 | """
41 | # disable eager mode for this operation
42 | tf.compat.v1.disable_eager_execution()
43 |
44 | # normalize the image's RGB values about the RGB channel means for the
45 | # ImageNet dataset
46 | style = normalize(style[None, ...].astype('float'))
47 |
48 | # load the style image into Keras as a constant, it never changes
49 | style = K.constant(style, name='Style')
50 | # create a placeholder for the trained image, this variable trains
51 | canvas = K.placeholder(style.shape, name='Canvas')
52 | # combine the style and canvas tensors along the frame axis (0) into a
53 | # 4D tensor of shape [2, height, width, channels]
54 | input_tensor = K.concatenate([style, canvas], axis=0)
55 | # build the model with the 4D input tensor of style and canvas
56 | model = VGG19(include_top=False, input_tensor=input_tensor, pooling='avg')
57 |
58 | # initialize the loss to accumulate iteratively over the layers
59 | loss = K.variable(0.0)
60 |
61 | # iterate over the list of all the layers that we want to include
62 | for layer_name in layer_names:
63 | # extract the layer's out that we have interest in for reconstruction
64 | layer = model.get_layer(layer_name)
65 | # calculate the loss between the output of the layer on the style (0)
66 | # and the canvas (1). The style loss needs to know the size of the
67 | # image as well by width (shape[2]) and height (shape[1])
68 | loss = loss + style_loss(layer.output[0], layer.output[1])
69 |
70 | # Gatys et al. use a w_l of 1/5 for their example with the 5 layers. As
71 | # such, we'll simply and say for any length of layers, just take the
72 | # average. (mirroring what they did)
73 | loss /= len(layer_names)
74 |
75 | # calculate the gradients
76 | grads = K.gradients(loss, canvas)[0]
77 | # generate the iteration function for gradient descent optimization
78 | step = K.function([canvas], [loss, grads])
79 |
80 | # generate random noise as the starting point
81 | noise = np.random.uniform(*noise_range, size=canvas.shape)
82 |
83 | # optimize the white noise to reconstruct the content
84 | image = optimize(noise, canvas.shape, step, iterations, callback)
85 |
86 | # clear the Keras session
87 | K.clear_session()
88 |
89 | # de-normalize the image (from ImageNet means) and convert back to binary
90 | return np.clip(denormalize(image.reshape(canvas.shape)[0]), 0, 255).astype('uint8')
91 |
92 |
93 | # explicitly define the outward facing API of this module
94 | __all__ = [reconstruct_style.__name__]
95 |
--------------------------------------------------------------------------------
/src/transfer_style.py:
--------------------------------------------------------------------------------
1 | """A mechanism for transferring style of art to content."""
2 | from typing import Callable, List
3 | import numpy as np
4 | import tensorflow as tf
5 | from tensorflow.keras import backend as K
6 | from tensorflow.keras.applications.vgg19 import VGG19
7 | from .normalize import normalize, denormalize
8 | from .loss_functions import content_loss, style_loss, total_variation_loss
9 |
10 |
11 | # the template for the class's __repr__ method
12 | TEMPLATE = """{}(
13 | content_layer_name={},
14 | content_weight={},
15 | style_layer_names={},
16 | style_layer_weights={},
17 | style_weight={},
18 | total_variation_weight={}
19 | )""".lstrip()
20 |
21 |
22 | class Stylizer(object):
23 | """An algorithm for stylizing images based on artwork."""
24 |
25 | def __init__(self,
26 | content_layer_name: str='block4_conv2',
27 | content_weight: float=1.0,
28 | style_layer_names: List[str]=[
29 | 'block1_conv1',
30 | 'block2_conv1',
31 | 'block3_conv1',
32 | 'block4_conv1',
33 | 'block5_conv1'
34 | ],
35 | style_layer_weights: List[float]=None,
36 | style_weight: float=10000.0,
37 | total_variation_weight: float=0.0
38 | ) -> None:
39 | """
40 | Initialize a new neural stylization algorithm.
41 |
42 | Args:
43 | content_layer_name: the name of the layer to extract content from
44 | content_weight: the weight, alpha, to attribute to content loss
45 | style_layer_names: the names of the layers to extract style from
46 | style_weight: the weight, beta, to attribute to style loss
47 | style_layer_weights: the set of weights to apply to the individual
48 | losses from each style layer. If None, the default is to take
49 | the average, i.e. divide each by len(style_layer_names).
50 | total_variation_weight: the amount of total variation de-noising
51 | to apply to the synthetic images
52 |
53 | Returns:
54 | None
55 |
56 | """
57 | # get the names of the layers from the model to error check
58 | layer_names = [l.name for l in VGG19(include_top=False).layers]
59 |
60 | # type and value check: content_layer_name
61 | if not isinstance(content_layer_name, str):
62 | raise TypeError('`content_layer_name` must be of type: str')
63 | if content_layer_name not in layer_names:
64 | raise ValueError(
65 | '`content_layer_name` must be a layer name in VGG19'
66 | )
67 | self.content_layer_name = content_layer_name
68 |
69 | # type and value check: content_weight
70 | if not isinstance(content_weight, (int, float)):
71 | raise TypeError('`content_weight` must be of type: int or float')
72 | if content_weight < 0:
73 | raise ValueError('`content_weight` must be >= 0')
74 | self.content_weight = content_weight
75 |
76 | # type and value check: content_layer_name
77 | if not isinstance(style_layer_names, list):
78 | raise TypeError('`style_layer_names` must be of type: list')
79 | if not all(layer in layer_names for layer in style_layer_names):
80 | raise ValueError(
81 | '`style_layer_names` must be a list of layer names in VGG19'
82 | )
83 | self.style_layer_names = style_layer_names
84 |
85 | # type and value check: style_layer_weights
86 | if style_layer_weights is None:
87 | # initialize style layer weights as an average between them.
88 | total = len(style_layer_names)
89 | style_layer_weights = total * [1.0 / total]
90 | else:
91 | if not isinstance(style_layer_weights, list):
92 | raise TypeError(
93 | '`style_layer_weights` must be of type: None or list'
94 | )
95 | if not all(isinstance(w, (float, int)) for w in style_layer_weights):
96 | raise ValueError(
97 | '`style_layer_weights` must be a list of numbers or None'
98 | )
99 | self.style_layer_weights = style_layer_weights
100 |
101 | # type and value check: style_weight
102 | if not isinstance(style_weight, (int, float)):
103 | raise TypeError('`style_weight` must be of type: int or float')
104 | if style_weight < 0:
105 | raise ValueError('`style_weight` must be >= 0')
106 | self.style_weight = style_weight
107 |
108 | # type and value check: total_variation_weight
109 | if not isinstance(total_variation_weight, (int, float)):
110 | raise TypeError(
111 | '`total_variation_weight` must be of type: int or float'
112 | )
113 | if total_variation_weight < 0:
114 | raise ValueError('`total_variation_weight` must be >= 0')
115 | self.total_variation_weight = total_variation_weight
116 |
117 | # disable eager mode for this operation
118 | tf.compat.v1.disable_eager_execution()
119 |
120 | def __repr__(self) -> str:
121 | """Return an executable string representation of this object."""
122 | return TEMPLATE.format(*[
123 | self.__class__.__name__,
124 | repr(self.content_layer_name),
125 | self.content_weight,
126 | self.style_layer_names,
127 | self.style_layer_weights,
128 | self.style_weight,
129 | self.total_variation_weight
130 | ])
131 |
132 | @property
133 | def content_style_ratio(self) -> float:
134 | """Return the ratio of content weight to style weight."""
135 | return self.content_weight / self.style_weight
136 |
137 | def _build_model(self, content: np.ndarray, style: np.ndarray) -> tuple:
138 | """
139 | Build a synthesis model with the given content and style.
140 |
141 | Args:
142 | content: the content to fuse the artwork into
143 | style: the artwork to get the style from
144 |
145 | Returns:
146 | a tuple of:
147 | - the constructed VGG19 model from the input images
148 | - the canvas tensor for the synthesized image
149 |
150 | """
151 | # load the content image into Keras as a constant, it never changes
152 | content_tensor = K.constant(content, name='Content')
153 | # load the style image into Keras as a constant, it never changes
154 | style_tensor = K.constant(style, name='Style')
155 | # create a placeholder for the trained image, this variable changes
156 | canvas = K.placeholder(content.shape, name='Canvas')
157 | # combine the content, style, and canvas tensors along the frame
158 | # axis (0) into a 4D tensor of shape [3, height, width, channels]
159 | tensor = K.concatenate([content_tensor, style_tensor, canvas], axis=0)
160 | # build the model with the input tensor of content, style, and canvas
161 | model = VGG19(include_top=False, input_tensor=tensor, pooling='avg')
162 |
163 | return model, canvas
164 |
165 | def _build_loss_grads(self, model, canvas: 'Tensor') -> Callable:
166 | """
167 | Build the optimization methods for stylizing the image from a model.
168 |
169 | Args:
170 | model: the model to extract layers from
171 | canvas: the input to the model thats being mutated
172 |
173 | Returns:
174 | a function to calculate loss and gradients from an input X
175 |
176 | """
177 | # initialize a variable to store the loss into
178 | loss = K.variable(0.0, name='Loss')
179 |
180 | # CONTENT LOSS
181 | # extract the content layer tensor for optimizing content loss
182 | content_layer_output = model.get_layer(self.content_layer_name).output
183 | # calculate the loss between the output of the layer on the
184 | # content (0) and the canvas (2)
185 | cl = content_loss(content_layer_output[0],
186 | content_layer_output[2])
187 | loss = loss + self.content_weight * cl
188 |
189 | # STYLE LOSS
190 | sl = K.variable(0.0)
191 | # iterate over the list of all the layers that we want to include
192 | for style_layer_name, layer_weight in zip(self.style_layer_names,
193 | self.style_layer_weights):
194 | # extract the layer out that we have interest in
195 | style_layer_output = model.get_layer(style_layer_name).output
196 | # calculate the loss between the output of the layer on the
197 | # style (1) and the canvas (2).
198 | sl = sl + layer_weight * style_loss(style_layer_output[1],
199 | style_layer_output[2])
200 | # apply the style weight to style loss and add it to the total loss
201 | loss = loss + self.style_weight * sl
202 |
203 | # TOTAL VARIATION LOSS
204 | # Gatys et al. don't use the total variation de-noising in their paper
205 | # (or at least they never mention it) so the weight is 0.0 by
206 | # default, but can be applied if desired
207 | loss = loss + self.total_variation_weight * total_variation_loss(canvas)
208 |
209 | # GRADIENTS
210 | # calculate the gradients of the input image with respect to
211 | # the loss. i.e. back-propagate the loss through the network
212 | # to the input layer (only the canvas though)
213 | grads = K.gradients(loss, canvas)[0]
214 |
215 | # generate the iteration function for gradient descent optimization
216 | # Args:
217 | # noise: the input to the noise placeholder in the model
218 | # this effectively takes a the white noise image being
219 | # optimized and passes it forward and backward through
220 | # the model collecting the loss and gradient along the
221 | # way
222 | #
223 | # Returns:
224 | # a tuple of (loss, gradients)
225 | # - loss: the content loss between the content, style image
226 | # and the white noise
227 | # - gradients: the gradients of the inputs with respect
228 | # to the loss
229 | return K.function([canvas], [loss, grads])
230 |
231 | def __call__(self, content: np.ndarray, style: np.ndarray, optimize: Callable,
232 | iterations: int = 10,
233 | initialization_strat: str = 'noise',
234 | noise_range: tuple = (0, 1),
235 | callback: Callable = None
236 | ):
237 | """
238 | Stylize the given content image with the give style image.
239 |
240 | Args:
241 | content: the content image to use
242 | style: the style image to use
243 | optimize: the black-box optimizer to use. This is a callable
244 | method conforming to the API for optimizers
245 | iterations: the number of optimization iterations to perform
246 | image_size: the custom size to load images with, if any. When
247 | set to None, the size of the content image will be used
248 | initialization_strat: the way to initialize the canvas for the
249 | style transfer. Can be one of:
250 | - 'noise': initialize the canvas as random noise with the
251 | given noise range for sampling pixels
252 | - 'content': initialize the canvas as the content image
253 | - 'style': initialize the canvas as the style image
254 | noise_range: the custom range for initializing random noise. This
255 | option is only used when `initialization_strat` is 'noise'.
256 | callback: the optional callback method for optimizer iterations
257 |
258 | Returns:
259 | the image as a result of blending content with style
260 |
261 | """
262 | # normalize the input data
263 | content = normalize(content[None, ...].astype('float'))
264 | style = normalize(style[None, ...].astype('float'))
265 |
266 | # disable eager mode for this operation
267 | tf.compat.v1.disable_eager_execution()
268 | # build the inputs tensor from the images
269 | model, canvas = self._build_model(content, style)
270 | # build the iteration function
271 | loss_grads = self._build_loss_grads(model, canvas)
272 |
273 | # setup the initial image for the optimizer
274 | if initialization_strat == 'noise':
275 | # generate white noise in the shape of the canvas
276 | initial = np.random.uniform(*noise_range, size=canvas.shape)
277 | elif initialization_strat == 'content':
278 | # copy the content as the initial image
279 | initial = content.copy()
280 | elif initialization_strat == 'style':
281 | # copy the style as the initial image
282 | initial = style.copy()
283 | else:
284 | raise ValueError(
285 | "`initialization_strat` must be one of: ",
286 | " 'noise', 'content', 'style' "
287 | )
288 |
289 | # optimize the initial image into a synthetic painting. Name all args
290 | # by keyword to help catch erroneous optimize callables.
291 | image = optimize(
292 | X=initial,
293 | shape=canvas.shape,
294 | loss_grads=loss_grads,
295 | iterations=iterations,
296 | callback=callback
297 | )
298 |
299 | # clear the Keras session (this removes the variables from memory).
300 | K.clear_session()
301 |
302 | # reshape the image in case the optimizer did something funky with the
303 | # shape. `denormalize` expects a vector of shape [h, w, c], but
304 | # canvas has an additional dimension for frame, [frame, h, w, c]. Ss
305 | # such, take the first item along the frame axis
306 | image = image.reshape(canvas.shape)[0]
307 | # denormalize the image about the ImageNet means. this will invert the
308 | # channel dimension turning image from BGR to RGB
309 | return np.clip(denormalize(image), 0, 255).astype('uint8')
310 |
311 |
312 | # explicitly define the outward facing API of this module
313 | __all__ = [Stylizer.__name__]
314 |
--------------------------------------------------------------------------------