├── tests ├── __init__.py ├── utils │ ├── __init__.py │ ├── test_general.py │ ├── test_lossutils.py │ └── test_imutils.py ├── fixture │ ├── red.png │ ├── blue.png │ ├── green.png │ ├── grey.png │ ├── model_sum │ │ ├── last_weights.hdf5 │ │ └── archi.json │ ├── model_convt2d │ │ ├── last_weights.hdf5 │ │ └── archi.json │ ├── model_batchnorm │ │ ├── last_weights.hdf5 │ │ └── archi.json │ └── model_conv2d_relu │ │ ├── last_weights.hdf5 │ │ └── archi.json ├── test_convolution_transpose_2d.py └── test_atrous_convolution_2d.py ├── utils ├── __init__.py ├── optimizers.py ├── freeze_graph.py ├── imutils.py ├── general.py └── lossutils.py ├── vgg19 ├── __init__.py ├── download_weights.sh ├── dump_headless_weights.py ├── README.md ├── model.py └── model_headless.py ├── data ├── output │ └── .gitkeep ├── urls.txt ├── download_images.sh ├── build_overfit_folder.sh └── prepare_data.py ├── mobile_app ├── .watchmanconfig ├── data │ ├── tf-frozen_model.pb │ └── Linked_Frameworks_Libraries.png ├── .buckconfig ├── ios │ ├── ReactNativeTF │ │ ├── RunModel.h │ │ ├── UIImage+Alpha.h │ │ ├── AppDelegate.h │ │ ├── UIImage+RoundedCorner.h │ │ ├── main.m │ │ ├── ImageManager.h │ │ ├── Images.xcassets │ │ │ └── AppIcon.appiconset │ │ │ │ └── Contents.json │ │ ├── UIImage+Resize.h │ │ ├── Info.plist │ │ ├── AppDelegate.m │ │ ├── Base.lproj │ │ │ └── LaunchScreen.xib │ │ ├── UIImage+RoundedCorner.m │ │ ├── ImageManager.m │ │ ├── RunModel.mm │ │ └── UIImage+Alpha.m │ ├── ReactNativeTF.xcodeproj │ │ ├── project.xcworkspace │ │ │ ├── contents.xcworkspacedata │ │ │ ├── xcuserdata │ │ │ │ └── tom.xcuserdatad │ │ │ │ │ └── UserInterfaceState.xcuserstate │ │ │ └── xcshareddata │ │ │ │ └── ReactNativeTF.xcscmblueprint │ │ ├── xcuserdata │ │ │ └── tom.xcuserdatad │ │ │ │ ├── xcschemes │ │ │ │ └── xcschememanagement.plist │ │ │ │ └── xcdebugger │ │ │ │ └── Breakpoints_v2.xcbkptlist │ │ └── xcshareddata │ │ │ └── xcschemes │ │ │ └── ReactNativeTF.xcscheme │ └── ReactNativeTFTests │ │ ├── Info.plist │ │ └── ReactNativeTFTests.m ├── package.json ├── README.md ├── .flowconfig └── index.ios.js ├── models ├── __init__.py ├── layers │ ├── ScaledSigmoid.py │ ├── __init__.py │ ├── InstanceNormalization.py │ ├── ReflectPadding2D.py │ └── PhaseShift.py └── fast_style_transfer.py ├── perf ├── model.png └── st_perf.py ├── screenshot-tensorboard.png ├── docker ├── keras.json └── DOCKERFILE ├── pretrain_model.sh ├── predict.sh ├── requirements.txt ├── __init__.py ├── pre_analysis.sh ├── export_keras_model.py ├── gatys_paper_analysis.sh ├── torch └── forward.lua ├── .gitignore ├── predict.py ├── ltv.py ├── alpha.py ├── layer_influence.py ├── pretrain_model.py ├── layer_reconstruction.py ├── README.md └── gatys_paper.py /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /vgg19/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/output/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /mobile_app/.watchmanconfig: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /models/__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | from . import layers -------------------------------------------------------------------------------- /perf/model.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/perf/model.png -------------------------------------------------------------------------------- /tests/fixture/red.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/red.png -------------------------------------------------------------------------------- /tests/fixture/blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/blue.png -------------------------------------------------------------------------------- /tests/fixture/green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/green.png -------------------------------------------------------------------------------- /tests/fixture/grey.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/grey.png -------------------------------------------------------------------------------- /screenshot-tensorboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/screenshot-tensorboard.png -------------------------------------------------------------------------------- /mobile_app/data/tf-frozen_model.pb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/mobile_app/data/tf-frozen_model.pb -------------------------------------------------------------------------------- /tests/fixture/model_sum/last_weights.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/model_sum/last_weights.hdf5 -------------------------------------------------------------------------------- /docker/keras.json: -------------------------------------------------------------------------------- 1 | { 2 | "image_dim_ordering": "tf", 3 | "epsilon": 1e-07, 4 | "floatx": "float32", 5 | "backend": "tensorflow" 6 | } -------------------------------------------------------------------------------- /tests/fixture/model_convt2d/last_weights.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/model_convt2d/last_weights.hdf5 -------------------------------------------------------------------------------- /mobile_app/data/Linked_Frameworks_Libraries.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/mobile_app/data/Linked_Frameworks_Libraries.png -------------------------------------------------------------------------------- /tests/fixture/model_batchnorm/last_weights.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/model_batchnorm/last_weights.hdf5 -------------------------------------------------------------------------------- /tests/fixture/model_conv2d_relu/last_weights.hdf5: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/tests/fixture/model_conv2d_relu/last_weights.hdf5 -------------------------------------------------------------------------------- /mobile_app/.buckconfig: -------------------------------------------------------------------------------- 1 | 2 | [android] 3 | target = Google Inc.:Google APIs:23 4 | 5 | [maven_repositories] 6 | central = https://repo1.maven.org/maven2 7 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/RunModel.h: -------------------------------------------------------------------------------- 1 | #import 2 | 3 | @interface RunModel : NSObject 4 | 5 | + (CGFloat *)transformImage:(CGFloat *)imgTensor height:(int)height width:(int)width channels:(int)channels; 6 | 7 | @end 8 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/project.xcworkspace/contents.xcworkspacedata: -------------------------------------------------------------------------------- 1 | 2 | 4 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /data/urls.txt: -------------------------------------------------------------------------------- 1 | http://msvocds.blob.core.windows.net/coco2014/train2014.zip 2 | http://msvocds.blob.core.windows.net/coco2014/val2014.zip 3 | http://msvocds.blob.core.windows.net/coco2014/test2014.zip 4 | https://s3-eu-west-1.amazonaws.com/explee-deep-learning/paintings.zip -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/project.xcworkspace/xcuserdata/tom.xcuserdatad/UserInterfaceState.xcuserstate: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/metaflow-ai/neural_style/HEAD/mobile_app/ios/ReactNativeTF.xcodeproj/project.xcworkspace/xcuserdata/tom.xcuserdatad/UserInterfaceState.xcuserstate -------------------------------------------------------------------------------- /vgg19/download_weights.sh: -------------------------------------------------------------------------------- 1 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 2 | 3 | wget --no-check-certificate 'https://s3-eu-west-1.amazonaws.com/explee-deep-learning/vgg19_weights.h5' -O $DIR/vgg-19_weights.hdf5 4 | KERAS_BACKEND="theano" python $DIR/dump_headless_weights.py 5 | python $DIR/dump_headless_weights.py 6 | -------------------------------------------------------------------------------- /pretrain_model.sh: -------------------------------------------------------------------------------- 1 | if [ $1 ]; then 2 | MODEL=$1 3 | else 4 | MODEL='fast_st_ps' 5 | fi 6 | 7 | for i in $(seq 0 6); 8 | do 9 | python pretrain_model.py --training_mode overfit --model $MODEL --nb_epoch 100 --nb_res_layer $i 10 | python pretrain_model.py --training_mode identity --model $MODEL --nb_epoch 100 --nb_res_layer $i 11 | done -------------------------------------------------------------------------------- /predict.sh: -------------------------------------------------------------------------------- 1 | if [ $1 ]; then 2 | MDIR=$1 3 | else 4 | MDIR='st' 5 | fi 6 | 7 | python predict.py --models_dir $PWD/models/data/$MDIR 8 | for dir in $(ls -d data/output/$MDIR/*) 9 | do 10 | rm -f $PWD/$dir/mosaic.png 11 | montage $(ls -1 $PWD/$dir/* | grep '.*png' | xargs) -tile 10x9 -geometry +0+0 $PWD/$dir/mosaic.png 12 | # rm -f $dir/\d\d\d\d.* 13 | done -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cycler==0.10.0 2 | graphviz==0.4.10 3 | h5py==2.6.0 4 | Keras==1.0.4 5 | matplotlib==1.5.1 6 | numpy==1.11.2 7 | Pillow==3.2.0 8 | protobuf==3.0.0b2 9 | pydot-ng==1.0.0 10 | pydot3==1.0.8 11 | pyparsing==2.1.1 12 | pypng==0.0.18 13 | python-dateutil==2.5.3 14 | pytz==2016.4 15 | PyYAML==3.11 16 | scandir==1.2 17 | scipy==0.17.1 18 | six==1.10.0 19 | tensorflow==0.10.0 20 | Theano==0.8.2 21 | -------------------------------------------------------------------------------- /mobile_app/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ReactNativeTF", 3 | "author": "Thomas Olivier", 4 | "version": "0.0.1", 5 | "private": true, 6 | "scripts": { 7 | "start": "node node_modules/react-native/local-cli/cli.js start" 8 | }, 9 | "dependencies": { 10 | "react": "15.1.*", 11 | "react-native": "^0.27.*", 12 | "react-native-camera": "git+https://github.com/lwansbrough/react-native-camera.git" 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | import os 3 | 4 | from . import utils 5 | from . import models 6 | 7 | __version__ = '0.0.1' 8 | 9 | _data_dir = os.path.dirname(os.path.realpath(__file__)) + '/data' 10 | if not os.path.exists(_data_dir): 11 | os.makedirs(_data_dir) 12 | 13 | _models_data_dir = os.path.dirname(os.path.realpath(__file__)) + '/models/data' 14 | if not os.path.exists(_models_data_dir): 15 | os.makedirs(_models_data_dir) 16 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/UIImage+Alpha.h: -------------------------------------------------------------------------------- 1 | // UIImage+Alpha.h 2 | // Created by Trevor Harmon on 9/20/09. 3 | // Free for personal or commercial use, with or without modification. 4 | // No warranty is expressed or implied. 5 | @import UIKit; 6 | // Helper methods for adding an alpha layer to an image 7 | @interface UIImage (Alpha) 8 | - (BOOL)hasAlpha; 9 | - (UIImage *)imageWithAlpha; 10 | - (UIImage *)transparentBorderImage:(NSUInteger)borderSize; 11 | - (CGImageRef)newBorderMask:(NSUInteger)borderSize size:(CGSize)size; 12 | @end 13 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/AppDelegate.h: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #import 11 | 12 | @interface AppDelegate : UIResponder 13 | 14 | @property (nonatomic, strong) UIWindow *window; 15 | 16 | @end 17 | -------------------------------------------------------------------------------- /tests/fixture/model_batchnorm/archi.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 3, 3, 3], "name": "input_1", "input_dtype": "float32"}, "inbound_nodes": [], "name": "input_1"}, {"class_name": "BatchNormalization", "config": {"name": "batchnormalization_1", "epsilon": 1e-06, "trainable": true, "mode": 0, "momentum": 0.1, "axis": 1}, "inbound_nodes": [[["input_1", 0, 0]]], "name": "batchnormalization_1"}], "input_layers": [["input_1", 0, 0]], "output_layers": [["batchnormalization_1", 0, 0]], "name": "model_1"}} -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/UIImage+RoundedCorner.h: -------------------------------------------------------------------------------- 1 | // UIImage+RoundedCorner.h 2 | // Created by Trevor Harmon on 9/20/09. 3 | // Free for personal or commercial use, with or without modification. 4 | // No warranty is expressed or implied. 5 | @import UIKit; 6 | // Extends the UIImage class to support making rounded corners 7 | @interface UIImage (RoundedCorner) 8 | - (UIImage *)roundedCornerImage:(NSInteger)cornerSize borderSize:(NSInteger)borderSize; 9 | - (void)addRoundedRectToPath:(CGRect)rect context:(CGContextRef)context ovalWidth:(CGFloat)ovalWidth ovalHeight:(CGFloat)ovalHeight; 10 | @end 11 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/main.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #import 11 | 12 | #import "AppDelegate.h" 13 | 14 | int main(int argc, char * argv[]) { 15 | @autoreleasepool { 16 | return UIApplicationMain(argc, argv, nil, NSStringFromClass([AppDelegate class])); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /data/download_images.sh: -------------------------------------------------------------------------------- 1 | head -1 urls.txt | tail -1 | xargs wget --no-check-certificate -t 1 --timeout=5 2 | unzip train2014.zip && mv train2014 train 3 | head -2 urls.txt | tail -1 | xargs wget --no-check-certificate -t 1 --timeout=5 4 | unzip val2014.zip && mv val2014 val 5 | head -3 urls.txt | tail -1 | xargs wget --no-check-certificate -t 1 --timeout=5 6 | unzip test2014.zip && mv test2014 test 7 | head -4 urls.txt | tail -1 | xargs wget --no-check-certificate -t 1 --timeout=5 -P paintings/ 8 | cd ./paintings && unzip paintings.zip 9 | 10 | # Dump preprocess paintings for 600x600 images by default 11 | python prepare_data.py -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/ImageManager.h: -------------------------------------------------------------------------------- 1 | // 2 | // ImageManager.h 3 | // ReactNativeTorch 4 | // 5 | // Created by ThomasOlivier on 08/06/2016. 6 | // Copyright © 2016 Facebook. All rights reserved. 7 | // 8 | 9 | #ifndef ImageManager_h 10 | #define ImageManager_h 11 | 12 | 13 | #endif /* ImageManager_h */ 14 | 15 | #import 16 | #import "RCTBridgeModule.h" 17 | 18 | @interface ImageManager : NSObject 19 | 20 | + (CGFloat *)getRGBAsFromImage:(UIImage*)image; 21 | + (UIImage *)getImageFromRGBA:(CGFloat*)imgTensor channels:(int)channels width:(int)width height:(int)height; 22 | 23 | @end -------------------------------------------------------------------------------- /models/layers/ScaledSigmoid.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.engine import Layer 3 | 4 | class ScaledSigmoid(Layer): 5 | def __init__(self, scaling=1., **kwargs): 6 | self.supports_masking = False 7 | self.scaling = scaling 8 | super(ScaledSigmoid, self).__init__(**kwargs) 9 | 10 | 11 | def call(self, x, mask=None): 12 | return self.scaling * K.sigmoid(x / self.scaling) 13 | 14 | def get_config(self): 15 | config = {'scaling': self.scaling} 16 | base_config = super(ScaledSigmoid, self).get_config() 17 | return dict(list(base_config.items()) + list(config.items())) -------------------------------------------------------------------------------- /models/layers/__init__.py: -------------------------------------------------------------------------------- 1 | from .ConvolutionTranspose2D import ConvolutionTranspose2D 2 | from .ATrousConvolution2D import ATrousConvolution2D 3 | from .ScaledSigmoid import ScaledSigmoid 4 | from .PhaseShift import PhaseShift 5 | from .InstanceNormalization import InstanceNormalization 6 | from .ReflectPadding2D import ReflectPadding2D 7 | 8 | custom_objects={ 9 | 'ATrousConvolution2D': ATrousConvolution2D, 10 | 'ConvolutionTranspose2D': ConvolutionTranspose2D, 11 | 'ScaledSigmoid': ScaledSigmoid, 12 | 'PhaseShift': PhaseShift, 13 | 'InstanceNormalization': InstanceNormalization, 14 | 'ReflectPadding2D': ReflectPadding2D, 15 | } -------------------------------------------------------------------------------- /tests/fixture/model_sum/archi.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 3, 3, 3], "name": "input_1", "input_dtype": "float32"}, "inbound_nodes": [], "name": "input_1"}, {"class_name": "InputLayer", "config": {"batch_input_shape": [null, 3, 3, 3], "name": "input_2", "input_dtype": "float32"}, "inbound_nodes": [], "name": "input_2"}, {"class_name": "Merge", "config": {"name": "merge_1", "concat_axis": -1, "mode_type": "raw", "dot_axes": [-1, -1], "mode": "sum", "output_shape": null, "output_shape_type": "raw"}, "inbound_nodes": [[["input_1", 0, 0], ["input_2", 0, 0]]], "name": "merge_1"}], "input_layers": [["input_1", 0, 0], ["input_2", 0, 0]], "output_layers": [["merge_1", 0, 0]], "name": "model_1"}} -------------------------------------------------------------------------------- /docker/DOCKERFILE: -------------------------------------------------------------------------------- 1 | FROM morgangiraud/tensorflow-cuda7.5-cudnn5 2 | 3 | MAINTAINER Morgan Giraud 4 | 5 | RUN apt-get update && apt-get install -y \ 6 | libjpeg8-dev \ 7 | vim \ 8 | imagemagick \ 9 | wget \ 10 | && \ 11 | apt-get clean && \ 12 | rm -rf /var/lib/apt/lists/* 13 | 14 | RUN pip install \ 15 | cycler \ 16 | graphviz \ 17 | h5py \ 18 | Keras \ 19 | matplotlib \ 20 | numpy \ 21 | Pillow \ 22 | protobuf \ 23 | pydot \ 24 | pyparsing \ 25 | pypng \ 26 | python-dateutil \ 27 | pytz \ 28 | PyYAML \ 29 | scandir \ 30 | scipy \ 31 | six 32 | 33 | COPY keras.json /root/.keras/ -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/xcuserdata/tom.xcuserdatad/xcschemes/xcschememanagement.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | SchemeUserState 6 | 7 | ReactNativeTF.xcscheme_^#shared#^_ 8 | 9 | orderHint 10 | 0 11 | 12 | 13 | SuppressBuildableAutocreation 14 | 15 | 00E356ED1AD99517003FC87E 16 | 17 | primary 18 | 19 | 20 | 13B07F861A680F5B00A75B9A 21 | 22 | primary 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /tests/fixture/model_convt2d/archi.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 3, 4, 4], "name": "input", "input_dtype": "float32"}, "inbound_nodes": [], "name": "input"}, {"class_name": "ConvolutionTranspose2D", "config": {"W_constraint": null, "b_constraint": null, "name": "convolutiontranspose2d_1", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "he_normal", "bias": true, "nb_filter": 3, "border_mode": "same", "b_regularizer": null, "W_regularizer": null, "activation": "linear", "nb_row": 3}, "inbound_nodes": [[["input", 0, 0]]], "name": "convolutiontranspose2d_1"}], "input_layers": [["input", 0, 0]], "output_layers": [["convolutiontranspose2d_1", 0, 0]], "name": "model_1"}} -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/Images.xcassets/AppIcon.appiconset/Contents.json: -------------------------------------------------------------------------------- 1 | { 2 | "images" : [ 3 | { 4 | "idiom" : "iphone", 5 | "size" : "29x29", 6 | "scale" : "2x" 7 | }, 8 | { 9 | "idiom" : "iphone", 10 | "size" : "29x29", 11 | "scale" : "3x" 12 | }, 13 | { 14 | "idiom" : "iphone", 15 | "size" : "40x40", 16 | "scale" : "2x" 17 | }, 18 | { 19 | "idiom" : "iphone", 20 | "size" : "40x40", 21 | "scale" : "3x" 22 | }, 23 | { 24 | "idiom" : "iphone", 25 | "size" : "60x60", 26 | "scale" : "2x" 27 | }, 28 | { 29 | "idiom" : "iphone", 30 | "size" : "60x60", 31 | "scale" : "3x" 32 | } 33 | ], 34 | "info" : { 35 | "version" : 1, 36 | "author" : "xcode" 37 | } 38 | } -------------------------------------------------------------------------------- /utils/optimizers.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | def adam(x, dx, config=None): 4 | if config is None: config = {} 5 | config.setdefault('learning_rate', 1e-3) 6 | config.setdefault('beta1', 0.9) 7 | config.setdefault('beta2', 0.999) 8 | config.setdefault('epsilon', 1e-8) 9 | config.setdefault('m', np.zeros_like(x)) 10 | config.setdefault('v', np.zeros_like(x)) 11 | config.setdefault('t', 0) 12 | 13 | next_x = None 14 | config['t'] += 1 15 | config['m'] = config['beta1'] * config['m'] + (1 - config['beta1']) * dx 16 | config['v'] = config['beta2'] * config['v'] + (1 - config['beta2']) * dx**2 17 | m_hat = config['m'] / (1 - config['beta1']**config['t']) 18 | v_hat = config['v'] / (1 - config['beta2']**config['t']) 19 | next_x = x - config['learning_rate'] * m_hat / (np.sqrt(v_hat) + config['epsilon']) 20 | 21 | return next_x, config -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTFTests/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | org.reactjs.native.example.$(PRODUCT_NAME:rfc1034identifier) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | BNDL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1 23 | 24 | 25 | -------------------------------------------------------------------------------- /tests/fixture/model_conv2d_relu/archi.json: -------------------------------------------------------------------------------- 1 | {"class_name": "Model", "config": {"layers": [{"class_name": "InputLayer", "config": {"batch_input_shape": [null, 3, 3, 3], "name": "input", "input_dtype": "float32"}, "inbound_nodes": [], "name": "input"}, {"class_name": "Convolution2D", "config": {"W_constraint": null, "b_constraint": null, "name": "convolution2d_1", "activity_regularizer": null, "trainable": true, "dim_ordering": "th", "nb_col": 3, "subsample": [1, 1], "init": "he_normal", "bias": true, "nb_filter": 3, "border_mode": "same", "b_regularizer": null, "W_regularizer": null, "activation": "linear", "nb_row": 3}, "inbound_nodes": [[["input", 0, 0]]], "name": "convolution2d_1"}, {"class_name": "Activation", "config": {"activation": "relu", "trainable": true, "name": "activation_1"}, "inbound_nodes": [[["convolution2d_1", 0, 0]]], "name": "activation_1"}], "input_layers": [["input", 0, 0]], "output_layers": [["activation_1", 0, 0]], "name": "model_2"}} -------------------------------------------------------------------------------- /pre_analysis.sh: -------------------------------------------------------------------------------- 1 | python layer_reconstruction.py 2 | montage $(ls -d1 $PWD/data/output/vgg19/reconstruction/* | grep '\(content\|style\)_conv.*png' | xargs) -tile 16x2 -geometry +0+0 data/output/vgg19/reconstruction/layer_reconstruction.png 3 | # rm -f data/output/vgg19/reconstruction/.*(content|style)_conv.*png 4 | 5 | python ltv.py 6 | montage $(ls -d1 $PWD/data/output/vgg19/ltv/* | grep '_gamma.*png' | xargs) -tile 6x2 -geometry +0+0 data/output/vgg19/ltv/layer_ltv.png 7 | # rm -f data/output/vgg19/ltv/.*(content|style)_conv.*png 8 | 9 | python alpha.py 10 | montage $(ls -d1 $PWD/data/output/vgg19/alpha/* | grep 'styleconv.*png' | xargs) -tile 6x2 -geometry +0+0 data/output/vgg19/alpha/layer_alpha.png 11 | # rm -f data/output/vgg19/alpha/.*(content|style)_conv.*png 12 | 13 | python layer_influence.py 14 | montage $(ls -d1 $PWD/data/output/vgg19/influence/* | grep '_style.*_content.*png' | xargs) -tile 12x12 -geometry +0+0 data/output/vgg19/influence/layer_influence.png 15 | # rm -f data/output/vgg19/influence/.*(content|style)_conv.*png -------------------------------------------------------------------------------- /export_keras_model.py: -------------------------------------------------------------------------------- 1 | import os, argparse 2 | 3 | from keras import backend as K 4 | 5 | from utils.general import export_model, import_model 6 | from models.layers import custom_objects 7 | 8 | dir = os.path.dirname(os.path.realpath(__file__)) 9 | 10 | 11 | parser = argparse.ArgumentParser( 12 | description='Neural artistic style. Generates an image by combining ' 13 | 'the content of an image and the style of another.', 14 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 15 | ) 16 | parser.add_argument('--model_dir', type=str, help='Model absolute directory') 17 | args = parser.parse_args() 18 | 19 | if not os.path.isdir(args.model_dir): 20 | raise Exception("The model_dir is not a directory") 21 | model = import_model(args.model_dir, should_convert=False, custom_objects=custom_objects) 22 | print('Model input node name: %s' % model.input.name) 23 | print('Model output node name: %s' % model.output.name) 24 | 25 | if K._BACKEND == 'tensorflow': 26 | import tensorflow as tf 27 | saver = tf.train.Saver() 28 | else: 29 | saver = None 30 | export_model(model, args.model_dir, saver=saver) 31 | -------------------------------------------------------------------------------- /data/build_overfit_folder.sh: -------------------------------------------------------------------------------- 1 | DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" 2 | SIZE=600 3 | OVERFITDIR=$DIR/overfit_$SIZE 4 | OVERFITLABELSDIR=$OVERFITDIR/labels 5 | CVDIR=$OVERFITDIR/cv 6 | CVLABELSDIR=$CVDIR/labels 7 | 8 | mkdir -p $OVERFITLABELSDIR 9 | mkdir -p $CVLABELSDIR 10 | 11 | # Training files 12 | for file in $(ls $DIR/train | grep jpg | head -16) 13 | do 14 | convert $DIR/train/$file -resize ${SIZE}x${SIZE}\! $OVERFITDIR/$file 15 | done 16 | python gatys_paper.py --content $OVERFITDIR --image_size $SIZE --max_iter 500 --no_dump_losses 1 --output_dir $OVERFITLABELSDIR --alpha 100 --beta 1 --gamma 0.00005 17 | for file in $(ls $OVERFITLABELSDIR/ | grep jpg) 18 | do 19 | mv $OVERFITLABELSDIR/$file $OVERFITLABELSDIR/${file/_contentconv**\.jpg/.jpg} 20 | done 21 | 22 | # Cross validation files 23 | for file in $(ls $DIR/val | grep jpg | head -16) 24 | do 25 | convert $DIR/val/$file -resize ${SIZE}x${SIZE}\! $CVDIR/$file 26 | done 27 | python gatys_paper.py --content $CVDIR --image_size $SIZE --max_iter 500 --no_dump_losses 1 --output_dir $CVLABELSDIR --alpha 100 --beta 1 --gamma 0.00005 28 | for file in $(ls $CVLABELSDIR/ | grep jpg) 29 | do 30 | mv $CVLABELSDIR/$file $CVLABELSDIR/${file/_contentconv**\.jpg/.jpg} 31 | done 32 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/UIImage+Resize.h: -------------------------------------------------------------------------------- 1 | // UIImage+Resize.h 2 | // Created by Trevor Harmon on 8/5/09. 3 | // Free for personal or commercial use, with or without modification. 4 | // No warranty is expressed or implied. 5 | @import UIKit; 6 | // Extends the UIImage class to support resizing/cropping 7 | @interface UIImage (Resize) 8 | - (NSString *)base64String; 9 | - (UIImage *)croppedImage:(CGRect)bounds; 10 | - (UIImage *)thumbnailImage:(NSInteger)thumbnailSize 11 | transparentBorder:(NSUInteger)borderSize 12 | cornerRadius:(NSUInteger)cornerRadius 13 | interpolationQuality:(CGInterpolationQuality)quality; 14 | - (UIImage *)resizedImage:(CGSize)newSize 15 | interpolationQuality:(CGInterpolationQuality)quality; 16 | - (UIImage *)resizedImageWithContentMode:(UIViewContentMode)contentMode 17 | bounds:(CGSize)bounds 18 | interpolationQuality:(CGInterpolationQuality)quality; 19 | - (UIImage *)resizedImage:(CGSize)newSize 20 | transform:(CGAffineTransform)transform 21 | drawTransposed:(BOOL)transpose 22 | interpolationQuality:(CGInterpolationQuality)quality; 23 | - (CGAffineTransform)transformForOrientation:(CGSize)newSize; 24 | @end 25 | -------------------------------------------------------------------------------- /vgg19/dump_headless_weights.py: -------------------------------------------------------------------------------- 1 | import os, sys 2 | 3 | dir = os.path.dirname(os.path.realpath(__file__)) 4 | sys.path.append(dir + '/..') 5 | 6 | from keras import backend as K 7 | from utils.general import copySeqWeights 8 | from model_headless import VGG_19_headless_5, VGG_19_headless_4 9 | 10 | 11 | K.set_image_dim_ordering('tf') 12 | copySeqWeights(VGG_19_headless_5(input_shape=(256, 256, 3)), 13 | dir + '/vgg-19_weights.hdf5', 14 | dir + '/vgg-19-' + K.image_dim_ordering() + '-' + K._BACKEND + '_headless_5_weights.hdf5' 15 | ) 16 | 17 | K.set_image_dim_ordering('th') 18 | copySeqWeights(VGG_19_headless_5(input_shape=(3, 256, 256)), 19 | dir + '/vgg-19_weights.hdf5', 20 | dir + '/vgg-19-' + K.image_dim_ordering() + '-' + K._BACKEND + '_headless_5_weights.hdf5' 21 | ) 22 | 23 | K.set_image_dim_ordering('tf') 24 | copySeqWeights(VGG_19_headless_4(input_shape=(256, 256, 3)), 25 | dir + '/vgg-19_weights.hdf5', 26 | dir + '/vgg-19-' + K.image_dim_ordering() + '-' + K._BACKEND + '_headless_4_weights.hdf5' 27 | ) 28 | 29 | K.set_image_dim_ordering('th') 30 | copySeqWeights(VGG_19_headless_4(input_shape=(3, 256, 256)), 31 | dir + '/vgg-19_weights.hdf5', 32 | dir + '/vgg-19-' + K.image_dim_ordering() + '-' + K._BACKEND + '_headless_4_weights.hdf5' 33 | ) 34 | 35 | -------------------------------------------------------------------------------- /models/layers/InstanceNormalization.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.engine import Layer 3 | 4 | if K._BACKEND == 'theano': 5 | raise Exception('theano backend is not supported') 6 | else: 7 | import tensorflow as tf 8 | 9 | class InstanceNormalization(Layer): 10 | def __init__(self, name, **kwargs): 11 | self.supports_masking = False 12 | self.name = name 13 | 14 | super(InstanceNormalization, self).__init__(**kwargs) 15 | 16 | 17 | def call(self, X, mask=None): 18 | (mean, variance) = tf.nn.moments(X, [1, 2], keep_dims=True) 19 | shape = X.get_shape().as_list() 20 | # shape[0] = tf.shape(X)[0] 21 | # offset = tf.get_variable('%s-offset' % self.name, shape=[shape[0], 1, 1, shape[3]]) 22 | # scale = tf.get_variable('%s-scale' % self.name, shape=[shape[0], 1, 1, shape[3]]) 23 | variance_epsilon = 1e-7 24 | 25 | A = tf.nn.batch_normalization(X, mean, variance, None, None, variance_epsilon) 26 | Y = tf.nn.relu(A) 27 | 28 | return Y 29 | 30 | def get_config(self): 31 | config = { 32 | 'name': self.name 33 | } 34 | base_config = super(InstanceNormalization, self).get_config() 35 | return dict(list(base_config.items()) + list(config.items())) -------------------------------------------------------------------------------- /vgg19/README.md: -------------------------------------------------------------------------------- 1 | # VGG-16 2 | ## Information 3 | 4 | **Reference**: 5 | ```bibtex 6 | @article{DBLP:journals/corr/SimonyanZ14a, 7 | author = {Karen Simonyan and 8 | Andrew Zisserman}, 9 | title = {Very Deep Convolutional Networks for Large-Scale Image Recognition}, 10 | journal = {CoRR}, 11 | volume = {abs/1409.1556}, 12 | year = {2014}, 13 | url = {http://arxiv.org/abs/1409.1556}, 14 | timestamp = {Wed, 01 Oct 2014 15:00:05 +0200}, 15 | biburl = {http://dblp.uni-trier.de/rec/bib/journals/corr/SimonyanZ14a}, 16 | bibsource = {dblp computer science bibliography, http://dblp.org} 17 | } 18 | ``` 19 | 20 | **Framework used**: *Caffe* 21 | 22 | **License**: *unrestricted use* 23 | 24 | **Dataset used to train**: ILSVRC-2014 25 | 26 | 27 | ## Description 28 | 29 | This is the [Keras](http://keras.io/) model of the 16-layer network used by the VGG team in the ILSVRC-2014 competition. Project [site](http://www.robots.ox.ac.uk/~vgg/research/very_deep/). Gist where the model was obtained [here](https://gist.github.com/baraldilorenzo/07d7802847aaad0a35d3). 30 | 31 | It has been obtained by directly converting the [Caffe model](https://gist.github.com/ksimonyan/211839e770f7b538e2d8#file-readme-md) provived by the authors. 32 | 33 | In the paper, the VGG-16 model is denoted as configuration `D`. It achieves 7.5% top-5 error on ILSVRC-2012-val, 7.4% top-5 error on ILSVRC-2012-test. 34 | 35 | Please cite the paper if you use the models. 36 | -------------------------------------------------------------------------------- /gatys_paper_analysis.sh: -------------------------------------------------------------------------------- 1 | python gatys_paper.py --pooling_type max --input_type random 2 | for folder in $(ls -d data/output/vgg19/gatys_max_random*) 3 | do 4 | if [ "$(ls -A $PWD/$folder)" ]; then 5 | montage $(ls -d1 $PWD/$folder/* | grep '_contentconv.*png' | xargs) -tile 3x3 -geometry +0+0 $PWD/$folder/mosaic.png 6 | # rm -f $PWD/$folder/.*_contentconv.*png 7 | fi 8 | done 9 | 10 | python gatys_paper.py --pooling_type avg --input_type random 11 | for folder in $(ls -d data/output/vgg19/gatys_avg_random*) 12 | do 13 | if [ "$(ls -A $PWD/$folder)" ]; then 14 | montage $(ls -d1 $PWD/$folder/* | grep '_contentconv.*png' | xargs) -tile 3x3 -geometry +0+0 $PWD/$folder/mosaic.png 15 | # rm -f $PWD/$folder/.*_contentconv.*png 16 | fi 17 | done 18 | 19 | python gatys_paper.py --pooling_type max --input_type content 20 | for folder in $(ls -d data/output/vgg19/gatys_max_content*) 21 | do 22 | if [ "$(ls -A $PWD/$folder)" ]; then 23 | montage $(ls -d1 $PWD/$folder/* | grep '_contentconv.*png' | xargs) -tile 3x3 -geometry +0+0 $PWD/$folder/mosaic.png 24 | # rm -f $PWD/$folder/.*_contentconv.*png 25 | fi 26 | done 27 | 28 | python gatys_paper.py --pooling_type avg --input_type content 29 | for folder in $(ls -d data/output/vgg19/gatys_avg_content*) 30 | do 31 | if [ "$(ls -A $PWD/$folder)" ]; then 32 | montage $(ls -d1 $PWD/$folder/* | grep '_contentconv.*png' | xargs) -tile 3x3 -geometry +0+0 $PWD/$folder/mosaic.png 33 | # rm -f $PWD/$folder/.*_contentconv.*png 34 | fi 35 | done 36 | 37 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/Info.plist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | CFBundleDevelopmentRegion 6 | en 7 | CFBundleExecutable 8 | $(EXECUTABLE_NAME) 9 | CFBundleIdentifier 10 | $(PRODUCT_BUNDLE_IDENTIFIER) 11 | CFBundleInfoDictionaryVersion 12 | 6.0 13 | CFBundleName 14 | $(PRODUCT_NAME) 15 | CFBundlePackageType 16 | APPL 17 | CFBundleShortVersionString 18 | 1.0 19 | CFBundleSignature 20 | ???? 21 | CFBundleVersion 22 | 1 23 | LSRequiresIPhoneOS 24 | 25 | NSAppTransportSecurity 26 | 27 | NSAllowsArbitraryLoads 28 | 29 | 30 | NSLocationWhenInUseUsageDescription 31 | 32 | UILaunchStoryboardName 33 | LaunchScreen 34 | UIRequiredDeviceCapabilities 35 | 36 | armv7 37 | 38 | UISupportedInterfaceOrientations 39 | 40 | UIInterfaceOrientationPortrait 41 | UIInterfaceOrientationLandscapeLeft 42 | UIInterfaceOrientationLandscapeRight 43 | 44 | UIViewControllerBasedStatusBarAppearance 45 | 46 | 47 | 48 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/project.xcworkspace/xcshareddata/ReactNativeTF.xcscmblueprint: -------------------------------------------------------------------------------- 1 | { 2 | "DVTSourceControlWorkspaceBlueprintPrimaryRemoteRepositoryKey" : "7A45C7EE3431A70E12FBADF5D728BB6CB0D9F905", 3 | "DVTSourceControlWorkspaceBlueprintWorkingCopyRepositoryLocationsKey" : { 4 | 5 | }, 6 | "DVTSourceControlWorkspaceBlueprintWorkingCopyStatesKey" : { 7 | "9F4E48B0E71B018CFB650DCC9B8A187E5C305294" : 0, 8 | "7A45C7EE3431A70E12FBADF5D728BB6CB0D9F905" : 0 9 | }, 10 | "DVTSourceControlWorkspaceBlueprintIdentifierKey" : "FC4E5238-2F79-46A8-91A5-0A781CC03448", 11 | "DVTSourceControlWorkspaceBlueprintWorkingCopyPathsKey" : { 12 | "9F4E48B0E71B018CFB650DCC9B8A187E5C305294" : "", 13 | "7A45C7EE3431A70E12FBADF5D728BB6CB0D9F905" : "mobile_app\/" 14 | }, 15 | "DVTSourceControlWorkspaceBlueprintNameKey" : "ReactNativeTF", 16 | "DVTSourceControlWorkspaceBlueprintVersion" : 204, 17 | "DVTSourceControlWorkspaceBlueprintRelativePathToProjectKey" : "ios\/ReactNativeTF.xcodeproj", 18 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoriesKey" : [ 19 | { 20 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryURLKey" : "https:\/\/github.com\/Explee\/ReactNativeTF.git", 21 | "DVTSourceControlWorkspaceBlueprintRemoteRepositorySystemKey" : "com.apple.dt.Xcode.sourcecontrol.Git", 22 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryIdentifierKey" : "7A45C7EE3431A70E12FBADF5D728BB6CB0D9F905" 23 | }, 24 | { 25 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryURLKey" : "https:\/\/github.com\/Explee\/deepback.git", 26 | "DVTSourceControlWorkspaceBlueprintRemoteRepositorySystemKey" : "com.apple.dt.Xcode.sourcecontrol.Git", 27 | "DVTSourceControlWorkspaceBlueprintRemoteRepositoryIdentifierKey" : "9F4E48B0E71B018CFB650DCC9B8A187E5C305294" 28 | } 29 | ] 30 | } -------------------------------------------------------------------------------- /models/layers/ReflectPadding2D.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.engine import Layer, InputSpec 3 | 4 | if K._BACKEND == 'theano': 5 | raise Exception('theano backend is not supported') 6 | else: 7 | import tensorflow as tf 8 | 9 | 10 | class ReflectPadding2D(Layer): 11 | def __init__(self, padding=(1, 1), **kwargs): 12 | super(ReflectPadding2D, self).__init__(**kwargs) 13 | self.padding = tuple(padding) 14 | self.dim_ordering = K.image_dim_ordering() 15 | self.input_spec = [InputSpec(ndim=4)] 16 | 17 | def get_output_shape_for(self, input_shape): 18 | if self.dim_ordering == 'th': 19 | width = input_shape[2] + 2 * self.padding[0] if input_shape[2] is not None else None 20 | height = input_shape[3] + 2 * self.padding[1] if input_shape[3] is not None else None 21 | return (input_shape[0], 22 | input_shape[1], 23 | width, 24 | height) 25 | elif self.dim_ordering == 'tf': 26 | width = input_shape[1] + 2 * self.padding[0] if input_shape[1] is not None else None 27 | height = input_shape[2] + 2 * self.padding[1] if input_shape[2] is not None else None 28 | return (input_shape[0], 29 | width, 30 | height, 31 | input_shape[3]) 32 | else: 33 | raise Exception('Invalid dim_ordering: ' + self.dim_ordering) 34 | 35 | def call(self, x, mask=None): 36 | return tf.pad(x, [ 37 | [ 0, 0 ], 38 | [ self.padding[0], self.padding[0] ], 39 | [ self.padding[1], self.padding[1] ], 40 | [ 0, 0 ] 41 | ], "REFLECT") 42 | 43 | def get_config(self): 44 | config = {'padding': self.padding} 45 | base_config = super(ReflectPadding2D, self).get_config() 46 | return dict(list(base_config.items()) + list(config.items())) -------------------------------------------------------------------------------- /torch/forward.lua: -------------------------------------------------------------------------------- 1 | require 'torch' 2 | require 'nn' 3 | require 'nngraph' 4 | require 'image' 5 | 6 | -------------------------------------------------------------------------------- 7 | 8 | local cmd = torch.CmdLine() 9 | 10 | -- Basic options 11 | cmd:option('-input_img', '../tests/fixture/blue.png', 12 | 'Image to be processsed') 13 | cmd:option('-gpu', -1, 'Zero-indexed ID of the GPU to use; for CPU mode set -gpu = -1') 14 | cmd:option('-output_image', 'out.png') 15 | cmd:option('-image_size', 3, 'Maximum height / width of generated image') 16 | 17 | cmd:option('-model_name', 'models/my_model.dat') 18 | 19 | local function main(params) 20 | -- Load input img 21 | inputImg = loadImg(params.input_img, params.image_size) 22 | 23 | -- Load net 24 | net = torch.load(params.model_name) 25 | 26 | -- Forward input img 27 | local outImg = net:forward(inputImg) 28 | -- print(inputImg, outImg:int()) 29 | 30 | -- Sage output img 31 | saveImg(outImg, params.output_image) 32 | end 33 | 34 | function loadImg(path, size) 35 | local inputImg = image.load(path, 3) 36 | inputImg = image.scale(inputImg, size, 'bilinear') 37 | inputImg = preprocess(inputImg) 38 | 39 | return inputImg 40 | end 41 | 42 | -- We need to rescale from [0, 1] to [0, 255], convert from RGB to BGR, 43 | function preprocess(img) 44 | local perm = torch.LongTensor{3, 2, 1} 45 | local dims = #img 46 | img = img:index(1, perm):mul(255.0):resize(1, dims[1], dims[2], dims[3]):double() 47 | return img 48 | end 49 | 50 | function saveImg(img, filename) 51 | local disp = deprocess(img:double()) 52 | disp = image.minmax{tensor=disp, min=0, max=1} 53 | image.save(filename, disp) 54 | end 55 | 56 | 57 | -- Undo the above preprocessing. 58 | function deprocess(img) 59 | img = img[1] 60 | local perm = torch.LongTensor{3, 2, 1} 61 | img = img:index(1, perm):div(255.0) 62 | return img 63 | end 64 | 65 | 66 | local params = cmd:parse(arg) 67 | main(params) -------------------------------------------------------------------------------- /models/layers/PhaseShift.py: -------------------------------------------------------------------------------- 1 | from keras import backend as K 2 | from keras.engine import Layer 3 | 4 | if K._BACKEND == 'theano': 5 | raise Exception('theano backend is not supported') 6 | else: 7 | import tensorflow as tf 8 | 9 | class PhaseShift(Layer): 10 | def __init__(self, ratio=4, color=False, **kwargs): 11 | self.supports_masking = False 12 | self.ratio = ratio 13 | self.color = color 14 | 15 | super(PhaseShift, self).__init__(**kwargs) 16 | 17 | def _phase_shift(self, I, r): 18 | bsize, a, b, c = I.get_shape().as_list() 19 | bsize = tf.shape(I)[0] 20 | X = tf.reshape(I, (bsize, a, b, r, r)) 21 | X = tf.transpose(X, (0, 1, 2, 4, 3)) # bsize, a, b, 1, 1 22 | X = tf.split(1, a, X) # a, [bsize, b, r, r] 23 | X = tf.concat(2, [tf.squeeze(x) for x in X]) # bsize, b, a*r, r 24 | X = tf.split(1, b, X) # b, [bsize, a*r, r] 25 | X = tf.concat(2, [tf.squeeze(x) for x in X]) # bsize, a*r, b*r 26 | return tf.reshape(X, (bsize, a*r, b*r, 1)) 27 | 28 | def call(self, X, mask=None): 29 | total_channels = X.get_shape()[3] 30 | if total_channels % (self.ratio * self.ratio) != 0: 31 | raise ValueError('total_channels % (self.ratio * self.ratio) must equal to 0') 32 | 33 | nb_output_channels = total_channels // (self.ratio * self.ratio) 34 | Xc = tf.split(3, nb_output_channels, X) 35 | Y = tf.concat(3, [self._phase_shift(x, self.ratio) for x in Xc]) 36 | 37 | return Y 38 | 39 | def get_output_shape_for(self, input_shape): 40 | return (input_shape[0], input_shape[1] * self.ratio, input_shape[2] * self.ratio, input_shape[3] // (self.ratio * self.ratio)) 41 | 42 | def get_config(self): 43 | config = { 44 | 'ratio': self.ratio, 45 | 'color': self.color, 46 | } 47 | base_config = super(PhaseShift, self).get_config() 48 | return dict(list(base_config.items()) + list(config.items())) -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTFTests/ReactNativeTFTests.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #import 11 | #import 12 | 13 | #import "RCTLog.h" 14 | #import "RCTRootView.h" 15 | 16 | #define TIMEOUT_SECONDS 600 17 | #define TEXT_TO_LOOK_FOR @"Welcome to React Native!" 18 | 19 | @interface ReactNativeTFTests : XCTestCase 20 | 21 | @end 22 | 23 | @implementation ReactNativeTFTests 24 | 25 | - (BOOL)findSubviewInView:(UIView *)view matching:(BOOL(^)(UIView *view))test 26 | { 27 | if (test(view)) { 28 | return YES; 29 | } 30 | for (UIView *subview in [view subviews]) { 31 | if ([self findSubviewInView:subview matching:test]) { 32 | return YES; 33 | } 34 | } 35 | return NO; 36 | } 37 | 38 | - (void)testRendersWelcomeScreen 39 | { 40 | UIViewController *vc = [[[[UIApplication sharedApplication] delegate] window] rootViewController]; 41 | NSDate *date = [NSDate dateWithTimeIntervalSinceNow:TIMEOUT_SECONDS]; 42 | BOOL foundElement = NO; 43 | 44 | __block NSString *redboxError = nil; 45 | RCTSetLogFunction(^(RCTLogLevel level, RCTLogSource source, NSString *fileName, NSNumber *lineNumber, NSString *message) { 46 | if (level >= RCTLogLevelError) { 47 | redboxError = message; 48 | } 49 | }); 50 | 51 | while ([date timeIntervalSinceNow] > 0 && !foundElement && !redboxError) { 52 | [[NSRunLoop mainRunLoop] runMode:NSDefaultRunLoopMode beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; 53 | [[NSRunLoop mainRunLoop] runMode:NSRunLoopCommonModes beforeDate:[NSDate dateWithTimeIntervalSinceNow:0.1]]; 54 | 55 | foundElement = [self findSubviewInView:vc.view matching:^BOOL(UIView *view) { 56 | if ([view.accessibilityLabel isEqualToString:TEXT_TO_LOOK_FOR]) { 57 | return YES; 58 | } 59 | return NO; 60 | }]; 61 | } 62 | 63 | RCTSetLogFunction(RCTDefaultLogFunction); 64 | 65 | XCTAssertNil(redboxError, @"RedBox error: %@", redboxError); 66 | XCTAssertTrue(foundElement, @"Couldn't find element with text '%@' in %d seconds", TEXT_TO_LOOK_FOR, TIMEOUT_SECONDS); 67 | } 68 | 69 | 70 | @end 71 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/AppDelegate.m: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright (c) 2015-present, Facebook, Inc. 3 | * All rights reserved. 4 | * 5 | * This source code is licensed under the BSD-style license found in the 6 | * LICENSE file in the root directory of this source tree. An additional grant 7 | * of patent rights can be found in the PATENTS file in the same directory. 8 | */ 9 | 10 | #import "AppDelegate.h" 11 | 12 | #import "RCTRootView.h" 13 | 14 | @implementation AppDelegate 15 | 16 | - (BOOL)application:(UIApplication *)application didFinishLaunchingWithOptions:(NSDictionary *)launchOptions 17 | { 18 | NSURL *jsCodeLocation; 19 | 20 | /** 21 | * Loading JavaScript code - uncomment the one you want. 22 | * 23 | * OPTION 1 24 | * Load from development server. Start the server from the repository root: 25 | * 26 | * $ npm start 27 | * 28 | * To run on device, change `localhost` to the IP address of your computer 29 | * (you can get this by typing `ifconfig` into the terminal and selecting the 30 | * `inet` value under `en0:`) and make sure your computer and iOS device are 31 | * on the same Wi-Fi network. 32 | */ 33 | 34 | jsCodeLocation = [NSURL URLWithString:@"http://localhost:8081/index.ios.bundle?platform=ios&dev=true"]; 35 | 36 | /** 37 | * OPTION 2 38 | * Load from pre-bundled file on disk. The static bundle is automatically 39 | * generated by the "Bundle React Native code and images" build step when 40 | * running the project on an actual device or running the project on the 41 | * simulator in the "Release" build configuration. 42 | */ 43 | 44 | // jsCodeLocation = [[NSBundle mainBundle] URLForResource:@"main" withExtension:@"jsbundle"]; 45 | 46 | RCTRootView *rootView = [[RCTRootView alloc] initWithBundleURL:jsCodeLocation 47 | moduleName:@"ReactNativeTF" 48 | initialProperties:nil 49 | launchOptions:launchOptions]; 50 | rootView.backgroundColor = [[UIColor alloc] initWithRed:1.0f green:1.0f blue:1.0f alpha:1]; 51 | 52 | self.window = [[UIWindow alloc] initWithFrame:[UIScreen mainScreen].bounds]; 53 | UIViewController *rootViewController = [UIViewController new]; 54 | rootViewController.view = rootView; 55 | self.window.rootViewController = rootViewController; 56 | [self.window makeKeyAndVisible]; 57 | return YES; 58 | } 59 | 60 | @end 61 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # OSX 2 | .DS_Store 3 | 4 | # Project 5 | data/**/*.jpeg 6 | data/**/*.jpg 7 | data/**/*.png 8 | data/**/*.hdf5 9 | data/output/* 10 | data/overfit* 11 | vgg19/vgg-19*weights* 12 | models/data/** 13 | models/*.hdf5 14 | tmp/* 15 | torch/**/*.png 16 | torch/**/*.jpe?g 17 | torch/**/my_model* 18 | 19 | # Byte-compiled / optimized / DLL files 20 | __pycache__/ 21 | *.py[cod] 22 | *$py.class 23 | 24 | # C extensions 25 | *.so 26 | 27 | # Distribution / packaging 28 | .Python 29 | env/ 30 | build/ 31 | develop-eggs/ 32 | dist/ 33 | downloads/ 34 | eggs/ 35 | .eggs/ 36 | lib/ 37 | lib64/ 38 | parts/ 39 | sdist/ 40 | var/ 41 | *.egg-info/ 42 | .installed.cfg 43 | *.egg 44 | 45 | # PyInstaller 46 | # Usually these files are written by a python script from a template 47 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 48 | *.manifest 49 | *.spec 50 | 51 | # Installer logs 52 | pip-log.txt 53 | pip-delete-this-directory.txt 54 | 55 | # Unit test / coverage reports 56 | htmlcov/ 57 | .tox/ 58 | .coverage 59 | .coverage.* 60 | .cache 61 | nosetests.xml 62 | coverage.xml 63 | *,cover 64 | .hypothesis/ 65 | 66 | # Translations 67 | *.mo 68 | *.pot 69 | 70 | # Django stuff: 71 | *.log 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | target/ 78 | 79 | #Ipython Notebook 80 | .ipynb_checkpoints 81 | 82 | ## MOBILE 83 | 84 | # OSX 85 | # 86 | mobile_app/.DS_Store 87 | 88 | # Xcode 89 | # 90 | mobile_app/build/ 91 | mobile_app/*.pbxuser 92 | mobile_app/!default.pbxuser 93 | mobile_app/*.mode1v3 94 | mobile_app/!default.mode1v3 95 | mobile_app/*.mode2v3 96 | mobile_app/!default.mode2v3 97 | mobile_app/*.perspectivev3 98 | mobile_app/!default.perspectivev3 99 | mobile_app/**/xcuserdata/ 100 | mobile_app/*.xccheckout 101 | mobile_app/*.moved-aside 102 | mobile_app/DerivedData 103 | mobile_app/*.hmap 104 | mobile_app/*.ipa 105 | mobile_app/*.xcuserstate 106 | mobile_app/project.xcworkspace 107 | 108 | # Android/IJ 109 | # 110 | mobile_app/.idea 111 | mobile_app/.gradle 112 | mobile_app/local.properties 113 | 114 | # node.js 115 | # 116 | mobile_app/node_modules/ 117 | mobile_app/npm-debug.log 118 | 119 | # BUCK 120 | mobile_app/buck-out/ 121 | mobile_app/\.buckd/ 122 | mobile_app/android/app/libs 123 | mobile_app/android/keystores/debug.keystore 124 | 125 | #LIBS (should be generated by each on its computer) 126 | mobile_app/lib/ 127 | 128 | -------------------------------------------------------------------------------- /perf/st_perf.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os, sys, time, argparse, re 4 | import numpy as np 5 | 6 | dir = os.path.dirname(os.path.realpath(__file__)) 7 | sys.path.append(dir + '/..') 8 | 9 | from keras import backend as K 10 | 11 | from utils.imutils import load_images, resize 12 | from utils.general import import_model 13 | from models.layers import custom_objects 14 | 15 | if K._BACKEND == "tensorflow": 16 | K.set_image_dim_ordering('tf') 17 | else: 18 | K.set_image_dim_ordering('th') 19 | 20 | dir = os.path.dirname(os.path.realpath(__file__)) + '/..' 21 | dataDir = dir + '/data' 22 | output_dir = dataDir + '/output' 23 | test_dir = dataDir + '/test' 24 | 25 | parser = argparse.ArgumentParser( 26 | description='Neural artistic style. Generates an image by combining ' 27 | 'the content of an image and the style of another.', 28 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 29 | ) 30 | parser.add_argument('--models_dir', default=dir + '/models/data/st', type=str, help='Models top directories.') 31 | parser.add_argument('--batch_size', default=30, type=int, help='batch size.') 32 | parser.add_argument('--image_size', default=600, type=int, help='Input image size.') 33 | args = parser.parse_args() 34 | 35 | width = args.image_size 36 | height = args.image_size 37 | 38 | X_test = load_images(test_dir, limit=args.batch_size, size=(height, width), preprocess_type='st', verbose=True) 39 | print('X_test.shape: ' + str(X_test.shape)) 40 | 41 | current_iter = 0 42 | subdirs = [x[0] for x in os.walk(args.models_dir)] 43 | subdirs.pop(0) # First element is the parent dir 44 | for idx, absolute_model_dir in enumerate(subdirs): 45 | print('Loading model in %s' % absolute_model_dir) 46 | st_model = import_model(absolute_model_dir, best=True, 47 | should_convert=False, custom_objects=custom_objects) 48 | 49 | if not len(re.findall('superresolution', absolute_model_dir)): 50 | print('Timing batching') 51 | start = time.clock() 52 | results = st_model.predict(X_test) 53 | end = time.clock() 54 | duration_batch = (end-start)/X_test.shape[0] 55 | 56 | print('Timing looping') 57 | start = time.clock() 58 | num_loop = X_test.shape[0] 59 | for i in range(num_loop): 60 | if len(re.findall('superresolution', absolute_model_dir)): 61 | im = resize(X_test[0], (args.image_size/4, args.image_size/4)) 62 | else: 63 | im = X_test[0] 64 | results = st_model.predict(np.array([im])) 65 | end = time.clock() 66 | duration_loop = (end-start)/num_loop 67 | 68 | print("duration taken on 1 average call when batching: " + str(duration_batch)) 69 | print("duration taken on 1 average call when looping: " + str(duration_loop)) -------------------------------------------------------------------------------- /mobile_app/README.md: -------------------------------------------------------------------------------- 1 | # React-Native + Tensorflow 2 | 3 | **Beware! The tensorflow master branch is moving a lot! One day you might be able to compile it, the other day you may not. We won't comment on any tensorflow compilation related issues here** 4 | 5 | This folder is an example of how to use Tensorflow with iOS to run a neural-style transfer network. 6 | 7 | ### Tensorflow: 8 | - Clone tensorflow: `git clone git@github.com:tensorflow/tensorflow.git && cd tensorflow` where you cloned deepback. 9 | - Install all depedencies needed for [**compiling from sources**](https://www.tensorflow.org/versions/r0.9/get_started/os_setup.html#installing-from-sources) 10 | - Regenerate deps files: `tensorflow/contrib/makefile/gen_file_lists.sh` 11 | - Build the libs: 12 | ```bash 13 | cd tensorflow/contrib/makefile 14 | 15 | # The 'abs()' operations is missing from the build file, which is a major problem since 16 | # we use it, so first add this line in tf_op_files.txt 17 | # tensorflow/core/kernels/cwise_op_abs.cc 18 | 19 | ./build_all_ios.sh 20 | mkdir ../../../../deepback/mobile_app/lib/ 21 | cp gen/lib/libtensorflow-core.a ../../../../deepback/mobile_app/lib/ 22 | cp gen/protobuf_ios/lib/libprotobuf-lite.a ../../../../deepback/mobile_app/lib/ 23 | cp gen/protobuf_ios/lib/libprotobuf.a ../../../../deepback/mobile_app/lib/ 24 | ``` 25 | 26 | If you have any trouble with the Tensorflow's iOS build, please check the official doc [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/makefile/README.md#ios) 27 | 28 | ### React-Native 29 | Install the mobile app's dependencies 30 | ```bash 31 | cd mobile_app && npm i 32 | ``` 33 | 34 | ### Xcode 35 | - Open the project located at `deepback/mobile_app/ios/ReactNativeTF.xcodeproj` with XCode. 36 | 37 | If you cannot see the libRCTCamera.a added to your *Linked Frameworks and Libraries*, you should link it: 38 | 39 | ![alt text](data/Linked_Frameworks_Libraries.png "Your linked Frameworks and Libs") 40 | 41 | ```bash 42 | npm install rnpm --global 43 | rnpm link react-native-camera 44 | ``` 45 | 46 | or, if you'd rather [manual install](https://github.com/lwansbrough/react-native-camera#ios) 47 | - Build the project and run it from the Simulator. 48 | 49 | ### Run the project from your phone 50 | 51 | To run the project from your phone, you'll have to package it. Edit `deepback/mobile_app/ios/ReactNativeTF/AppDelegate.m` 52 | 53 | Comment this line 54 | ```javascript 55 | jsCodeLocation = [NSURL URLWithString:@"http://localhost:8081/index.ios.bundle?platform=ios&dev=true"]; 56 | ``` 57 | 58 | And uncomment his line 59 | ```javascript 60 | jsCodeLocation = [[NSBundle mainBundle] URLForResource:@"main" withExtension:@"jsbundle"]; 61 | ``` 62 | 63 | Then, from XCode, change the build target to select your connected iPhone. 64 | 65 | 66 | ## Problems 67 | - `Missing op` : Check tensorflow/contrib/makefile/tf_op_files.txt and add yout missing depepdencies -------------------------------------------------------------------------------- /vgg19/model.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | 3 | from keras.layers.convolutional import (Convolution2D, MaxPooling2D, 4 | ZeroPadding2D) 5 | from keras.layers.core import Dense, Dropout, Flatten 6 | from keras.models import Sequential 7 | 8 | 9 | def VGG_19(weights_path=None): 10 | model = Sequential() 11 | model.add(ZeroPadding2D((1,1),input_shape=(3,224,224))) 12 | model.add(Convolution2D(64, 3, 3, activation='relu')) 13 | model.add(ZeroPadding2D((1,1))) 14 | model.add(Convolution2D(64, 3, 3, activation='relu')) 15 | model.add(MaxPooling2D((2,2), strides=(2,2))) 16 | 17 | model.add(ZeroPadding2D((1,1))) 18 | model.add(Convolution2D(128, 3, 3, activation='relu')) 19 | model.add(ZeroPadding2D((1,1))) 20 | model.add(Convolution2D(128, 3, 3, activation='relu')) 21 | model.add(MaxPooling2D((2,2), strides=(2,2))) 22 | 23 | model.add(ZeroPadding2D((1,1))) 24 | model.add(Convolution2D(256, 3, 3, activation='relu')) 25 | model.add(ZeroPadding2D((1,1))) 26 | model.add(Convolution2D(256, 3, 3, activation='relu')) 27 | model.add(ZeroPadding2D((1,1))) 28 | model.add(Convolution2D(256, 3, 3, activation='relu')) 29 | model.add(ZeroPadding2D((1,1))) 30 | model.add(Convolution2D(256, 3, 3, activation='relu')) 31 | model.add(MaxPooling2D((2,2), strides=(2,2))) 32 | 33 | model.add(ZeroPadding2D((1,1))) 34 | model.add(Convolution2D(512, 3, 3, activation='relu')) 35 | model.add(ZeroPadding2D((1,1))) 36 | model.add(Convolution2D(512, 3, 3, activation='relu')) 37 | model.add(ZeroPadding2D((1,1))) 38 | model.add(Convolution2D(512, 3, 3, activation='relu')) 39 | model.add(ZeroPadding2D((1,1))) 40 | model.add(Convolution2D(512, 3, 3, activation='relu')) 41 | model.add(MaxPooling2D((2,2), strides=(2,2))) 42 | 43 | model.add(ZeroPadding2D((1,1))) 44 | model.add(Convolution2D(512, 3, 3, activation='relu')) 45 | model.add(ZeroPadding2D((1,1))) 46 | model.add(Convolution2D(512, 3, 3, activation='relu')) 47 | model.add(ZeroPadding2D((1,1))) 48 | model.add(Convolution2D(512, 3, 3, activation='relu')) 49 | model.add(ZeroPadding2D((1,1))) 50 | model.add(Convolution2D(512, 3, 3, activation='relu')) 51 | model.add(MaxPooling2D((2,2), strides=(2,2))) 52 | 53 | model.add(Flatten()) 54 | model.add(Dense(4096, activation='relu')) 55 | model.add(Dropout(0.5)) 56 | model.add(Dense(4096, activation='relu')) 57 | model.add(Dropout(0.5)) 58 | model.add(Dense(1000, activation='softmax')) 59 | 60 | if weights_path: 61 | model.load_weights(weights_path) 62 | 63 | return model 64 | 65 | # The mean is ordered as BGR 66 | def VGG_19_mean(dim_ordering='tf'): 67 | if dim_ordering == 'th': 68 | return np.array([[[[103.939]], [[116.779]], [[123.68]]]]) 69 | elif dim_ordering == 'tf': 70 | return np.array([[[[103.939, 116.779, 123.68]]]]) 71 | else: 72 | raise Exception('Invalid dim_ordering: ' + dim_ordering) 73 | -------------------------------------------------------------------------------- /tests/utils/test_general.py: -------------------------------------------------------------------------------- 1 | import os, sys, unittest 2 | from scipy import misc 3 | import numpy as np 4 | 5 | dir = os.path.dirname(os.path.realpath(__file__)) 6 | sys.path.append(dir + '/../..') 7 | 8 | from keras import backend as K 9 | from keras.layers.convolutional import Convolution2D 10 | from keras.layers.normalization import BatchNormalization 11 | from keras.layers.core import Activation 12 | from keras.layers import Input 13 | from keras.models import Model 14 | 15 | from utils.general import export_model, import_model 16 | from utils.imutils import load_image 17 | 18 | dir = os.path.dirname(os.path.realpath(__file__)) 19 | 20 | class TestGeneralUtils(unittest.TestCase): 21 | 22 | def test_export_model(self): 23 | input = Input(shape=(3, 4, 4), name='input', dtype='float32') 24 | out = Convolution2D(3, 3, 3, 25 | init='he_normal', subsample=(1, 1), border_mode='same', activation='linear')(input) 26 | out = Activation('relu')(out) 27 | model = Model(input=[input], output=[out]) 28 | 29 | data_model_folder = dir + "/../fixture/model_export" 30 | if K._BACKEND == 'tensorflow': 31 | import tensorflow as tf 32 | saver = tf.train.Saver() 33 | else: 34 | saver = None 35 | export_model(model, data_model_folder, saver=saver) 36 | 37 | os.remove(data_model_folder + '/archi.json') 38 | os.remove(data_model_folder + '/last_weights.hdf5') 39 | if K._BACKEND == 'tensorflow': 40 | os.remove(data_model_folder + '/checkpoint') 41 | os.remove(data_model_folder + '/tf-last_weights') 42 | os.remove(data_model_folder + '/tf-last_weights.meta') 43 | os.remove(data_model_folder + '/tf-model_graph') 44 | os.remove(data_model_folder + '/tf-frozen_model.pb') 45 | os.rmdir(data_model_folder) 46 | 47 | def test_import_model(self): 48 | data_model_folder = dir + "/../fixture/model_conv2d_relu" 49 | 50 | should_convert = K._BACKEND == "theano" 51 | model = import_model(data_model_folder, should_convert=should_convert) 52 | input_img = np.array([load_image(dir + '/../fixture/blue.png', size=None, preprocess_type='st', verbose=False)]) 53 | 54 | 55 | 56 | output = model.predict([input_img]).astype('int32') 57 | true_output = np.array([ 58 | [ 59 | [ 60 | [0, 0, 0], 61 | [0, 0, 0], 62 | [0, 0, 0] 63 | ], 64 | [ 65 | [131, 116, 153], 66 | [153, 281, 364], 67 | [103, 254, 318] 68 | ], 69 | [ 70 | [52, 1, 0], 71 | [0, 0, 0], 72 | [0, 0, 0] 73 | ] 74 | ] 75 | ]) 76 | 77 | self.assertEqual(len(model.layers),3) 78 | self.assertEqual(True, (output==true_output).all()) 79 | 80 | if __name__ == '__main__': 81 | unittest.main() -------------------------------------------------------------------------------- /mobile_app/.flowconfig: -------------------------------------------------------------------------------- 1 | [ignore] 2 | 3 | # We fork some components by platform. 4 | .*/*.web.js 5 | .*/*.android.js 6 | 7 | # Some modules have their own node_modules with overlap 8 | .*/node_modules/node-haste/.* 9 | 10 | # Ugh 11 | .*/node_modules/babel.* 12 | .*/node_modules/babylon.* 13 | .*/node_modules/invariant.* 14 | 15 | # Ignore react and fbjs where there are overlaps, but don't ignore 16 | # anything that react-native relies on 17 | .*/node_modules/fbjs/lib/Map.js 18 | .*/node_modules/fbjs/lib/ErrorUtils.js 19 | 20 | # Flow has a built-in definition for the 'react' module which we prefer to use 21 | # over the currently-untyped source 22 | .*/node_modules/react/react.js 23 | .*/node_modules/react/lib/React.js 24 | .*/node_modules/react/lib/ReactDOM.js 25 | 26 | .*/__mocks__/.* 27 | .*/__tests__/.* 28 | 29 | .*/commoner/test/source/widget/share.js 30 | 31 | # Ignore commoner tests 32 | .*/node_modules/commoner/test/.* 33 | 34 | # See https://github.com/facebook/flow/issues/442 35 | .*/react-tools/node_modules/commoner/lib/reader.js 36 | 37 | # Ignore jest 38 | .*/node_modules/jest-cli/.* 39 | 40 | # Ignore Website 41 | .*/website/.* 42 | 43 | # Ignore generators 44 | .*/local-cli/generator.* 45 | 46 | # Ignore BUCK generated folders 47 | .*\.buckd/ 48 | 49 | # Ignore RNPM 50 | .*/local-cli/rnpm/.* 51 | 52 | .*/node_modules/is-my-json-valid/test/.*\.json 53 | .*/node_modules/iconv-lite/encodings/tables/.*\.json 54 | .*/node_modules/y18n/test/.*\.json 55 | .*/node_modules/spdx-license-ids/spdx-license-ids.json 56 | .*/node_modules/spdx-exceptions/index.json 57 | .*/node_modules/resolve/test/subdirs/node_modules/a/b/c/x.json 58 | .*/node_modules/resolve/lib/core.json 59 | .*/node_modules/jsonparse/samplejson/.*\.json 60 | .*/node_modules/json5/test/.*\.json 61 | .*/node_modules/ua-parser-js/test/.*\.json 62 | .*/node_modules/builtin-modules/builtin-modules.json 63 | .*/node_modules/binary-extensions/binary-extensions.json 64 | .*/node_modules/url-regex/tlds.json 65 | .*/node_modules/joi/.*\.json 66 | .*/node_modules/isemail/.*\.json 67 | .*/node_modules/tr46/.*\.json 68 | 69 | 70 | [include] 71 | 72 | [libs] 73 | node_modules/react-native/Libraries/react-native/react-native-interface.js 74 | node_modules/react-native/flow 75 | flow/ 76 | 77 | [options] 78 | module.system=haste 79 | 80 | esproposal.class_static_fields=enable 81 | esproposal.class_instance_fields=enable 82 | 83 | munge_underscores=true 84 | 85 | module.name_mapper='^image![a-zA-Z0-9$_-]+$' -> 'GlobalImageStub' 86 | module.name_mapper='^[./a-zA-Z0-9$_-]+\.\(bmp\|gif\|jpg\|jpeg\|png\|psd\|svg\|webp\|m4v\|mov\|mp4\|mpeg\|mpg\|webm\|aac\|aiff\|caf\|m4a\|mp3\|wav\|html\|pdf\)$' -> 'RelativeImageStub' 87 | 88 | suppress_type=$FlowIssue 89 | suppress_type=$FlowFixMe 90 | suppress_type=$FixMe 91 | 92 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixMe\\($\\|[^(]\\|(\\(>=0\\.\\(2[0-5]\\|1[0-9]\\|[0-9]\\).[0-9]\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\) 93 | suppress_comment=\\(.\\|\n\\)*\\$FlowIssue\\((\\(>=0\\.\\(2[0-5]\\|1[0-9]\\|[0-9]\\).[0-9]\\)? *\\(site=[a-z,_]*react_native[a-z,_]*\\)?)\\)?:? #[0-9]+ 94 | suppress_comment=\\(.\\|\n\\)*\\$FlowFixedInNextDeploy 95 | 96 | [version] 97 | ^0.25.0 98 | -------------------------------------------------------------------------------- /mobile_app/index.ios.js: -------------------------------------------------------------------------------- 1 | import React, { Component } from 'react' 2 | import ReactNative from 'react-native' 3 | 4 | import { 5 | AppRegistry, 6 | StyleSheet, 7 | CameraRoll, 8 | Image, 9 | Text, 10 | View, 11 | NativeModules, 12 | TextInput, 13 | Dimensions 14 | } from 'react-native' 15 | 16 | import Camera from 'react-native-camera' 17 | 18 | 19 | class ReactNativeTF extends Component { 20 | 21 | constructor(props) { 22 | super(props) 23 | 24 | this.state = { 25 | v1: 0., 26 | v2: 0., 27 | result: 0., 28 | camera: null, 29 | imagePath: '', 30 | image: null, 31 | } 32 | 33 | this.takePicture = this.takePicture.bind(this) 34 | this.resetPicture = this.resetPicture.bind(this) 35 | } 36 | 37 | 38 | takePicture() { 39 | this.state.camera.capture() 40 | .then((data) => { 41 | console.log("taking picture ... ", data.path) 42 | 43 | const IM = NativeModules.ImageManager 44 | IM.chill(data.path, (err, data) => { 45 | this.setState({ 46 | image: "data:image/png;base64," + data 47 | }) 48 | }) 49 | }) 50 | .catch(err => console.error(err)); 51 | } 52 | 53 | 54 | resetPicture() { 55 | this.setState({ 56 | image: null 57 | }); 58 | } 59 | 60 | 61 | render() { 62 | return ( 63 | 64 | 65 | [RESET] 66 | 67 | {this.state.image === null ? ( 68 | { 70 | this.state.camera = cam; 71 | }} 72 | style={styles.preview} 73 | aspect={Camera.constants.Aspect.fill} 74 | captureTarget={Camera.constants.CaptureTarget.tmp} 75 | captureQuality="high" 76 | > 77 | [CAPTURE] 78 | 79 | ) : ( 80 | 81 | 85 | 86 | )} 87 | 88 | ); 89 | } 90 | } 91 | 92 | const styles = StyleSheet.create({ 93 | container: { 94 | flex: 1, 95 | justifyContent: 'center', 96 | alignItems: 'center', 97 | }, 98 | preview: { 99 | flex: 1, 100 | justifyContent: 'flex-end', 101 | alignItems: 'center', 102 | height: Dimensions.get('window').height, 103 | width: Dimensions.get('window').width 104 | }, 105 | cropped: { 106 | alignSelf: 'center', 107 | width: 300, 108 | height: 300, 109 | borderColor: 'red', 110 | borderWidth: 3 111 | }, 112 | capture: { 113 | flex: 0, 114 | backgroundColor: '#fff', 115 | borderRadius: 5, 116 | color: '#000', 117 | padding: 10, 118 | margin: 40 119 | }, 120 | reset: { 121 | flex: 0, 122 | backgroundColor: '#fff', 123 | borderRadius: 5, 124 | color: '#000', 125 | padding: 10, 126 | margin: 20 127 | } 128 | }); 129 | 130 | AppRegistry.registerComponent('ReactNativeTF', () => ReactNativeTF); 131 | -------------------------------------------------------------------------------- /predict.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os, argparse, time 4 | 5 | from keras import backend as K 6 | 7 | from utils.imutils import load_images, save_image 8 | from utils.general import import_model 9 | from models.layers import custom_objects 10 | 11 | if K._BACKEND == "tensorflow": 12 | K.set_image_dim_ordering('tf') 13 | else: 14 | K.set_image_dim_ordering('th') 15 | 16 | dir = os.path.dirname(os.path.realpath(__file__)) 17 | 18 | parser = argparse.ArgumentParser( 19 | description='Neural artistic style. Generates an image by combining ' 20 | 'the content of an image and the style of another.', 21 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 22 | ) 23 | parser.add_argument('--models_dir', default=dir + '/models/data/st', type=str, help='Models top directories.') 24 | parser.add_argument('--batch_size', default=20, type=int, help='batch size.') 25 | parser.add_argument('--image_size', default=600, type=int, help='Input image size.') 26 | args = parser.parse_args() 27 | 28 | dataDir = dir + '/data' 29 | output_dir = dataDir + '/output' 30 | overfit_dir = dataDir + '/overfit_' + str(args.image_size) 31 | test_dir = dataDir + '/test' 32 | 33 | dim_ordering = K.image_dim_ordering() 34 | channels = 3 35 | width = args.image_size 36 | height = args.image_size 37 | if dim_ordering == 'th': 38 | input_shape = (channels, width, height) 39 | else: 40 | input_shape = (width, height, channels) 41 | 42 | X_overfit = load_images(overfit_dir, limit=args.batch_size, size=(height, width), preprocess_type='st', verbose=True) 43 | X_test = load_images(test_dir, limit=args.batch_size, size=(height, width), preprocess_type='st', verbose=True) 44 | print('X_test.shape: ' + str(X_test.shape)) 45 | print('X_overfit.shape: ' + str(X_overfit.shape)) 46 | 47 | current_iter = 0 48 | subdirs = [x[0] for x in os.walk(args.models_dir)] 49 | subdirs.pop(0) # First element is the parent dir 50 | for absolute_model_dir in subdirs: 51 | print('Loading model in %s' % absolute_model_dir) 52 | st_model = import_model(absolute_model_dir, best=True, 53 | should_convert=False, custom_objects=custom_objects) 54 | 55 | print('Predicting') 56 | start = time.clock() 57 | results = st_model.predict(X_test) # Equivalent to predict([X_test, False]) 58 | results_overfit = st_model.predict(X_overfit) # Equivalent to predict([X_test, False]) 59 | end = time.clock() 60 | duration_batch = (end-start)/(X_test.shape[0] * X_overfit.shape[0]) 61 | 62 | print('Dumping results') 63 | tmp_output_dir = output_dir + '/' + absolute_model_dir.split('/')[-2] + '/' + absolute_model_dir.split('/')[-1] 64 | if not os.path.isdir(tmp_output_dir): 65 | os.makedirs(tmp_output_dir) 66 | for idx, im in enumerate(results): 67 | prefix = str(current_iter).zfill(4) 68 | fullOutPath = tmp_output_dir + '/' + prefix + "_" + str(idx) + ".png" 69 | save_image(fullOutPath, im, deprocess_type='st') 70 | fullOriPath = tmp_output_dir + '/' + prefix + "_" + str(idx) + "_ori.png" 71 | save_image(fullOriPath, X_test[idx], deprocess_type='st') 72 | 73 | current_iter += 1 74 | 75 | for idx, im in enumerate(results_overfit): 76 | prefix = str(current_iter).zfill(4) 77 | fullOutPath = tmp_output_dir + '/' + prefix + str(idx) + "_overfit.png" 78 | save_image(fullOutPath, im, deprocess_type='st') 79 | fullOriPath = tmp_output_dir + '/' + prefix + str(idx) + "_overfit_ori.png" 80 | save_image(fullOriPath, X_overfit[idx], deprocess_type='st') 81 | 82 | current_iter += 1 83 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/Base.lproj/LaunchScreen.xib: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 21 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/UIImage+RoundedCorner.m: -------------------------------------------------------------------------------- 1 | // UIImage+RoundedCorner.m 2 | // Created by Trevor Harmon on 9/20/09. 3 | // Free for personal or commercial use, with or without modification. 4 | // No warranty is expressed or implied. 5 | 6 | #import "UIImage+RoundedCorner.h" 7 | #import "UIImage+Alpha.h" 8 | 9 | @implementation UIImage (RoundedCorner) 10 | 11 | // Creates a copy of this image with rounded corners 12 | // If borderSize is non-zero, a transparent border of the given size will also be added 13 | // Original author: Björn Sållarp. Used with permission. See: http://blog.sallarp.com/iphone-uiimage-round-corners/ 14 | - (UIImage *)roundedCornerImage:(NSInteger)cornerSize borderSize:(NSInteger)borderSize { 15 | // If the image does not have an alpha layer, add one 16 | UIImage *image = [self imageWithAlpha]; 17 | 18 | CGFloat scale = MAX(self.scale,1.0f); 19 | NSUInteger scaledBorderSize = borderSize * scale; 20 | 21 | // Build a context that's the same dimensions as the new size 22 | CGContextRef context = CGBitmapContextCreate(NULL, 23 | image.size.width*scale, 24 | image.size.height*scale, 25 | CGImageGetBitsPerComponent(image.CGImage), 26 | 0, 27 | CGImageGetColorSpace(image.CGImage), 28 | CGImageGetBitmapInfo(image.CGImage)); 29 | 30 | // Create a clipping path with rounded corners 31 | 32 | CGContextBeginPath(context); 33 | [self addRoundedRectToPath:CGRectMake(scaledBorderSize, scaledBorderSize, image.size.width*scale - borderSize * 2, image.size.height*scale - borderSize * 2) 34 | context:context 35 | ovalWidth:cornerSize*scale 36 | ovalHeight:cornerSize*scale]; 37 | CGContextClosePath(context); 38 | CGContextClip(context); 39 | 40 | // Draw the image to the context; the clipping path will make anything outside the rounded rect transparent 41 | CGContextDrawImage(context, CGRectMake(0, 0, image.size.width*scale, image.size.height*scale), image.CGImage); 42 | 43 | // Create a CGImage from the context 44 | CGImageRef clippedImage = CGBitmapContextCreateImage(context); 45 | CGContextRelease(context); 46 | 47 | // Create a UIImage from the CGImage 48 | UIImage *roundedImage = [UIImage imageWithCGImage:clippedImage scale:self.scale orientation:UIImageOrientationUp]; 49 | 50 | CGImageRelease(clippedImage); 51 | 52 | return roundedImage; 53 | } 54 | 55 | #pragma mark - 56 | #pragma mark Private helper methods 57 | 58 | // Adds a rectangular path to the given context and rounds its corners by the given extents 59 | // Original author: Björn Sållarp. Used with permission. See: http://blog.sallarp.com/iphone-uiimage-round-corners/ 60 | - (void)addRoundedRectToPath:(CGRect)rect context:(CGContextRef)context ovalWidth:(CGFloat)ovalWidth ovalHeight:(CGFloat)ovalHeight { 61 | if (ovalWidth == 0 || ovalHeight == 0) { 62 | CGContextAddRect(context, rect); 63 | return; 64 | } 65 | CGContextSaveGState(context); 66 | CGContextTranslateCTM(context, CGRectGetMinX(rect), CGRectGetMinY(rect)); 67 | CGContextScaleCTM(context, ovalWidth, ovalHeight); 68 | CGFloat fw = CGRectGetWidth(rect) / ovalWidth; 69 | CGFloat fh = CGRectGetHeight(rect) / ovalHeight; 70 | CGContextMoveToPoint(context, fw, fh/2); 71 | CGContextAddArcToPoint(context, fw, fh, fw/2, fh, 1); 72 | CGContextAddArcToPoint(context, 0, fh, 0, fh/2, 1); 73 | CGContextAddArcToPoint(context, 0, 0, fw/2, 0, 1); 74 | CGContextAddArcToPoint(context, fw, 0, fw, fh/2, 1); 75 | CGContextClosePath(context); 76 | CGContextRestoreGState(context); 77 | } 78 | 79 | @end 80 | -------------------------------------------------------------------------------- /data/prepare_data.py: -------------------------------------------------------------------------------- 1 | import os, argparse, sys 2 | 3 | dir = os.path.dirname(os.path.realpath(__file__)) 4 | sys.path.append(dir + '/..') 5 | 6 | import numpy as np 7 | import h5py 8 | from keras import backend as K 9 | 10 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 11 | 12 | from utils.imutils import (load_image, load_images, create_noise_tensor, 13 | deprocess, save_image, plot_losses, get_image_list) 14 | from utils.lossutils import (frobenius_error, total_variation_error, 15 | grams, norm_l2, train_input 16 | ) 17 | 18 | dataDir = os.path.dirname(os.path.realpath(__file__)) 19 | vgg19Dir = dataDir + '/../vgg19' 20 | 21 | parser = argparse.ArgumentParser( 22 | description='Neural artistic style. Generates an image by combining ' 23 | 'the content of an image and the style of another.', 24 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 25 | ) 26 | parser.add_argument('--style_dir', default=dataDir + '/paintings', type=str, help='Style folder') 27 | parser.add_argument('--content_dir', default='', type=str, help='Content folder') 28 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 29 | parser.add_argument('--image_size', default=600, type=int, help='Input image size.') 30 | args = parser.parse_args() 31 | 32 | if args.content_dir != '': 33 | results_content_dir = args.content_dir + '/results' 34 | if not os.path.isdir(results_content_dir): 35 | os.makedirs(results_content_dir) 36 | if args.style_dir != '': 37 | results_style_dir = args.style_dir + '/results' 38 | if not os.path.isdir(results_style_dir): 39 | os.makedirs(results_style_dir) 40 | 41 | dim_ordering = K.image_dim_ordering() 42 | channels = 3 43 | width = args.image_size 44 | height = args.image_size 45 | if dim_ordering == 'th': 46 | input_shape = (channels, width, height) 47 | else: 48 | input_shape = (width, height, channels) 49 | 50 | print('Loading VGG model') 51 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 52 | vgg_model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 53 | layer_dict, layers_names = get_layer_data(vgg_model, 'conv_') 54 | 55 | print('Compiling predict functions') 56 | style_layers = ['conv_1_2', 'conv_2_2', 'conv_3_4', 'conv_4_2'] 57 | style_output_layers = [grams(layer_dict[ls_name].output) for ls_name in style_layers] 58 | predict_style = K.function([vgg_model.input], style_output_layers) 59 | 60 | content_layers = ['conv_3_2'] 61 | content_output_layers = [layer_dict[lc_name].output for lc_name in content_layers] 62 | predict_content = K.function([vgg_model.input], content_output_layers) 63 | 64 | if 'results_style_dir' in locals(): 65 | image_list = get_image_list(args.style_dir) 66 | for image_path in image_list: 67 | X_train_style = np.array([load_image(image_path, size=(height,width), preprocess_type='vgg19', verbose=True)]) 68 | results = predict_style([X_train_style]) 69 | 70 | filename = image_path.split('/')[-1].split('.')[0] 71 | output_filename = results_style_dir + '/' + filename + '_' + str(args.image_size) + '.hdf5' 72 | with h5py.File(output_filename, 'w') as hf: 73 | for idx, style_layer in enumerate(style_layers): 74 | hf.create_dataset(style_layer, data=results[idx][0]) 75 | 76 | if 'results_content_dir' in locals(): 77 | print('be carefull, every file dumped is taking 22mb, check you have enough space') 78 | image_list = get_image_list(args.content_dir) 79 | for image_path in image_list: 80 | X_train_content = np.array([load_image(image_path, size=(height, width), preprocess_type='vgg19', verbose=True)]) 81 | results = predict_content([X_train_content]) 82 | 83 | filename = image_path.split('/')[-1].split('.')[0] 84 | output_filename = results_content_dir + '/' + filename + '_' + str(args.image_size) + '.hdf5' 85 | with h5py.File(output_filename, 'w') as hf: 86 | for idx, content_layer in enumerate(content_layers): 87 | hf.create_dataset(content_layer, data=results[idx][0]) 88 | 89 | -------------------------------------------------------------------------------- /ltv.py: -------------------------------------------------------------------------------- 1 | import os, argparse 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | 6 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 7 | 8 | from utils.imutils import (load_image, create_noise_tensor, 9 | deprocess, save_image, plot_losses) 10 | from utils.lossutils import (frobenius_error, grams, 11 | norm_l2, train_input, total_variation_error) 12 | 13 | if K._BACKEND == "tensorflow": 14 | K.set_image_dim_ordering('tf') 15 | else: 16 | K.set_image_dim_ordering('th') 17 | 18 | optimizer = 'adam' 19 | if optimizer == 'lbfgs': 20 | K.set_floatx('float64') # scipy needs float64 to use lbfgs 21 | 22 | dir = os.path.dirname(os.path.realpath(__file__)) 23 | vgg19Dir = dir + '/vgg19' 24 | dataDir = dir + '/data' 25 | resultsDir = dataDir + '/output/vgg19/ltv' 26 | if not os.path.isdir(resultsDir): 27 | os.makedirs(resultsDir) 28 | 29 | parser = argparse.ArgumentParser( 30 | description='Neural artistic style. Generates an image by combining ' 31 | 'the content of an image and the style of another.', 32 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 33 | ) 34 | parser.add_argument('--content', default=dataDir + '/overfit/COCO_val2014_000000000074.jpg', type=str, help='Content image.') 35 | parser.add_argument('--style', default=dataDir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 36 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 37 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 38 | parser.add_argument('--max_iter', default=600, type=int, help='Number of training iter.') 39 | args = parser.parse_args() 40 | 41 | dim_ordering = K.image_dim_ordering() 42 | channels = 3 43 | width = args.image_size 44 | height = args.image_size 45 | size = (height, width) 46 | if dim_ordering == 'th': 47 | input_shape = (channels, width, height) 48 | else: 49 | input_shape = (width, height, channels) 50 | 51 | X_train = np.array([load_image(args.content, size=(height, width), preprocess_type='vgg19', verbose=True)]) 52 | print("X_train shape:", X_train.shape) 53 | 54 | X_train_style = np.array([load_image(args.style, size=(height, width), preprocess_type='vgg19', verbose=True)]) 55 | print("X_train_style shape:", X_train_style.shape) 56 | 57 | print('Loading VGG headless 5') 58 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 59 | model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 60 | layer_dict, layers_names = get_layer_data(model, 'conv_') 61 | print('Layers found:' + ', '.join(layers_names)) 62 | 63 | input_layer = model.input 64 | 65 | layer_name = layers_names[3] 66 | print('Creating labels for ' + layer_name) 67 | out = layer_dict[layer_name].output 68 | predict = K.function([input_layer], [out]) 69 | y_style = predict([X_train_style])[0] 70 | 71 | print('Building white noise images') 72 | input_data = create_noise_tensor(height, width, channels) 73 | 74 | print('Using optimizer: ' + optimizer) 75 | current_iter = 1 76 | for gamma in [1e-7, 3e-7, 6e-7, 77 | 1e-6, 3e-6, 6e-6, 78 | 1e-5, 3e-5, 6e-5, 79 | 1e-4, 3e-4, 6e-4]: 80 | print('gamma: %.7f' % gamma) 81 | prefix = str(current_iter).zfill(4) 82 | 83 | print('Compiling gamma') 84 | loss_style = frobenius_error(grams(y_style), grams(out)) 85 | reg_TV = total_variation_error(input_layer, 2) 86 | rtv = gamma * reg_TV 87 | total_loss = loss_style + rtv 88 | grads_style = K.gradients(total_loss, input_layer) 89 | if optimizer == 'adam': 90 | grads_style = norm_l2(grads_style) 91 | iterate_style = K.function([input_layer], [total_loss, grads_style, loss_style, rtv]) 92 | 93 | print('Training the image for style') 94 | config = {'learning_rate': 5e-1} 95 | best_input_style_data, style_losses = train_input( 96 | input_data, 97 | iterate_style, 98 | optimizer, 99 | config, 100 | max_iter=args.max_iter, 101 | ) 102 | fullOutPath = resultsDir + '/' + prefix + '_gamma' + str(gamma) + ".png" 103 | save_image(fullOutPath, best_input_style_data[0], deprocess_type='vgg19') 104 | plot_losses(style_losses, resultsDir, prefix) 105 | 106 | current_iter += 1 107 | -------------------------------------------------------------------------------- /tests/utils/test_lossutils.py: -------------------------------------------------------------------------------- 1 | import os, sys, unittest 2 | 3 | dir = os.path.dirname(os.path.realpath(__file__)) 4 | sys.path.append(dir + '/../..') 5 | 6 | from keras import backend as K 7 | from scipy import misc 8 | 9 | from utils.lossutils import * 10 | 11 | 12 | dir = os.path.dirname(os.path.realpath(__file__)) 13 | 14 | class TestLossUtils(unittest.TestCase): 15 | 16 | def test_total_variation_error(self): 17 | # Prepare input 18 | input = np.zeros((1, 3, 4, 4)) 19 | iter = 0 20 | for i in range(input.shape[1]): 21 | for j in range(input.shape[2]): 22 | for k in range(input.shape[3]): 23 | input[0][i][j][k] = iter 24 | iter += 1 25 | input = input.astype('float32') 26 | 27 | x = K.placeholder(input.shape, name='x') 28 | loss = total_variation_error(x, 2) 29 | grad = K.gradients(loss, x) 30 | get_grads = K.function([x], grad) 31 | 32 | # GradInput result for beta = 2 33 | true_grad = np.array([[ 34 | [ 35 | [-5, -4, -4, 1], 36 | [-1, 0, 0, 1], 37 | [-1, 0, 0, 1], 38 | [4, 4, 4, 0] 39 | ], 40 | [ 41 | [-5, -4, -4, 1], 42 | [-1, 0, 0, 1], 43 | [-1, 0, 0, 1], 44 | [4, 4, 4, 0] 45 | ], 46 | [ 47 | [-5, -4, -4, 1], 48 | [-1, 0, 0, 1], 49 | [-1, 0, 0, 1], 50 | [4, 4, 4, 0] 51 | ], 52 | ]]).astype(K.floatx()) 53 | 54 | self.assertEqual(True, (get_grads([input])==true_grad).all()) 55 | 56 | def test_grams_th(self): 57 | previous_image_dim_ordering = K.image_dim_ordering() 58 | K.set_image_dim_ordering('th') 59 | 60 | input = np.zeros((1, 3, 4, 4)) 61 | iter = 0 62 | for i in range(input.shape[1]): 63 | for j in range(input.shape[2]): 64 | for k in range(input.shape[3]): 65 | input[0][i][j][k] = iter 66 | iter += 1 67 | input = input.astype(K.floatx()) 68 | 69 | true_grams = np.array([[ 70 | [1240, 3160, 5080], 71 | [3160, 9176,15192], 72 | [5080, 15192, 25304] 73 | ]]).astype(K.floatx()) 74 | true_grams /= input.shape[1] * input.shape[2] * input.shape[3] 75 | 76 | x = K.placeholder(input.shape, name='x') 77 | gram_mat = grams(x) 78 | get_grams = K.function([x], [gram_mat]) 79 | K.set_image_dim_ordering(previous_image_dim_ordering) 80 | 81 | pred_grams = get_grams([input])[0] 82 | self.assertEqual(True, (pred_grams==true_grams).all()) 83 | 84 | def test_grams_tf(self): 85 | previous_image_dim_ordering = K.image_dim_ordering() 86 | K.set_image_dim_ordering('tf') 87 | 88 | input = np.zeros((1, 3, 4, 4)) 89 | iter = 0 90 | for i in range(input.shape[1]): 91 | for j in range(input.shape[2]): 92 | for k in range(input.shape[3]): 93 | input[0][i][j][k] = iter 94 | iter += 1 95 | input = input.astype(K.floatx()) 96 | input = np.transpose(input, (0, 2, 3, 1)) 97 | 98 | true_grams = np.array([[ 99 | [1240, 3160, 5080], 100 | [3160, 9176,15192], 101 | [5080, 15192, 25304] 102 | ]]).astype(K.floatx()) 103 | true_grams /= input.shape[1] * input.shape[2] * input.shape[3] 104 | 105 | x = K.placeholder(input.shape, name='x') 106 | gram_mat = grams(x) 107 | get_grams = K.function([x], [gram_mat]) 108 | K.set_image_dim_ordering(previous_image_dim_ordering) 109 | 110 | pred_grams = get_grams([input])[0] 111 | self.assertEqual(True, (pred_grams==true_grams).all()) 112 | 113 | def test_grams_loss(self): 114 | input = np.zeros((1, 3, 4, 4)) 115 | iter = 0 116 | for i in range(input.shape[1]): 117 | for j in range(input.shape[2]): 118 | for k in range(input.shape[3]): 119 | input[0][i][j][k] = iter 120 | iter += 1 121 | input = input.astype(K.floatx()) 122 | 123 | 124 | x = K.placeholder(input.shape, name='x') 125 | gram_mat = grams(x) 126 | loss = frobenius_error(gram_mat, np.ones((1, 3, 3))) 127 | get_loss = K.function([x], [loss]) 128 | 129 | error = get_loss([input])[0] 130 | true_error = 60344.299382716 131 | 132 | self.assertEqual(np.round(error.item(0)), np.round(true_error)) 133 | 134 | if __name__ == '__main__': 135 | unittest.main() -------------------------------------------------------------------------------- /alpha.py: -------------------------------------------------------------------------------- 1 | import os, argparse 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | 6 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 7 | 8 | from utils.imutils import (load_image, create_noise_tensor, 9 | save_image, plot_losses) 10 | from utils.lossutils import (frobenius_error, grams, norm_l2, train_input) 11 | 12 | if K._BACKEND == "tensorflow": 13 | K.set_image_dim_ordering('tf') 14 | else: 15 | K.set_image_dim_ordering('th') 16 | 17 | optimizer = 'adam' 18 | if optimizer == 'lbfgs': 19 | K.set_floatx('float64') # scipy needs float64 to use lbfgs 20 | 21 | dir = os.path.dirname(os.path.realpath(__file__)) 22 | vgg19Dir = dir + '/vgg19' 23 | dataDir = dir + '/data' 24 | resultsDir = dataDir + '/output/vgg19/alpha' 25 | if not os.path.isdir(resultsDir): 26 | os.makedirs(resultsDir) 27 | 28 | parser = argparse.ArgumentParser( 29 | description='Neural artistic style. Generates an image by combining ' 30 | 'the content of an image and the style of another.', 31 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 32 | ) 33 | parser.add_argument('--content', default=dataDir + '/overfit/COCO_val2014_000000000074.jpg', type=str, help='Content image.') 34 | parser.add_argument('--style', default=dataDir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 35 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 36 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 37 | parser.add_argument('--max_iter', default=600, type=int, help='Number of training iter.') 38 | args = parser.parse_args() 39 | 40 | dim_ordering = K.image_dim_ordering() 41 | channels = 3 42 | width = args.image_size 43 | height = args.image_size 44 | size = (height, width) 45 | if dim_ordering == 'th': 46 | input_shape = (channels, width, height) 47 | else: 48 | input_shape = (width, height, channels) 49 | 50 | X_train = np.array([load_image(args.content, size=(height, width), preprocess_type='vgg19', verbose=True)]) 51 | print("X_train shape:", X_train.shape) 52 | 53 | X_train_style = np.array([load_image(args.style, size=(height, width), preprocess_type='vgg19', verbose=True)]) 54 | print("X_train_style shape:", X_train_style.shape) 55 | 56 | print('Loading VGG headless 5') 57 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 58 | model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 59 | layer_dict, layers_names = get_layer_data(model, 'conv_') 60 | print('Layers found:' + ', '.join(layers_names)) 61 | 62 | input_layer = model.input 63 | 64 | print('Building white noise images') 65 | input_data = create_noise_tensor(height, width, channels) 66 | 67 | print('Using optimizer: ' + optimizer) 68 | current_iter = 1 69 | ls_name = layers_names[3] 70 | lc_name = layers_names[3] 71 | for alpha in [1e0, 3e0, 6e0, 1e1, 3e1, 6e1, 1e2, 3e2, 6e2, 1e3, 3e3, 6e3]: 72 | print('Creating labels for content ' + lc_name + ' and style ' + ls_name) 73 | out_style = layer_dict[ls_name].output 74 | predict_style = K.function([input_layer], [out_style]) 75 | y_style = predict_style([X_train_style])[0] 76 | 77 | out_content = layer_dict[lc_name].output 78 | predict_content = K.function([input_layer], [out_content]) 79 | y_content = predict_content([X_train])[0] 80 | 81 | loss_style = frobenius_error(grams(y_style), grams(out_style)) 82 | loss_content = frobenius_error(y_content, out_content) 83 | 84 | lc_weight = 1. 85 | ls_weight = 1. 86 | print("lc_weight: %f, ls_weight: %f" % (lc_weight, ls_weight)) 87 | 88 | print('Compiling VGG headless 5 for content ' + lc_name + ' and style ' + ls_name) 89 | ls = alpha * loss_style / lc_weight 90 | lc = loss_content / ls_weight 91 | loss = ls + lc 92 | grads = K.gradients(loss, input_layer) 93 | if optimizer == 'adam': 94 | grads = norm_l2(grads) 95 | iterate = K.function([input_layer], [loss, grads, lc, ls]) 96 | 97 | print('Training the image') 98 | config = {'learning_rate': 5e-1} 99 | best_input_data, losses = train_input( 100 | input_data, 101 | iterate, 102 | optimizer, 103 | config, 104 | max_iter=args.max_iter 105 | ) 106 | prefix = str(current_iter).zfill(4) 107 | suffix = "_lc_weight%f_ls_weight%f" % (lc_weight, ls_weight) 108 | fullOutPath = resultsDir + '/' + prefix + '_style' + ls_name + '_content' + lc_name + suffix + ".png" 109 | # dump_as_hdf5(resultsDir + '/' + prefix + '_style' + ls_name + '_content' + lc_name + suffix + ".hdf5", best_input_data[0]) 110 | save_image(fullOutPath, best_input_data[0], deprocess_type='vgg19') 111 | plot_losses(losses, resultsDir, prefix, suffix) 112 | 113 | current_iter += 1 114 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/xcshareddata/xcschemes/ReactNativeTF.xcscheme: -------------------------------------------------------------------------------- 1 | 2 | 5 | 8 | 9 | 15 | 21 | 22 | 23 | 29 | 35 | 36 | 37 | 38 | 39 | 44 | 45 | 47 | 53 | 54 | 55 | 56 | 57 | 63 | 64 | 65 | 66 | 67 | 68 | 78 | 80 | 86 | 87 | 88 | 89 | 90 | 91 | 97 | 99 | 105 | 106 | 107 | 108 | 110 | 111 | 114 | 115 | 116 | -------------------------------------------------------------------------------- /tests/test_convolution_transpose_2d.py: -------------------------------------------------------------------------------- 1 | import os, sys, unittest 2 | import numpy as np 3 | 4 | dir = os.path.dirname(os.path.realpath(__file__)) 5 | sys.path.append(dir + '/..') 6 | 7 | from keras import backend as K 8 | if K._BACKEND == 'theano': 9 | import theano 10 | import theano.tensor as T 11 | else: 12 | import tensorflow as tf 13 | 14 | from keras.layers import Input 15 | from keras.models import Model 16 | 17 | from models.layers.ConvolutionTranspose2D import ConvolutionTranspose2D 18 | 19 | class TestImUtils(unittest.TestCase): 20 | 21 | def test_convolution_transpose_th(self): 22 | border_mode = 'valid' 23 | 24 | K.set_image_dim_ordering('th') 25 | batch = 1 26 | height = 2 27 | width = 2 28 | channels_in = 1 29 | channels_out = 2 30 | kernel_size = 3 31 | strides = (1, 1) 32 | input_shape = (channels_in, height, width) 33 | 34 | input = Input(shape=input_shape, dtype=K.floatx()) 35 | conv_layer = ConvolutionTranspose2D(channels_out, kernel_size, kernel_size, 36 | dim_ordering=K.image_dim_ordering(), init='one', 37 | subsample=strides, border_mode=border_mode, activation='linear') 38 | output = conv_layer(input) 39 | model = Model(input=[input], output=[output]) 40 | model.compile(loss='mean_squared_error', optimizer='sgd') 41 | 42 | x = np.ones((batch,) + input_shape).astype(K.floatx()) 43 | kernel = conv_layer.W 44 | output_model = model.predict(x) 45 | if K._BACKEND == 'theano': 46 | output_shape = conv_layer.get_output_shape_for(K.shape(x)) 47 | y = T.nnet.abstract_conv.conv2d_grad_wrt_inputs(theano.shared(x), kernel, output_shape, 48 | filter_shape=None, border_mode=border_mode, subsample=strides, filter_flip=True) 49 | output = y.eval() 50 | else: 51 | sess = K.get_session() 52 | output_shape = conv_layer.get_output_shape_for(K.shape(x)) 53 | output_shape = tf.pack([1, output_shape[2], output_shape[3], output_shape[1]]) 54 | x = tf.transpose(x, (0, 2, 3, 1)) 55 | kernel = tf.transpose(kernel, (2, 3, 1, 0)) 56 | y = tf.nn.conv2d_transpose(x, kernel, output_shape, (1, ) + strides + (1, ), padding=border_mode.upper()) 57 | y = tf.transpose(y, (0, 3, 1, 2)) 58 | output = sess.run(y) 59 | 60 | self.assertEqual(output_model.shape, (1, 2, 4, 4)) 61 | self.assertEqual(output.shape, (1, 2, 4, 4)) 62 | self.assertEqual(True, (output==output_model).all()) 63 | 64 | # model.fit(x, x + 1, nb_epoch=1) 65 | 66 | def test_convolution_transpose_tf(self): 67 | border_mode = 'valid' 68 | 69 | K.set_image_dim_ordering('tf') 70 | batch = 1 71 | height = 2 72 | width = 2 73 | channels_in = 1 74 | channels_out = 2 75 | kernel_size = 3 76 | strides = (1, 1) 77 | input_shape = (height, width, channels_in) 78 | 79 | input = Input(shape=input_shape, dtype=K.floatx()) 80 | conv_layer = ConvolutionTranspose2D(channels_out, kernel_size, kernel_size, 81 | dim_ordering=K.image_dim_ordering(), init='one', 82 | subsample=strides, border_mode=border_mode, activation='linear') 83 | output = conv_layer(input) 84 | model = Model(input=[input], output=[output]) 85 | model.compile(loss='mean_squared_error', optimizer='sgd') 86 | 87 | x = np.ones((batch,) + input_shape).astype(K.floatx()) 88 | kernel = conv_layer.W 89 | output_model = model.predict(x) 90 | if K._BACKEND == 'theano': 91 | output_shape = conv_layer.get_output_shape_for(K.shape(x)) 92 | output_shape = (1, output_shape[3], output_shape[1], output_shape[2]) 93 | x = np.transpose(x, (0, 3, 1, 2)) 94 | kernel = T.transpose(kernel, (3, 2, 1, 0)) 95 | y = T.nnet.abstract_conv.conv2d_grad_wrt_inputs(theano.shared(x), kernel, output_shape, 96 | filter_shape=None, border_mode=border_mode, subsample=strides, filter_flip=True) 97 | y = T.transpose(y, (0, 2, 3, 1)) 98 | output = y.eval() 99 | else: 100 | sess = K.get_session() 101 | output_shape = conv_layer.get_output_shape_for(K.shape(x)) 102 | output_shape = tf.pack([1, output_shape[1], output_shape[2], output_shape[3]]) 103 | y = tf.nn.conv2d_transpose(x, kernel, output_shape, (1, ) + strides + (1, ), padding=border_mode.upper()) 104 | 105 | output = sess.run(y) 106 | 107 | self.assertEqual(output_model.shape, (1, 4, 4, 2)) 108 | self.assertEqual(output.shape, (1, 4, 4, 2)) 109 | self.assertEqual(True, (output==output_model).all()) 110 | 111 | # model.fit(x, x + 1, nb_epoch=1) 112 | 113 | if __name__ == "__main__": 114 | unittest.main() -------------------------------------------------------------------------------- /layer_influence.py: -------------------------------------------------------------------------------- 1 | import os, argparse, json 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | 6 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 7 | 8 | from utils.imutils import (load_image, create_noise_tensor, 9 | dump_as_hdf5, deprocess, save_image, 10 | plot_losses) 11 | from utils.lossutils import (frobenius_error, grams, norm_l2, train_input) 12 | 13 | if K._BACKEND == "tensorflow": 14 | K.set_image_dim_ordering('tf') 15 | else: 16 | K.set_image_dim_ordering('th') 17 | 18 | optimizer = 'adam' 19 | if optimizer == 'lbfgs': 20 | K.set_floatx('float64') # scipy needs float64 to use lbfgs 21 | 22 | dir = os.path.dirname(os.path.realpath(__file__)) 23 | vgg19Dir = dir + '/vgg19' 24 | dataDir = dir + '/data' 25 | resultsDir = dataDir + '/output/vgg19/influence' 26 | if not os.path.isdir(resultsDir): 27 | os.makedirs(resultsDir) 28 | 29 | parser = argparse.ArgumentParser( 30 | description='Neural artistic style. Generates an image by combining ' 31 | 'the content of an image and the style of another.', 32 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 33 | ) 34 | parser.add_argument('--content', default=dataDir + '/overfit/COCO_val2014_000000000074.jpg', type=str, help='Content image.') 35 | parser.add_argument('--style', default=dataDir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 36 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 37 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 38 | parser.add_argument('--max_iter', default=600, type=int, help='Number of training iter.') 39 | args = parser.parse_args() 40 | 41 | dim_ordering = K.image_dim_ordering() 42 | channels = 3 43 | width = args.image_size 44 | height = args.image_size 45 | size = (height, width) 46 | if dim_ordering == 'th': 47 | input_shape = (channels, width, height) 48 | else: 49 | input_shape = (width, height, channels) 50 | 51 | X_train = np.array([load_image(args.content, size=(height, width), preprocess_type='vgg19', verbose=True)]) 52 | print("X_train shape:", X_train.shape) 53 | 54 | X_train_style = np.array([load_image(args.style, size=(height, width), preprocess_type='vgg19', verbose=True)]) 55 | print("X_train_style shape:", X_train_style.shape) 56 | 57 | print('Loading VGG headless 5') 58 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 59 | model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 60 | layer_dict, layers_names = get_layer_data(model, 'conv_(1|2|3|4)') # remove conv_5_* layers 61 | print('Layers found:' + ', '.join(layers_names)) 62 | 63 | input_layer = model.input 64 | 65 | print('Building white noise images') 66 | input_data = create_noise_tensor(height, width, channels) 67 | 68 | print('Using optimizer: ' + optimizer) 69 | current_iter = 1 70 | alpha = 5e1 # 1e2 71 | for idx_content, lc_name in enumerate(layers_names): 72 | for idx_style, ls_name in enumerate(layers_names): 73 | print('Creating labels for content ' + lc_name + ' and style ' + ls_name) 74 | out_style = layer_dict[ls_name].output 75 | predict_style = K.function([input_layer], [out_style]) 76 | y_style = predict_style([X_train_style])[0] 77 | 78 | out_content = layer_dict[lc_name].output 79 | predict_content = K.function([input_layer], [out_content]) 80 | y_content = predict_content([X_train])[0] 81 | 82 | loss_style = frobenius_error(grams(y_style), grams(out_style)) 83 | loss_content = frobenius_error(y_content, out_content) 84 | 85 | lc_weight = 1. 86 | ls_weight = 1. 87 | print("lc_weight: %f, ls_weight: %f" % (lc_weight, ls_weight)) 88 | 89 | print('Compiling VGG headless 5 for content ' + lc_name + ' and style ' + ls_name) 90 | ls = alpha * loss_style / lc_weight 91 | lc = loss_content / ls_weight 92 | loss = ls + lc 93 | grads = K.gradients(loss, input_layer) 94 | if optimizer == 'adam': 95 | grads = norm_l2(grads) 96 | iterate = K.function([input_layer], [loss, grads, lc, ls]) 97 | 98 | print('Training the image') 99 | config = {'learning_rate': 5e-1} 100 | best_input_data, losses = train_input( 101 | input_data, 102 | iterate, 103 | optimizer, 104 | config, 105 | max_iter=args.max_iter 106 | ) 107 | prefix = str(current_iter).zfill(4) 108 | suffix = "_lc_weight%f_ls_weight%f" % (lc_weight, ls_weight) 109 | fullOutPath = resultsDir + '/' + prefix + '_style' + ls_name + '_content' + lc_name + suffix + ".png" 110 | # dump_as_hdf5(resultsDir + '/' + prefix + '_style' + ls_name + '_content' + lc_name + suffix + ".hdf5", best_input_data[0]) 111 | save_image(fullOutPath, best_input_data[0], deprocess_type='vgg19') 112 | plot_losses(losses, resultsDir, prefix, suffix) 113 | 114 | current_iter += 1 115 | -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/ImageManager.m: -------------------------------------------------------------------------------- 1 | // 2 | // ImageManager.m 3 | // ReactNativeTorch 4 | // 5 | // Created by ThomasOlivier on 08/06/2016. 6 | // Copyright © 2016 Facebook. All rights reserved. 7 | // 8 | 9 | #import 10 | #import "RCTLog.h" 11 | #import "ImageManager.h" 12 | 13 | @import AssetsLibrary; 14 | @import UIKit; 15 | 16 | #import "UIImage+Resize.h" 17 | #import "RunModel.h" 18 | 19 | 20 | @implementation ImageManager 21 | 22 | 23 | RCT_EXPORT_MODULE(); 24 | 25 | 26 | + (CGFloat *)getRGBAsFromImage:(UIImage*)image; 27 | { 28 | int count = image.size.width * image.size.height; 29 | 30 | // First get the image into your data buffer 31 | CGImageRef imageRef = [image CGImage]; 32 | NSUInteger width = CGImageGetWidth(imageRef); 33 | NSUInteger height = CGImageGetHeight(imageRef); 34 | 35 | CGColorSpaceRef rgb = CGColorSpaceCreateDeviceRGB(); 36 | unsigned char * pixels = (unsigned char *) calloc(height * width * 4, sizeof(unsigned char)); 37 | 38 | NSUInteger bytesPerPixel = 4; 39 | NSUInteger bytesPerRow = bytesPerPixel * width; 40 | NSUInteger bitsPerComponent = 8; 41 | 42 | CGContextRef context = CGBitmapContextCreate(pixels, width, height, 43 | bitsPerComponent, bytesPerRow, rgb, 44 | kCGImageAlphaPremultipliedLast | kCGBitmapByteOrder32Big); 45 | 46 | CGContextDrawImage(context, CGRectMake(0, 0, width, height), imageRef); 47 | 48 | NSUInteger byteIndex = 0; 49 | CGFloat *aResult = (CGFloat *) calloc(height * width * 3, sizeof(CGFloat)); 50 | 51 | for (int i = 0 ; i < count ; ++i) 52 | { 53 | aResult[i*2 + i] = (CGFloat) pixels[byteIndex + 2]; 54 | aResult[i*2 + i+1] = (CGFloat) pixels[byteIndex + 1]; 55 | aResult[i*2 + i+2] = (CGFloat) pixels[byteIndex]; 56 | 57 | byteIndex += bytesPerPixel; 58 | } 59 | 60 | CGContextRelease(context); 61 | CGColorSpaceRelease(rgb); 62 | free(pixels); 63 | 64 | return aResult; 65 | } 66 | 67 | 68 | 69 | + (UIImage *)getImageFromRGBA:(CGFloat*)imgTensor channels:(int)channels width:(int)width height:(int)height; 70 | { 71 | UIImage* result; 72 | 73 | int newChannels = 4; 74 | int totalPixels = height * width * newChannels; 75 | CGColorSpaceRef rgb = CGColorSpaceCreateDeviceRGB(); 76 | 77 | NSUInteger byteIndex = 0; 78 | unsigned char * pixels = (unsigned char *) calloc(totalPixels, sizeof(unsigned char)); 79 | 80 | for(int i=0; i < width*height; ++i) { 81 | pixels[newChannels*i] = imgTensor[byteIndex + 2]; 82 | pixels[newChannels*i+1] = imgTensor[byteIndex + 1]; 83 | pixels[newChannels*i+2] = imgTensor[byteIndex]; 84 | pixels[newChannels*i+3] = 1; 85 | 86 | byteIndex += channels; 87 | } 88 | 89 | CGDataProviderRef ref = CGDataProviderCreateWithData(NULL, pixels, width * height * newChannels, NULL); 90 | CGImageRef iref = CGImageCreate(width, height, 8, 8 * newChannels, width * newChannels, rgb, kCGBitmapByteOrderDefault, ref, NULL, true, kCGRenderingIntentDefault); 91 | 92 | result = [UIImage imageWithCGImage:iref]; 93 | 94 | CGImageRelease(iref); 95 | CGDataProviderRelease(ref); 96 | CGColorSpaceRelease(rgb); 97 | 98 | return result; 99 | } 100 | 101 | 102 | 103 | RCT_EXPORT_METHOD(chill:(NSURL *)imagePath 104 | callback:(RCTResponseSenderBlock)callback) 105 | { 106 | ALAssetsLibrary *assetsLibrary = [[ALAssetsLibrary alloc] init]; 107 | 108 | int channels = 3; 109 | int width = 600; 110 | int height = 600; 111 | 112 | 113 | [assetsLibrary assetForURL:imagePath resultBlock: ^(ALAsset *asset){ 114 | 115 | ALAssetRepresentation *representation = [asset defaultRepresentation]; 116 | CGImageRef imageRef = [representation fullResolutionImage]; 117 | 118 | if (imageRef) { 119 | 120 | UIImageView *imageView = [[UIImageView alloc] initWithFrame:CGRectMake(0, 0, 1080, 1920)]; 121 | imageView.image = [UIImage imageWithCGImage:imageRef scale:representation.scale orientation:representation.orientation]; 122 | 123 | CGSize newSize = CGSizeMake(width, height); 124 | UIImage *newImage = [imageView.image resizedImage:newSize interpolationQuality:1]; 125 | 126 | CGFloat *imgTensor = [ImageManager getRGBAsFromImage:newImage]; 127 | 128 | CGFloat *transformedImgTensor = [RunModel transformImage:imgTensor height:height width:width channels:channels]; 129 | 130 | UIImage *transformedImg = [ImageManager getImageFromRGBA:transformedImgTensor 131 | channels:channels width:width height:height]; 132 | 133 | // To save the newly created UIImage to the camera roll 134 | // UIImageWriteToSavedPhotosAlbum(transformedImg, nil, nil, nil); 135 | 136 | NSString *transformedImgAsBase64 = [transformedImg base64String]; 137 | 138 | callback(@[[NSNull null], transformedImgAsBase64]); 139 | 140 | } else { 141 | RCTLogInfo(@"ERROR ELSE"); 142 | NSLog(@"ERROR ELSE"); 143 | } 144 | } failureBlock: ^(NSError *error){ 145 | RCTLogInfo(@"FAILURE"); 146 | NSLog(@"Failure"); 147 | }]; 148 | } 149 | 150 | @end -------------------------------------------------------------------------------- /tests/test_atrous_convolution_2d.py: -------------------------------------------------------------------------------- 1 | import os, sys, unittest 2 | import numpy as np 3 | 4 | dir = os.path.dirname(os.path.realpath(__file__)) 5 | sys.path.append(dir + '/..') 6 | 7 | from keras import backend as K 8 | if K._BACKEND == 'theano': 9 | import theano 10 | import theano.tensor as T 11 | else: 12 | import tensorflow as tf 13 | 14 | from keras.layers import Input 15 | from keras.models import Model 16 | 17 | from models.layers.ATrousConvolution2D import ATrousConvolution2D 18 | 19 | class TestATrousConvolution2d(unittest.TestCase): 20 | 21 | def test_convolution_transpose_th(self): 22 | if K._BACKEND != 'tensorflow': 23 | return True 24 | K.set_image_dim_ordering('th') 25 | 26 | border_mode = 'valid' 27 | batch = 1 28 | height = 10 29 | width = 10 30 | channels_in = 1 31 | channels_out = 2 32 | kernel_size = 3 33 | rate = 2 34 | input_shape = (channels_in, height, width) 35 | 36 | input = Input(shape=input_shape, dtype=K.floatx()) 37 | conv_layer = ATrousConvolution2D(channels_out, kernel_size, kernel_size, 38 | rate, dim_ordering=K.image_dim_ordering(), init='one', 39 | border_mode=border_mode, activation='linear') 40 | output = conv_layer(input) 41 | model = Model(input=[input], output=[output]) 42 | model.compile(loss='mean_squared_error', optimizer='sgd') 43 | 44 | x = np.ones((batch,) + input_shape).astype(K.floatx()) 45 | kernel = conv_layer.W 46 | output_model = model.predict(x) 47 | if K._BACKEND == 'tensorflow': 48 | x = tf.transpose(x, (0, 2, 3, 1)) 49 | kernel = tf.transpose(kernel, (2, 3, 1, 0)) 50 | 51 | y = tf.nn.atrous_conv2d(x, kernel, rate, padding=border_mode.upper()) 52 | 53 | y = tf.transpose(y, (0, 3, 1, 2)) 54 | output = y.eval(session=K.get_session()) 55 | 56 | self.assertEqual(output_model.shape, (1, 2, 6, 6)) 57 | self.assertEqual(output.shape, (1, 2, 6, 6)) 58 | self.assertEqual(True, (output==output_model).all()) 59 | 60 | def test_convolution_transpose_tf(self): 61 | if K._BACKEND != 'tensorflow': 62 | return True 63 | K.set_image_dim_ordering('tf') 64 | 65 | border_mode = 'valid' 66 | batch = 1 67 | height = 10 68 | width = 10 69 | channels_in = 1 70 | channels_out = 2 71 | kernel_size = 3 72 | # effective kernel size: kernel_size + (kernel_size - 1) * (rate - 1) 73 | rate = 2 74 | input_shape = (height, width, channels_in) 75 | 76 | input = Input(shape=input_shape, dtype=K.floatx()) 77 | conv_layer = ATrousConvolution2D(channels_out, kernel_size, kernel_size, 78 | rate, dim_ordering=K.image_dim_ordering(), init='one', 79 | border_mode=border_mode, activation='linear') 80 | output = conv_layer(input) 81 | model = Model(input=[input], output=[output]) 82 | model.compile(loss='mean_squared_error', optimizer='sgd') 83 | 84 | x = np.ones((batch,) + input_shape).astype(K.floatx()) 85 | kernel = conv_layer.W 86 | output_model = model.predict(x) 87 | if K._BACKEND == 'tensorflow': 88 | y = tf.nn.atrous_conv2d(x, kernel, rate, padding=border_mode.upper()) 89 | output = y.eval(session=K.get_session()) 90 | 91 | self.assertEqual(output_model.shape, (1, 6, 6, 2)) 92 | self.assertEqual(output.shape, (1, 6, 6, 2)) 93 | self.assertEqual(True, (output==output_model).all()) 94 | 95 | def test_convolution_transpose_tf_sameborder(self): 96 | if K._BACKEND != 'tensorflow': 97 | return True 98 | K.set_image_dim_ordering('tf') 99 | 100 | border_mode = 'same' 101 | batch = 1 102 | height = 10 103 | width = 10 104 | channels_in = 1 105 | channels_out = 2 106 | kernel_size = 3 107 | # effective kernel size: kernel_size + (kernel_size - 1) * (rate - 1) 108 | rate = 2 109 | input_shape = (height, width, channels_in) 110 | 111 | input = Input(shape=input_shape, dtype=K.floatx()) 112 | conv_layer = ATrousConvolution2D(channels_out, kernel_size, kernel_size, 113 | rate, dim_ordering=K.image_dim_ordering(), init='one', 114 | border_mode=border_mode, activation='linear') 115 | output = conv_layer(input) 116 | model = Model(input=[input], output=[output]) 117 | model.compile(loss='mean_squared_error', optimizer='sgd') 118 | 119 | x = np.ones((batch,) + input_shape).astype(K.floatx()) 120 | kernel = conv_layer.W 121 | output_model = model.predict(x) 122 | if K._BACKEND == 'tensorflow': 123 | y = tf.nn.atrous_conv2d(x, kernel, rate, padding=border_mode.upper()) 124 | output = y.eval(session=K.get_session()) 125 | 126 | self.assertEqual(output_model.shape, (1, 10, 10, 2)) 127 | self.assertEqual(output.shape, (1, 10, 10, 2)) 128 | self.assertEqual(True, (output==output_model).all()) 129 | 130 | 131 | if __name__ == "__main__": 132 | unittest.main() -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF.xcodeproj/xcuserdata/tom.xcuserdatad/xcdebugger/Breakpoints_v2.xcbkptlist: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 8 | 20 | 21 | 22 | 24 | 36 | 37 | 38 | 40 | 52 | 53 | 54 | 56 | 68 | 69 | 70 | 72 | 84 | 85 | 86 | 88 | 100 | 101 | 102 | 104 | 116 | 117 | 118 | 119 | 120 | -------------------------------------------------------------------------------- /pretrain_model.py: -------------------------------------------------------------------------------- 1 | import os, json, time, argparse 2 | 3 | from keras import backend as K 4 | from keras.optimizers import Adam 5 | # from keras.utils.visualize_util import plot as plot_model 6 | 7 | from models.style_transfer import (st_convt, st_conv_inception, st_convt_inception_prelu, 8 | st_conv_inception_4, st_conv_inception_4_fast, 9 | st_conv_inception_4_superresolution, fast_st_ps) 10 | 11 | from utils.imutils import plot_losses, load_images, load_data, resize 12 | from utils.general import export_model 13 | 14 | if K._BACKEND == "tensorflow": 15 | K.set_image_dim_ordering('tf') 16 | else: 17 | K.set_image_dim_ordering('th') 18 | 19 | dir = os.path.dirname(os.path.realpath(__file__)) 20 | dataDir = dir + '/data' 21 | trainDir = dataDir + '/train' 22 | overfitDir = dataDir + '/overfit_600' 23 | 24 | parser = argparse.ArgumentParser( 25 | description='Neural artistic style. Generates an image by combining ' 26 | 'the content of an image and the style of another.', 27 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 28 | ) 29 | parser.add_argument('--model', default='fast_st_ps', type=str, choices=['transpose', 'inception', 'inception_prelu', 'inception_4', 'inception_4_fast', 'fast_st_ps', 'superresolution'], help='Load pretrained weights') 30 | parser.add_argument('--training_mode', default='identity', type=str, choices=['identity', 'overfit'], help='Load pretrained weights') 31 | parser.add_argument('--weights', default='', type=str, help='Load pretrained weights') 32 | parser.add_argument('--batch_size', default=4, type=int, help='batch size.') 33 | parser.add_argument('--image_size', default=600, type=int, help='Input image size.') 34 | parser.add_argument('--nb_epoch', default=30, type=int, help='Number of epoch.') 35 | parser.add_argument('--nb_res_layer', default=6, type=int, help='Number of residual layers in the style transfer model.') 36 | parser.add_argument('--lr', default=1e-3, type=float, help='The learning rate') 37 | args = parser.parse_args() 38 | 39 | results_dir = dir + '/models/data/' + args.training_mode 40 | if not os.path.isdir(results_dir): 41 | os.makedirs(results_dir) 42 | 43 | dim_ordering = K.image_dim_ordering() 44 | channels = 3 45 | width = args.image_size 46 | height = args.image_size 47 | size = (height, width) 48 | if dim_ordering == 'th': 49 | input_shape = (channels, width, height) 50 | else: 51 | input_shape = (width, height, channels) 52 | batch_size = args.batch_size 53 | 54 | print('Loading data') 55 | if args.training_mode == 'identity': 56 | X = load_images(overfitDir, size=(height, width), preprocess_type='st', verbose=False) 57 | y = X.copy() 58 | X_cv = load_images(overfitDir + '/cv', size=(height, width), preprocess_type='st', verbose=False) 59 | y_cv = X_cv.copy() 60 | elif args.training_mode == 'overfit': 61 | (X, y), (X_cv, y_cv) = load_data(overfitDir, size=(height, width), preprocess_type='st', verbose=False) 62 | else: 63 | raise Exception('training_mode unknown: %s' % args.training_mode) 64 | print('X.shape', X.shape) 65 | print('y.shape', y.shape) 66 | print('X_cv.shape', X_cv.shape) 67 | print('y_cv.shape', y_cv.shape) 68 | 69 | print('Loading model') 70 | # mode 1 should be possible but keras is complaining in the train.py file 71 | # it doesn't like the idea that i will call later `output = pretrain_model(input)` 72 | if args.model == 'transpose': 73 | st_model = st_convt(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 74 | elif args.model == 'inception': 75 | st_model = st_conv_inception(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 76 | elif args.model == 'inception_prelu': 77 | st_model = st_convt_inception_prelu(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 78 | elif args.model == 'inception_4': 79 | st_model = st_conv_inception_4(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 80 | elif args.model == 'inception_4_fast': 81 | st_model = st_conv_inception_4_fast(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 82 | elif args.model == 'fast_st_ps': 83 | st_model = fast_st_ps(input_shape, mode=2, nb_res_layer=args.nb_res_layer) 84 | elif args.model == 'superresolution': 85 | X = resize(X, (height/4, width/4)) 86 | X_cv = resize(X_cv, (height/4, width/4)) 87 | st_model = st_conv_inception_4_superresolution(input_shape, mode=1, nb_res_layer=args.nb_res_layer) 88 | else: 89 | raise Exception('Model name %s not allowed , should not happen anyway' % args.model) 90 | 91 | if os.path.isfile(args.weights): 92 | print("Loading weights") 93 | st_model.load_weights(args.weights) 94 | 95 | print('Compiling model') 96 | adam = Adam(lr=args.lr, clipnorm=5.) # Clipping the norm avoid gradient explosion, no needs to suffocate it 97 | st_model.compile(adam, loss='mse') # loss=frobenius_error (this is not giving the same loss) 98 | 99 | print('Training model for %d epochs' % args.nb_epoch) 100 | history = st_model.fit(X, y, batch_size=args.batch_size, nb_epoch=args.nb_epoch, verbose=1, validation_data=(X_cv, y_cv)) 101 | losses = history.history 102 | 103 | if args.nb_epoch > 0: 104 | print("Saving final data") 105 | prefixedDir = prefixed_dir = "%s/%s-%s-%s" % (results_dir, str(int(time.time())), K._BACKEND, dim_ordering) 106 | export_model(st_model, prefixedDir) 107 | with open(prefixedDir + '/losses.json', 'w') as outfile: 108 | json.dump(losses, outfile) 109 | plot_losses(losses, prefixedDir) -------------------------------------------------------------------------------- /layer_reconstruction.py: -------------------------------------------------------------------------------- 1 | from __future__ import absolute_import 2 | 3 | import os, argparse, json 4 | import numpy as np 5 | 6 | from keras import backend as K 7 | 8 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 9 | 10 | from utils.imutils import (load_image, create_noise_tensor, 11 | dump_as_hdf5, deprocess, save_image, 12 | plot_losses) 13 | from utils.lossutils import (frobenius_error, grams, norm_l2, train_input) 14 | 15 | if K._BACKEND == "tensorflow": 16 | K.set_image_dim_ordering('tf') 17 | else: 18 | K.set_image_dim_ordering('th') 19 | 20 | optimizer = 'adam' 21 | if optimizer == 'lbfgs': 22 | K.set_floatx('float64') # scipy needs float64 to use lbfgs 23 | 24 | dir = os.path.dirname(os.path.realpath(__file__)) 25 | vgg19Dir = dir + '/vgg19' 26 | dataDir = dir + '/data' 27 | resultsDir = dataDir + '/output/vgg19/reconstruction' 28 | if not os.path.isdir(resultsDir): 29 | os.makedirs(resultsDir) 30 | 31 | parser = argparse.ArgumentParser( 32 | description='Neural artistic style. Generates an image by combining ' 33 | 'the content of an image and the style of another.', 34 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 35 | ) 36 | parser.add_argument('--content', default=dataDir + '/overfit/COCO_val2014_000000000074.jpg', type=str, help='Content image.') 37 | parser.add_argument('--style', default=dataDir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 38 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 39 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 40 | parser.add_argument('--max_iter', default=800, type=int, help='Number of training iter.') 41 | args = parser.parse_args() 42 | 43 | 44 | dim_ordering = K.image_dim_ordering() 45 | print('dim_ordering: %s' % dim_ordering) 46 | channels = 3 47 | width = args.image_size 48 | height = args.image_size 49 | size = (height, width) 50 | if dim_ordering == 'th': 51 | input_shape = (channels, width, height) 52 | else: 53 | input_shape = (width, height, channels) 54 | 55 | X_train = np.array([load_image(args.content, size=(height, width), preprocess_type='vgg19', verbose=True)]) 56 | print("X_train shape:", X_train.shape) 57 | 58 | X_train_style = np.array([load_image(args.style, size=(height, width), preprocess_type='vgg19', verbose=True)]) 59 | print("X_train_style shape:", X_train_style.shape) 60 | 61 | print('Loading VGG headless 5') 62 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 63 | model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 64 | layer_dict, layers_names = get_layer_data(model, 'conv_') 65 | print('Layers found:' + ', '.join(layers_names)) 66 | 67 | input_layer = model.input 68 | 69 | print('Building white noise images') 70 | input_data = create_noise_tensor(height, width, channels) 71 | 72 | mean_losses = {} 73 | print('Using optimizer: ' + optimizer) 74 | current_iter = 1 75 | for layer_name in layers_names: 76 | prefix = str(current_iter).zfill(4) 77 | 78 | print('Creating labels for ' + layer_name) 79 | out = layer_dict[layer_name].output 80 | predict = K.function([input_layer], [out]) 81 | 82 | y_style = predict([X_train_style])[0] 83 | y_content = predict([X_train])[0] 84 | 85 | print('Compiling VGG headless 1 for ' + layer_name + ' style reconstruction') 86 | loss_style = frobenius_error(grams(y_style), grams(out)) 87 | grads_style = K.gradients(loss_style, input_layer) 88 | if optimizer == 'adam': 89 | grads_style = norm_l2(grads_style) 90 | iterate_style = K.function([input_layer], [loss_style, grads_style]) 91 | 92 | print('Training the image for style') 93 | config = {'learning_rate': 5e-1} 94 | best_input_style_data, style_losses = train_input( 95 | input_data, 96 | iterate_style, 97 | optimizer, 98 | config, 99 | max_iter=args.max_iter, 100 | ) 101 | fullOutPath = resultsDir + '/' + prefix + '_style_' + layer_name + ".png" 102 | save_image(fullOutPath, best_input_style_data[0], deprocess_type='vgg19') 103 | plot_losses(style_losses, resultsDir, prefix + '_style_') 104 | 105 | print('Compiling VGG headless 1 for ' + layer_name + ' content reconstruction') 106 | loss_content = frobenius_error(y_content, out) 107 | grads_content = K.gradients(loss_content, input_layer) 108 | if optimizer == 'adam': 109 | grads_content = norm_l2(grads_content) 110 | iterate_content = K.function([input_layer], [loss_content, grads_content]) 111 | 112 | print('Training the image for content') 113 | config = {'learning_rate': 5e-1} 114 | best_input_content_data, content_losses = train_input(input_data, iterate_content, optimizer, config, max_iter=args.max_iter) 115 | fullOutPath = resultsDir + '/' + prefix + '_content_' + layer_name + ".png" 116 | save_image(fullOutPath, best_input_content_data[0], deprocess_type='vgg19') 117 | plot_losses(content_losses, resultsDir, prefix + '_content_') 118 | 119 | mean_losses[layer_name] = { 120 | 'style': { 121 | 'min': min(style_losses['loss']), 122 | 'mean': np.mean(style_losses['loss']), 123 | 'max': max(style_losses['loss']) 124 | }, 125 | 'content': { 126 | 'min': min(content_losses['loss']), 127 | 'mean': np.mean(content_losses['loss']), 128 | 'max': max(content_losses['loss']) 129 | } 130 | } 131 | 132 | current_iter += 1 133 | 134 | with open(resultsDir + '/' + 'layer_weights.json', 'w') as outfile: 135 | json.dump(mean_losses, outfile) 136 | -------------------------------------------------------------------------------- /tests/utils/test_imutils.py: -------------------------------------------------------------------------------- 1 | import os, sys, unittest 2 | import numpy as np 3 | 4 | dir = os.path.dirname(os.path.realpath(__file__)) 5 | sys.path.append(dir + '/../..') 6 | 7 | from utils.imutils import (load_images, load_image, load_mean, 8 | preprocess, deprocess, create_noise_tensor, resize) 9 | from scipy import misc 10 | 11 | from keras import backend as K 12 | 13 | dir = os.path.dirname(os.path.realpath(__file__)) 14 | 15 | class TestImUtils(unittest.TestCase): 16 | 17 | def test_load_mean_tf(self): 18 | previous_image_dim_ordering = K.image_dim_ordering() 19 | K.set_image_dim_ordering('tf') 20 | mean = load_mean() 21 | real_mean = np.array([[[[103.939, 116.779, 123.68]]]]) 22 | K.set_image_dim_ordering(previous_image_dim_ordering) 23 | 24 | self.assertEqual(True, (mean==real_mean).all()) 25 | 26 | def test_load_mean_th(self): 27 | previous_image_dim_ordering = K.image_dim_ordering() 28 | K.set_image_dim_ordering('th') 29 | mean = load_mean() 30 | real_mean = np.array([[[[103.939]], [[116.779]], [[123.68]]]]) 31 | K.set_image_dim_ordering(previous_image_dim_ordering) 32 | 33 | self.assertEqual(True, (mean==real_mean).all()) 34 | 35 | # def test_load_mean_exception(self): 36 | # self.assertRaises(Exception, load_mean('test')) 37 | 38 | def test_preprocess_tf_vgg19(self): 39 | previous_image_dim_ordering = K.image_dim_ordering() 40 | K.set_image_dim_ordering('tf') 41 | blue_im = misc.imread(dir + '/../fixture/blue.png') 42 | red_im = np.array(misc.imread(dir + '/../fixture/red.png').astype(K.floatx())) 43 | red_im = (red_im - load_mean()[0]).astype('uint8') 44 | new_red_im = preprocess(blue_im, type='vgg19').astype('uint8') 45 | K.set_image_dim_ordering(previous_image_dim_ordering) 46 | 47 | self.assertEqual(True, (red_im==new_red_im).all()) 48 | 49 | def test_preprocess_tf_none(self): 50 | previous_image_dim_ordering = K.image_dim_ordering() 51 | K.set_image_dim_ordering('tf') 52 | blue_im = misc.imread(dir + '/../fixture/blue.png') 53 | new_blue_im = preprocess(blue_im).astype('uint8') 54 | K.set_image_dim_ordering(previous_image_dim_ordering) 55 | 56 | self.assertEqual(True, (blue_im==new_blue_im).all()) 57 | 58 | def test_preprocess_tf_st(self): 59 | previous_image_dim_ordering = K.image_dim_ordering() 60 | K.set_image_dim_ordering('tf') 61 | blue_im = misc.imread(dir + '/../fixture/blue.png') 62 | red_im = np.array(misc.imread(dir + '/../fixture/red.png').astype(K.floatx())) 63 | new_red_im = preprocess(blue_im, type='st').astype('uint8') 64 | K.set_image_dim_ordering(previous_image_dim_ordering) 65 | 66 | self.assertEqual(True, (red_im==new_red_im).all()) 67 | 68 | def test_load_image(self): 69 | previous_image_dim_ordering = K.image_dim_ordering() 70 | K.set_image_dim_ordering('tf') 71 | blue_im = load_image(dir + '/../fixture/blue.png') 72 | K.set_image_dim_ordering(previous_image_dim_ordering) 73 | 74 | self.assertEqual(blue_im.shape, (600, 600, 3)) 75 | 76 | def test_load_image_th(self): 77 | previous_image_dim_ordering = K.image_dim_ordering() 78 | K.set_image_dim_ordering('th') 79 | blue_im = load_image(dir + '/../fixture/blue.png') 80 | K.set_image_dim_ordering(previous_image_dim_ordering) 81 | 82 | self.assertEqual(blue_im.shape, (3, 600, 600)) 83 | 84 | def test_load_images(self): 85 | previous_image_dim_ordering = K.image_dim_ordering() 86 | K.set_image_dim_ordering('tf') 87 | files = load_images(dir + '/../fixture') 88 | K.set_image_dim_ordering(previous_image_dim_ordering) 89 | 90 | self.assertEqual(files.shape, (4, 600, 600, 3)) 91 | 92 | def test_load_images_limit(self): 93 | previous_image_dim_ordering = K.image_dim_ordering() 94 | K.set_image_dim_ordering('tf') 95 | file = load_images(dir + '/../fixture', 1) 96 | K.set_image_dim_ordering(previous_image_dim_ordering) 97 | 98 | self.assertEqual(file.shape, (1, 600, 600, 3)) 99 | 100 | def test_deprocess(self): 101 | previous_image_dim_ordering = K.image_dim_ordering() 102 | K.set_image_dim_ordering('tf') 103 | blue_im = misc.imread(dir + '/../fixture/blue.png') 104 | im = preprocess(blue_im) 105 | im = deprocess(im) 106 | K.set_image_dim_ordering(previous_image_dim_ordering) 107 | 108 | self.assertEqual(True, (blue_im==im).all()) 109 | 110 | def test_deprocess_th(self): 111 | previous_image_dim_ordering = K.image_dim_ordering() 112 | K.set_image_dim_ordering('th') 113 | blue_im = misc.imread(dir + '/../fixture/blue.png') 114 | im = preprocess(blue_im) 115 | im = deprocess(im) 116 | K.set_image_dim_ordering(previous_image_dim_ordering) 117 | 118 | self.assertEqual(True, (blue_im==im).all()) 119 | 120 | def test_create_noise_tensor(self): 121 | previous_image_dim_ordering = K.image_dim_ordering() 122 | K.set_image_dim_ordering('tf') 123 | file = create_noise_tensor(4, 5 ,3) 124 | K.set_image_dim_ordering(previous_image_dim_ordering) 125 | 126 | self.assertEqual(file.shape, (1, 4, 5, 3)) 127 | 128 | def test_resize(self): 129 | previous_image_dim_ordering = K.image_dim_ordering() 130 | K.set_image_dim_ordering('tf') 131 | 132 | ims = load_images(dir + '/../fixture') 133 | ims = resize(ims, (150, 150)) 134 | 135 | K.set_image_dim_ordering(previous_image_dim_ordering) 136 | 137 | self.assertEqual(ims.shape, (4, 150, 150, 3)) 138 | 139 | 140 | 141 | if __name__ == '__main__': 142 | unittest.main() -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # A journey into Neural Style 2 | Hello! 3 | 4 | This repository contains some work i did on the neural style algorithm to get a better sense of it. 5 | You can find all the different script to download, pretrain and train model. Using tensorflow, tensorboard and Keras. 6 | 7 | You potentially can change the Keras backend to Theano, but you might have to tweak some stuff here and there then. 8 | 9 | ## Architecture 10 | - **data:** Holds all images input/outputs 11 | - **docker:** Holds the dockerfile used to run those experiments (see the docker section for more information) 12 | - **mobile_app:** Holds all the mobile files and README to make your tensorflow model work on IOS 13 | - **models:** Holds models/layers python python files + models architecture/weights output files 14 | - **ATrouConvolution** layer for Keras 15 | - **ConvolutionTranspose2D** layer for Keras 16 | - **ScaledSigmoid** layer for Keras 17 | - **perf:** Holds very simple perf scripts to have an idea on how much you loose when you go from the titan X to an iphone 6s... 18 | - **tests:** Holds some tests sherlocks 19 | - **torch:** Holds some work done in torch, especially a keras neural net importer 20 | - **utils:** Holds utils to do preprocessing/training etc. 21 | - **vgg:** Holds vgg19 data and script files 22 | - Python files: 23 | - **alpha.py:** See the impact of alpha parameter 24 | - **layer_reconstruction.py:** Reconstruct data with iteration from the VGG 25 | - **layer_influence.py:** See the influence of each layer against each layer on VGG 26 | - **ltv.py:** See the influence of loss total variation 27 | - **predict.py:** Generate new image from a trained model 28 | - **pretrain_model.py:** Pretrain model on overfit data (see below) 29 | - **train.py:** Train a pretrain model to achieve deep art 30 | - **.sh:** files: Some script to do batch work 31 | 32 | ## Docker 33 | You should know if you want to use GPU or CPU (you might be an immortal dude, who knows) and if you want everything installed locally or on a container 34 | - Use `pip install -r requirements.txt` to have everything locally (use virtualenv, you fool) 35 | - Use nvidia-docker to run my docker-image **(Beware you need cuda 7.5 and cudnn5 locally)**: 36 | ```bash 37 | # port 6006 used for tensorboard 38 | nvidia-docker run -it -d -p 6006:6006 -v /my/aboslute/path/to/deepback:/root/deepback --name deepback morgangiraud/dlsak 39 | ``` 40 | 41 | ## I want to create deep art 42 | **First:** Download data! (Be aware, this will download around 20GB of data...) 43 | ```bash 44 | # From data directory 45 | ./download_images.sh 46 | ``` 47 | 48 | **Second:** Download the VGG19 and preprocess it! 49 | ```bash 50 | # From vgg19 directory 51 | ./download_weights.sh 52 | ``` 53 | 54 | **Third:** Build some overfit data to pretrain some models! (You can control how many overfit images you want) 55 | ```bash 56 | # From root directory 57 | ./data/build_overfit_folder.sh 58 | ``` 59 | 60 | **Fourth:** Pretrain models 61 | 62 | *If you only want to initialise a model weights, you can do it by setting the --nb_epoch parameter to 0 manually* 63 | ```bash 64 | # From root directory 65 | ./pretrain_model.sh 66 | ``` 67 | 68 | **Fifth:** Finally, train your model! 69 | ```bash 70 | # From root directory 71 | python train.py --model_dir models/path/to/model 72 | ``` 73 | 74 | **sixth:** Generate some images! 75 | ```bash 76 | # From root directory 77 | ./predict.sh path/to/my/model 78 | ``` 79 | **Amen** 80 | 81 | ## Making it happen in your Iphone: 82 | **First:** Export your trained model 83 | ``` 84 | python export_keras_model.py --model_dir models/path/to/model 85 | ``` 86 | 87 | **Second:** Copy the `pb` file into `mobile_app/data` directory 88 | 89 | **Third:** See the [Mobile Readme](mobile_app) 90 | 91 | ## Tensorboard 92 | Tensorboard feature is built-in only in the `train.py` file. 93 | 94 | While you are training our model, you can see in real-time your loss evolution and intermediate images result (train/cross val): 95 | ```bash 96 | docker exec -it my-nvidia-docker-name bash 97 | tensorboard --logdir /path/to/my/parent/models/folder --reload_interval 20 98 | # If you launch tensorbaord in an already populated folder, it might take time before it can 99 | # squash all the data 100 | ``` 101 | Access your board: http://my-dns.com:6006 and get something like that: 102 | ![Tensorboard](screenshot-tensorboard.png) 103 | 104 | Tensorboard doc [here](https://www.tensorflow.org/versions/r0.9/how_tos/summaries_and_tensorboard/index.html) 105 | 106 | ## Performance result 107 | **We can see a x703 time slower in a tensorflow CPU non-quantized implementation versus the AWS GPU** 108 | 109 | | Infrastructure | Time | 110 | |:--:|:--:| 111 | | tensorflow GPU K420 AWS: | ~0.0329s (batching) 0.026s (looping) | 112 | | tensorflow mac os: | 4.13s (batching) 3.31s (looping) | 113 | | tensorflow mac os simulateur: | 5.86s (float32) 16.9 (quantized, this is weird) | 114 | | tensorflow iphone 6s CPU: | 18.30s (float32) | 115 | 116 | **Image size**: 600x600 117 | 118 | **batching**: all image are processed in parallel 119 | 120 | **looping**: one image at a time 121 | 122 | **quantized**: Using tensorflow quantization system, (Shouldn't be slower, probably needs a cleaner tf graph) 123 | 124 | # Other stuffs 125 | ### Using ffmpeg to create unstable deep art videos 126 | Resize: `ffmpeg -i test_video.MOV -filter:v "crop=600:600:70:400" out.mp4` 127 | video to frames: `ffmpeg -i out.mp4 -r 30/1 $filename%03d.jpeg` 128 | **predict frames HERE** 129 | frames to video: `ffmpeg -framerate 30/1 -i %03d.jpeg -c:v libx264 -r 30 -pix_fmt yuv420p out.mp4` 130 | 131 | # Aknowledgment 132 | - Thanks to [@karpathy](https://github.com/karpathy), [@jcjohnson](https://github.com/jcjohnson) and [@leongatys](https://github.com/leongatys) for their work/code! :beers: 133 | - [A Neural Algorithm of Artistic Style](https://arxiv.org/abs/1508.06576) 134 | - [Perceptual Losses for Real-Time Style Transfer and Super-Resolution](http://arxiv.org/abs/1603.08155) 135 | - [Stanford course](http://cs231n.stanford.edu/) 136 | - :+1: Tensorflow 137 | - :+1: Keras -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/RunModel.mm: -------------------------------------------------------------------------------- 1 | #import "RunModel.h" 2 | 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "google/protobuf/io/coded_stream.h" 11 | #include "google/protobuf/io/zero_copy_stream_impl.h" 12 | #include "google/protobuf/io/zero_copy_stream_impl_lite.h" 13 | #include "google/protobuf/message_lite.h" 14 | #include "tensorflow/core/framework/tensor.h" 15 | #include "tensorflow/core/framework/types.pb.h" 16 | #include "tensorflow/core/platform/env.h" 17 | #include "tensorflow/core/platform/logging.h" 18 | #include "tensorflow/core/platform/mutex.h" 19 | #include "tensorflow/core/platform/types.h" 20 | #include "tensorflow/core/public/session.h" 21 | 22 | #include "ImageManager.h" 23 | 24 | NSString* RunInferenceOnImage(); 25 | 26 | 27 | namespace { 28 | class IfstreamInputStream : public ::google::protobuf::io::CopyingInputStream { 29 | public: 30 | explicit IfstreamInputStream(const std::string& file_name) 31 | : ifs_(file_name.c_str(), std::ios::in | std::ios::binary) {} 32 | ~IfstreamInputStream() { ifs_.close(); } 33 | 34 | int Read(void* buffer, int size) { 35 | if (!ifs_) { 36 | return -1; 37 | } 38 | ifs_.read(static_cast(buffer), size); 39 | return ifs_.gcount(); 40 | } 41 | 42 | private: 43 | std::ifstream ifs_; 44 | }; 45 | } 46 | 47 | 48 | 49 | 50 | bool PortableReadFileToProto(const std::string& file_name, 51 | ::google::protobuf::MessageLite* proto) { 52 | ::google::protobuf::io::CopyingInputStreamAdaptor stream( 53 | new IfstreamInputStream(file_name)); 54 | stream.SetOwnsCopyingStream(true); 55 | // TODO(jiayq): the following coded stream is for debugging purposes to allow 56 | // one to parse arbitrarily large messages for MessageLite. One most likely 57 | // doesn't want to put protobufs larger than 64MB on Android, so we should 58 | // eventually remove this and quit loud when a large protobuf is passed in. 59 | ::google::protobuf::io::CodedInputStream coded_stream(&stream); 60 | // Total bytes hard limit / warning limit are set to 1GB and 512MB 61 | // respectively. 62 | coded_stream.SetTotalBytesLimit(1024LL << 20, 512LL << 20); 63 | return proto->ParseFromCodedStream(&coded_stream); 64 | } 65 | 66 | 67 | 68 | NSString* FilePathForResourceName(NSString* name, NSString* extension) { 69 | NSString* file_path = [[NSBundle mainBundle] pathForResource:name ofType:extension]; 70 | if (file_path == NULL) { 71 | LOG(FATAL) << "Couldn't find '" << [name UTF8String] << "." 72 | << [extension UTF8String] << "' in bundle."; 73 | } 74 | return file_path; 75 | } 76 | 77 | 78 | 79 | @implementation RunModel { 80 | } 81 | 82 | 83 | + (CGFloat *)transformImage:(CGFloat *)imgTensor height:(int)height width:(int)width channels:(int)channels; 84 | { 85 | 86 | tensorflow::Tensor image_tensor( 87 | tensorflow::DT_FLOAT, 88 | tensorflow::TensorShape({1, height, width, channels}) 89 | ); 90 | 91 | 92 | auto image_tensor_mapped = image_tensor.tensor(); 93 | float* out = image_tensor_mapped.data(); 94 | 95 | for (int y = 0; y < height; ++y) { 96 | for (int x = 0; x < width; ++x) { 97 | for (int c = 0; c < channels; ++c) { 98 | out[y*width*channels + x*channels + c] = imgTensor[y*width*channels + x*channels + c]; 99 | } 100 | } 101 | } 102 | 103 | 104 | // 1. Create a Session with SessionOptions 105 | 106 | tensorflow::SessionOptions options; // Default Options for a Session 107 | tensorflow::Session* session_pointer = nullptr; 108 | 109 | tensorflow::Status session_status = tensorflow::NewSession(options, &session_pointer); 110 | 111 | if (!session_status.ok()) { 112 | std::string status_string = session_status.ToString(); 113 | } 114 | 115 | std::unique_ptr session(session_pointer); 116 | LOG(INFO) << "Session created."; 117 | 118 | tensorflow::GraphDef tensorflow_graph; 119 | LOG(INFO) << "Graph created."; 120 | 121 | 122 | // 2. Load the network 123 | 124 | NSString* network_path = FilePathForResourceName(@"tf-frozen_overfit_8b", @"pb"); 125 | PortableReadFileToProto([network_path UTF8String], &tensorflow_graph); 126 | 127 | LOG(INFO) << "Creating session."; 128 | tensorflow::Status s = session->Create(tensorflow_graph); 129 | 130 | if (!s.ok()) { 131 | LOG(ERROR) << "Could not create Tensorflow Graph: " << s; 132 | } 133 | 134 | 135 | // 3. Run the network 136 | 137 | std::string input_layer = "input_node:0"; 138 | std::string output_layer = "mul_105:0"; 139 | 140 | std::string keras_learning_phase = "keras_learning_phase"; 141 | tensorflow::Tensor keras_learning_phase_tensor( 142 | tensorflow::DT_UINT8, 143 | tensorflow::TensorShape({})); 144 | 145 | 146 | std::vector outputs; 147 | 148 | // tensorflow::Status run_status = session->Run({ 149 | // {input_layer, image_tensor}, {keras_learning_phase, keras_learning_phase_tensor} 150 | // }, {output_layer}, {}, &outputs); 151 | 152 | NSDate *methodStart = [NSDate date]; 153 | 154 | tensorflow::Status run_status = session->Run({ 155 | {input_layer, image_tensor} 156 | }, {output_layer}, {}, &outputs); 157 | 158 | NSDate *methodFinish = [NSDate date]; 159 | 160 | NSTimeInterval executionTime = [methodFinish timeIntervalSinceDate:methodStart]; 161 | LOG(INFO) << "executionTime: " << executionTime; 162 | 163 | if (!run_status.ok()) { 164 | LOG(ERROR) << "Running model failed: " << run_status; 165 | } 166 | 167 | tensorflow::string status_string = run_status.ToString(); 168 | tensorflow::Tensor* output = &outputs[0]; 169 | 170 | auto output_vec = output->shaped({1, 3, 600, 600}); 171 | 172 | CGFloat * myOutput = (CGFloat *) calloc(width*height*3, sizeof(CGFloat)); 173 | 174 | for (int i = 0; i < width*height*3; ++i) { 175 | myOutput[i] = MAX(MIN(output_vec(i), 255.), 0.); 176 | } 177 | 178 | // 4. Return the result 179 | return myOutput; 180 | } 181 | 182 | 183 | @end 184 | -------------------------------------------------------------------------------- /utils/freeze_graph.py: -------------------------------------------------------------------------------- 1 | # 2 | # This file should be removed when it will be definitely imported in tensorflow 3 | # 4 | 5 | # Copyright 2015 The TensorFlow Authors. All Rights Reserved. 6 | # 7 | # Licensed under the Apache License, Version 2.0 (the "License"); 8 | # you may not use this file except in compliance with the License. 9 | # You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, software 14 | # distributed under the License is distributed on an "AS IS" BASIS, 15 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | # See the License for the specific language governing permissions and 17 | # limitations under the License. 18 | # ============================================================================== 19 | """Converts checkpoint variables into Const ops in a standalone GraphDef file. 20 | 21 | This script is designed to take a GraphDef proto, a SaverDef proto, and a set of 22 | variable values stored in a checkpoint file, and output a GraphDef with all of 23 | the variable ops converted into const ops containing the values of the 24 | variables. 25 | 26 | It's useful to do this when we need to load a single file in C++, especially in 27 | environments like mobile or embedded where we may not have access to the 28 | RestoreTensor ops and file loading calls that they rely on. 29 | 30 | An example of command-line usage is: 31 | bazel build tensorflow/python/tools:freeze_graph && \ 32 | bazel-bin/tensorflow/python/tools/freeze_graph \ 33 | --input_graph=some_graph_def.pb \ 34 | --input_checkpoint=model.ckpt-8361242 \ 35 | --output_graph=/tmp/frozen_graph.pb --output_node_names=softmax 36 | 37 | You can also look at freeze_graph_test.py for an example of how to use it. 38 | 39 | """ 40 | from __future__ import absolute_import 41 | from __future__ import division 42 | from __future__ import print_function 43 | 44 | import tensorflow as tf 45 | 46 | from google.protobuf import text_format 47 | from tensorflow.python.framework import graph_util 48 | 49 | 50 | FLAGS = tf.app.flags.FLAGS 51 | 52 | tf.app.flags.DEFINE_string("input_graph", "", 53 | """TensorFlow 'GraphDef' file to load.""") 54 | tf.app.flags.DEFINE_string("input_saver", "", 55 | """TensorFlow saver file to load.""") 56 | tf.app.flags.DEFINE_string("input_checkpoint", "", 57 | """TensorFlow variables file to load.""") 58 | tf.app.flags.DEFINE_string("output_graph", "", 59 | """Output 'GraphDef' file name.""") 60 | tf.app.flags.DEFINE_boolean("input_binary", False, 61 | """Whether the input files are in binary format.""") 62 | tf.app.flags.DEFINE_string("output_node_names", "", 63 | """The name of the output nodes, comma separated.""") 64 | tf.app.flags.DEFINE_string("restore_op_name", "save/restore_all", 65 | """The name of the master restore operator.""") 66 | tf.app.flags.DEFINE_string("filename_tensor_name", "save/Const:0", 67 | """The name of the tensor holding the save path.""") 68 | tf.app.flags.DEFINE_boolean("clear_devices", True, 69 | """Whether to remove device specifications.""") 70 | tf.app.flags.DEFINE_string("initializer_nodes", "", "comma separated list of " 71 | "initializer nodes to run before freezing.") 72 | 73 | 74 | def freeze_graph(input_graph, input_saver, input_binary, input_checkpoint, 75 | output_node_names, restore_op_name, filename_tensor_name, 76 | output_graph, clear_devices, initializer_nodes, verbose=True): 77 | """Converts all variables in a graph and checkpoint into constants.""" 78 | 79 | if not tf.gfile.Exists(input_graph): 80 | print("Input graph file '" + input_graph + "' does not exist!") 81 | return -1 82 | 83 | if input_saver and not tf.gfile.Exists(input_saver): 84 | print("Input saver file '" + input_saver + "' does not exist!") 85 | return -1 86 | 87 | if not tf.gfile.Glob(input_checkpoint): 88 | print("Input checkpoint '" + input_checkpoint + "' doesn't exist!") 89 | return -1 90 | 91 | if not output_node_names: 92 | print("You need to supply the name of a node to --output_node_names.") 93 | return -1 94 | 95 | input_graph_def = tf.GraphDef() 96 | mode = "rb" if input_binary else "r" 97 | with tf.gfile.FastGFile(input_graph, mode) as f: 98 | if input_binary: 99 | input_graph_def.ParseFromString(f.read()) 100 | else: 101 | text_format.Merge(f.read(), input_graph_def) 102 | # Remove all the explicit device specifications for this node. This helps to 103 | # make the graph more portable. 104 | if clear_devices: 105 | for node in input_graph_def.node: 106 | node.device = "" 107 | _ = tf.import_graph_def(input_graph_def, name="") 108 | 109 | with tf.Session() as sess: 110 | if input_saver: 111 | with tf.gfile.FastGFile(input_saver, mode) as f: 112 | saver_def = tf.train.SaverDef() 113 | if input_binary: 114 | saver_def.ParseFromString(f.read()) 115 | else: 116 | text_format.Merge(f.read(), saver_def) 117 | saver = tf.train.Saver(saver_def=saver_def) 118 | saver.restore(sess, input_checkpoint) 119 | else: 120 | sess.run([restore_op_name], {filename_tensor_name: input_checkpoint}) 121 | if initializer_nodes: 122 | sess.run(initializer_nodes) 123 | output_graph_def = graph_util.convert_variables_to_constants( 124 | sess, input_graph_def, output_node_names.split(",")) 125 | 126 | with tf.gfile.GFile(output_graph, "wb") as f: 127 | f.write(output_graph_def.SerializeToString()) 128 | if verbose == True: 129 | print("%d ops in the final graph." % len(output_graph_def.node)) 130 | 131 | 132 | def main(unused_args): 133 | freeze_graph(FLAGS.input_graph, FLAGS.input_saver, FLAGS.input_binary, 134 | FLAGS.input_checkpoint, FLAGS.output_node_names, 135 | FLAGS.restore_op_name, FLAGS.filename_tensor_name, 136 | FLAGS.output_graph, FLAGS.clear_devices, FLAGS.initializer_nodes) 137 | 138 | if __name__ == "__main__": 139 | tf.app.run() -------------------------------------------------------------------------------- /mobile_app/ios/ReactNativeTF/UIImage+Alpha.m: -------------------------------------------------------------------------------- 1 | // UIImage+Alpha.m 2 | // Created by Trevor Harmon on 9/20/09. 3 | // Free for personal or commercial use, with or without modification. 4 | // No warranty is expressed or implied. 5 | 6 | #import "UIImage+Alpha.h" 7 | 8 | @implementation UIImage (Alpha) 9 | 10 | // Returns true if the image has an alpha layer 11 | - (BOOL)hasAlpha { 12 | CGImageAlphaInfo alpha = CGImageGetAlphaInfo(self.CGImage); 13 | return (alpha == kCGImageAlphaFirst || 14 | alpha == kCGImageAlphaLast || 15 | alpha == kCGImageAlphaPremultipliedFirst || 16 | alpha == kCGImageAlphaPremultipliedLast); 17 | } 18 | 19 | // Returns a copy of the given image, adding an alpha channel if it doesn't already have one 20 | - (UIImage *)imageWithAlpha { 21 | if ([self hasAlpha]) { 22 | return self; 23 | } 24 | 25 | CGFloat scale = MAX(self.scale, 1.0f); 26 | CGImageRef imageRef = self.CGImage; 27 | size_t width = CGImageGetWidth(imageRef)*scale; 28 | size_t height = CGImageGetHeight(imageRef)*scale; 29 | 30 | // The bitsPerComponent and bitmapInfo values are hard-coded to prevent an "unsupported parameter combination" error 31 | CGContextRef offscreenContext = CGBitmapContextCreate(NULL, 32 | width, 33 | height, 34 | 8, 35 | 0, 36 | CGImageGetColorSpace(imageRef), 37 | kCGBitmapByteOrderDefault | kCGImageAlphaPremultipliedFirst); 38 | 39 | // Draw the image into the context and retrieve the new image, which will now have an alpha layer 40 | CGContextDrawImage(offscreenContext, CGRectMake(0, 0, width, height), imageRef); 41 | CGImageRef imageRefWithAlpha = CGBitmapContextCreateImage(offscreenContext); 42 | UIImage *imageWithAlpha = [UIImage imageWithCGImage:imageRefWithAlpha scale:self.scale orientation:UIImageOrientationUp]; 43 | 44 | // Clean up 45 | CGContextRelease(offscreenContext); 46 | CGImageRelease(imageRefWithAlpha); 47 | 48 | return imageWithAlpha; 49 | } 50 | 51 | // Returns a copy of the image with a transparent border of the given size added around its edges. 52 | // If the image has no alpha layer, one will be added to it. 53 | - (UIImage *)transparentBorderImage:(NSUInteger)borderSize { 54 | // If the image does not have an alpha layer, add one 55 | UIImage *image = [self imageWithAlpha]; 56 | CGFloat scale = MAX(self.scale, 1.0f); 57 | NSUInteger scaledBorderSize = borderSize * scale; 58 | CGRect newRect = CGRectMake(0, 0, image.size.width * scale + scaledBorderSize * 2, image.size.height * scale + scaledBorderSize * 2); 59 | 60 | // Build a context that's the same dimensions as the new size 61 | CGContextRef bitmap = CGBitmapContextCreate(NULL, 62 | newRect.size.width, 63 | newRect.size.height, 64 | CGImageGetBitsPerComponent(self.CGImage), 65 | 0, 66 | CGImageGetColorSpace(self.CGImage), 67 | CGImageGetBitmapInfo(self.CGImage)); 68 | 69 | // Draw the image in the center of the context, leaving a gap around the edges 70 | CGRect imageLocation = CGRectMake(scaledBorderSize, scaledBorderSize, image.size.width*scale, image.size.height*scale); 71 | CGContextDrawImage(bitmap, imageLocation, self.CGImage); 72 | CGImageRef borderImageRef = CGBitmapContextCreateImage(bitmap); 73 | 74 | // Create a mask to make the border transparent, and combine it with the image 75 | CGImageRef maskImageRef = [self newBorderMask:scaledBorderSize size:newRect.size]; 76 | CGImageRef transparentBorderImageRef = CGImageCreateWithMask(borderImageRef, maskImageRef); 77 | UIImage *transparentBorderImage = [UIImage imageWithCGImage:transparentBorderImageRef scale:self.scale orientation:UIImageOrientationUp]; 78 | 79 | // Clean up 80 | CGContextRelease(bitmap); 81 | CGImageRelease(borderImageRef); 82 | CGImageRelease(maskImageRef); 83 | CGImageRelease(transparentBorderImageRef); 84 | 85 | return transparentBorderImage; 86 | } 87 | 88 | #pragma mark - 89 | #pragma mark Private helper methods 90 | 91 | // Creates a mask that makes the outer edges transparent and everything else opaque 92 | // The size must include the entire mask (opaque part + transparent border) 93 | // The caller is responsible for releasing the returned reference by calling CGImageRelease 94 | - (CGImageRef)newBorderMask:(NSUInteger)borderSize size:(CGSize)size { 95 | CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceGray(); 96 | 97 | // Build a context that's the same dimensions as the new size 98 | CGContextRef maskContext = CGBitmapContextCreate(NULL, 99 | size.width, 100 | size.height, 101 | 8, // 8-bit grayscale 102 | 0, 103 | colorSpace, 104 | kCGBitmapByteOrderDefault | kCGImageAlphaNone); 105 | 106 | // Start with a mask that's entirely transparent 107 | CGContextSetFillColorWithColor(maskContext, [UIColor blackColor].CGColor); 108 | CGContextFillRect(maskContext, CGRectMake(0, 0, size.width, size.height)); 109 | 110 | // Make the inner part (within the border) opaque 111 | CGContextSetFillColorWithColor(maskContext, [UIColor whiteColor].CGColor); 112 | CGContextFillRect(maskContext, CGRectMake(borderSize, borderSize, size.width - borderSize * 2, size.height - borderSize * 2)); 113 | 114 | // Get an image of the context 115 | CGImageRef maskImageRef = CGBitmapContextCreateImage(maskContext); 116 | 117 | // Clean up 118 | CGContextRelease(maskContext); 119 | CGColorSpaceRelease(colorSpace); 120 | 121 | return maskImageRef; 122 | } 123 | 124 | @end 125 | -------------------------------------------------------------------------------- /models/fast_style_transfer.py: -------------------------------------------------------------------------------- 1 | import os, sys, argparse 2 | 3 | import tensorflow as tf 4 | import numpy as np 5 | from tensorflow-vgg import vgg19 6 | 7 | dir = os.path.dirname(os.path.realpath(__file__)) 8 | sys.path.append(dir + '/..') 9 | 10 | def instance_norm_layer(name, X): 11 | with tf.variable_scope(name): 12 | (mean, variance) = tf.nn.moments(X, [1, 2], keep_dims=True) 13 | shape = tf.shape(X) 14 | offset = tf.get_variable('%s-offset' % name, shape=[shape[0], 1, 1, shape[3]]) 15 | scale = tf.get_variable('%s-scale' % name, shape=[shape[0], 1, 1, shape[3]]) 16 | variance_epsilon = 1e-7 17 | 18 | A = tf.nn.batch_normalization(X, mean, variance, offset, scale, variance_epsilon) 19 | Y = tf.nn.relu(A) 20 | 21 | return Y 22 | 23 | def res_layer(name, X): 24 | batch_dim, height, width, channels = tf.shape(X) 25 | with tf.variable_scope(name): 26 | W1 = tf.get_variable('%s-W1' % name, shape=[3, 3, channels, channels]) 27 | b1 = tf.get_variable('%s-b1', shape=[channels]) 28 | A1 = tf.nn.conv2d(X, W1, strides=[1, 1, 1, 1], padding="SAME") + b1 29 | Z1 = tf.nn.relu(A1) 30 | N1 = instance_norm_layer(Z1) 31 | 32 | W2 = tf.get_variable('%s-W2' % name, shape=[3, 3, channels, channels]) 33 | b2 = tf.get_variable('%s-b2', shape=[channels]) 34 | A2 = tf.nn.conv2d(N1, W2, strides=[1, 1, 1, 1], padding="SAME") + b2 35 | Z2 = tf.nn.relu(A2) 36 | N2 = instance_norm_layer(Z2) 37 | 38 | Y = tf.add(X, N2) 39 | 40 | return Y 41 | 42 | def subpixel_layer(name, X, r, color=False): 43 | def _phase_shift(I, r): 44 | bsize, a, b, c = I.shape().as_list() 45 | bsize = tf.shape(I)[0] # I've made a minor change to the original implementation to enable Dimension(None) for the batch dim 46 | X = tf.reshape(I, (bsize, a, b, r, r)) 47 | X = tf.transpose(X, (0, 1, 2, 4, 3)) # bsize, a, b, 1, 1 48 | X = tf.split(1, a, X) # a, [bsize, b, r, r] 49 | X = tf.concat(2, [tf.squeeze(x) for x in X]) # bsize, b, a*r, r 50 | X = tf.split(1, b, X) # b, [bsize, a*r, r] 51 | X = tf.concat(2, [tf.squeeze(x) for x in X]) # bsize, a*r, b*r 52 | return tf.reshape(X, (bsize, a*r, b*r, 1)) 53 | 54 | with tf.variable_scope(name): 55 | if color: 56 | Xc = tf.split(3, 3, X) 57 | Y = tf.concat(3, [_phase_shift(x, r) for x in Xc]) 58 | else: 59 | Y = _phase_shift(X, r) 60 | return Y 61 | 62 | 63 | # inputs th ordering, BGR 64 | def fast_style_transfer(input_shape, ratio=4, nb_res_layer=5): 65 | with tf.variable_scope('Placeholder'): 66 | X = tf.placeholder(tf.int32, shape=input_shape, name='Inputs_placeholder') 67 | 68 | with tf.variable_scope('Encoder'): 69 | W1 = tf.get_variable('encoder-W1', shape=[3, 3, 3, 32]) 70 | b1 = tf.get_variable('encoder-b1', shape=[32]) 71 | A1 = tf.nn.conv2d(X, W1, strides=[1, 2, 2, 1], padding="SAME") + b1 72 | Z1 = tf.nn.relu(A1) 73 | Y = instance_norm_layer(Z1) 74 | 75 | W2 = tf.get_variable('encoder-W2', shape=[3, 3, 32, 128]) 76 | b2 = tf.get_variable('encoder-b2', shape=[128]) 77 | A2 = tf.nn.conv2d(Y, W2, strides=[1, 2, 2, 1], padding="SAME") + b2 78 | Z2 = tf.nn.relu(A2) 79 | Y = instance_norm_layer(Z2) 80 | 81 | if ratio is 8: 82 | W3 = tf.get_variable('encoder-W3', shape=[3, 3, 128, 256]) 83 | b3 = tf.get_variable('encoder-b3', shape=[256]) 84 | A3 = tf.nn.conv2d(Y, W3, strides=[1, 2, 2, 1], padding="SAME") + b3 85 | Z3 = tf.nn.relu(A3) 86 | Y = instance_norm_layer(Z3) 87 | 88 | for i in range(nb_res_layer): 89 | Y = res_layer('residual_layer_%d' % i, Y) 90 | 91 | with tf.variable_scope('Decoder'): 92 | if ratio is 8: 93 | W_decoder = tf.get_variable('encoder-W', shape=[3, 3, 256, 192]) 94 | b_decoder = tf.get_variable('encoder-b', shape=[192]) 95 | else: 96 | W_decoder = tf.get_variable('encoder-W', shape=[3, 3, 256, 48]) 97 | b_decoder = tf.get_variable('encoder-b', shape=[48]) 98 | 99 | Z = tf.nn.conv2d(Y, W_decoder, strides=[1, 1, 1, 1], padding="SAME") + b_decoder 100 | Y = subpixel_layer('subpixel_layer', Z, ratio, color=True) 101 | 102 | return { 103 | 'input': X, 104 | 'output': Y 105 | } 106 | 107 | 108 | if __name__ == "__main__": 109 | data_dir = dir + '/../data' 110 | 111 | parser = argparse.ArgumentParser( 112 | description='Neural artistic style. Generates an image by combining ' 113 | 'the content of an image and the style of another.', 114 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 115 | ) 116 | parser.add_argument('--content', default=data_dir + '/overfit/COCO_val2014_000000000074.jpg', type=str, help='Content image.') 117 | parser.add_argument('--style', default=data_dir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 118 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 119 | parser.add_argument('--ratio', default=4, type=int, help='Ratio between encoding and decoding') 120 | parser.add_argument('--nb_res_layer', default=5, type=int, help='Number of residual layer.') 121 | args = parser.parse_args() 122 | 123 | dir = os.path.dirname(os.path.realpath(__file__)) 124 | results_dir = dir + '/data/st' 125 | 126 | input_shape = [None, args.image_size, args.image_size, 3] 127 | fst = fast_style_transfer(input_shape, ratio=args.ratio, nb_res_layer=args.nb_res_layer) 128 | tf.image_summary('input_img', fst['input'], 2) 129 | tf.image_summary('output_img', fst['output'], 2) 130 | 131 | vgg = vgg19.Vgg19() 132 | 133 | 134 | style_loss = 1 135 | content_loss = 1 136 | tv_loss = 1 137 | tf.scalar_summary('style_loss', style_loss) 138 | tf.scalar_summary('content_loss', content_loss) 139 | tf.scalar_summary('tv_loss', tv_loss) 140 | 141 | adam = tf.AdamOptimizer(1e-3) 142 | train_op = adam.minimize(total_loss) 143 | 144 | for i in range(10): 145 | style_coef = np.random.random_integers(50,150) 146 | content_coef = np.random.random_integers(5,10) 147 | tv_coef = np.random.random_integers(5,10) 148 | 149 | total_loss = style_coef * style_loss + content_coef * content_loss + tv_coef * tv_loss 150 | 151 | tf.scalar_summary('total_loss', total_loss) 152 | 153 | summaries_op = tf.merge_all_summaries() 154 | -------------------------------------------------------------------------------- /vgg19/model_headless.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | from keras import backend as K 4 | from keras.layers.convolutional import (Convolution2D, MaxPooling2D, AveragePooling2D, 5 | ZeroPadding2D) 6 | from keras.layers import Input 7 | from keras.models import Model 8 | 9 | def VGG_19_headless_5(input_shape, weights_path=None, trainable=False, pooling_type='max'): 10 | do = K.image_dim_ordering() 11 | 12 | input = Input(shape=input_shape, name='input', dtype=K.floatx()) 13 | zp11 = ZeroPadding2D((1, 1), dim_ordering=do, trainable=trainable)(input) 14 | c11 = Convolution2D(64, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_1_1")(zp11) 15 | zp12 = ZeroPadding2D((1, 1), dim_ordering=do)(c11) 16 | c12 = Convolution2D(64, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_1_2")(zp12) 17 | if pooling_type == 'avg': 18 | p1 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c12) 19 | else: 20 | p1 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c12) 21 | 22 | zp21 = ZeroPadding2D((1, 1), dim_ordering=do)(p1) 23 | c21 = Convolution2D(128, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_2_1")(zp21) 24 | zp22 = ZeroPadding2D((1, 1), dim_ordering=do)(c21) 25 | c22 = Convolution2D(128, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_2_2")(zp22) 26 | if pooling_type == 'avg': 27 | p2 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c22) 28 | else: 29 | p2 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c22) 30 | 31 | zp31 = ZeroPadding2D((1, 1), dim_ordering=do)(p2) 32 | c31 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_1")(zp31) 33 | zp32 = ZeroPadding2D((1, 1), dim_ordering=do)(c31) 34 | c32 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_2")(zp32) 35 | zp33 = ZeroPadding2D((1, 1), dim_ordering=do)(c32) 36 | c33 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_3")(zp33) 37 | zp34 = ZeroPadding2D((1, 1), dim_ordering=do)(c33) 38 | c34 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_4")(zp34) 39 | if pooling_type == 'avg': 40 | p3 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c34) 41 | else: 42 | p3 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c34) 43 | 44 | 45 | zp41 = ZeroPadding2D((1, 1), dim_ordering=do)(p3) 46 | c41 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_1")(zp41) 47 | zp42 = ZeroPadding2D((1, 1), dim_ordering=do)(c41) 48 | c42 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_2")(zp42) 49 | zp43 = ZeroPadding2D((1, 1), dim_ordering=do)(c42) 50 | c43 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_3")(zp43) 51 | zp44 = ZeroPadding2D((1, 1), dim_ordering=do)(c43) 52 | c44 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_4")(zp44) 53 | if pooling_type == 'avg': 54 | p4 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c44) 55 | else: 56 | p4 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c44) 57 | 58 | zp51 = ZeroPadding2D((1, 1), dim_ordering=do)(p4) 59 | c51 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_5_1")(zp51) 60 | zp52 = ZeroPadding2D((1, 1), dim_ordering=do)(c51) 61 | c52 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_5_2")(zp52) 62 | zp53 = ZeroPadding2D((1, 1), dim_ordering=do)(c52) 63 | c53 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_5_3")(zp53) 64 | zp54 = ZeroPadding2D((1, 1), dim_ordering=do)(c53) 65 | c54 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_5_4")(zp54) 66 | 67 | model = Model(input=[input], output=[ 68 | c11, c12, 69 | c21, c22, 70 | c31, c32, c33, c34, 71 | c41, c42, c43, c44, 72 | c51, c52, c53, c54] 73 | ) 74 | 75 | if weights_path: 76 | model.load_weights(weights_path) 77 | 78 | return model 79 | 80 | def VGG_19_headless_4(input_shape, weights_path=None, trainable=False, pooling_type='max'): 81 | do = K.image_dim_ordering() 82 | 83 | input = Input(shape=input_shape, name='input', dtype=K.floatx()) 84 | zp11 = ZeroPadding2D((1, 1), dim_ordering=do, trainable=trainable)(input) 85 | c11 = Convolution2D(64, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_1_1")(zp11) 86 | zp12 = ZeroPadding2D((1, 1), dim_ordering=do)(c11) 87 | c12 = Convolution2D(64, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_1_2")(zp12) 88 | if pooling_type == 'avg': 89 | p1 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c12) 90 | else: 91 | p1 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c12) 92 | 93 | zp21 = ZeroPadding2D((1, 1), dim_ordering=do)(p1) 94 | c21 = Convolution2D(128, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_2_1")(zp21) 95 | zp22 = ZeroPadding2D((1, 1), dim_ordering=do)(c21) 96 | c22 = Convolution2D(128, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_2_2")(zp22) 97 | if pooling_type == 'avg': 98 | p2 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c22) 99 | else: 100 | p2 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c22) 101 | 102 | zp31 = ZeroPadding2D((1, 1), dim_ordering=do)(p2) 103 | c31 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_1")(zp31) 104 | zp32 = ZeroPadding2D((1, 1), dim_ordering=do)(c31) 105 | c32 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_2")(zp32) 106 | zp33 = ZeroPadding2D((1, 1), dim_ordering=do)(c32) 107 | c33 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_3")(zp33) 108 | zp34 = ZeroPadding2D((1, 1), dim_ordering=do)(c33) 109 | c34 = Convolution2D(256, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_3_4")(zp34) 110 | if pooling_type == 'avg': 111 | p3 = AveragePooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c34) 112 | else: 113 | p3 = MaxPooling2D((2, 2), dim_ordering=do, strides=(2, 2))(c34) 114 | 115 | 116 | zp41 = ZeroPadding2D((1, 1), dim_ordering=do)(p3) 117 | c41 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_1")(zp41) 118 | zp42 = ZeroPadding2D((1, 1), dim_ordering=do)(c41) 119 | c42 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_2")(zp42) 120 | zp43 = ZeroPadding2D((1, 1), dim_ordering=do)(c42) 121 | c43 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_3")(zp43) 122 | zp44 = ZeroPadding2D((1, 1), dim_ordering=do)(c43) 123 | c44 = Convolution2D(512, 3, 3, dim_ordering=do, activation='relu', trainable=trainable, name="conv_4_4")(zp44) 124 | 125 | model = Model(input=[input], output=[ 126 | c11, c12, 127 | c21, c22, 128 | c31, c32, c33, c34, 129 | c41, c42, c43, c44] 130 | ) 131 | 132 | if weights_path: 133 | model.load_weights(weights_path) 134 | 135 | return model 136 | 137 | def get_layer_data(model, pattern=''): 138 | layer_dict = dict([(layer.name, layer) for layer in model.layers]) 139 | if re != '': 140 | layers_names = [l for l in layer_dict if len(re.findall(pattern, l))] 141 | else: 142 | layers_names = [l for l in layer_dict] 143 | layers_names.sort() 144 | 145 | return (layer_dict, layers_names) 146 | -------------------------------------------------------------------------------- /utils/imutils.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import os, re, functools 3 | import h5py 4 | import matplotlib as mpl 5 | mpl.use('Agg') 6 | import matplotlib.pyplot as plt 7 | 8 | from scipy import misc 9 | from scipy.misc import imsave 10 | 11 | from keras import backend as K 12 | 13 | from vgg19.model import VGG_19_mean 14 | 15 | dir = os.path.dirname(os.path.realpath(__file__)) 16 | vgg19Dir = dir + '/../vgg19' 17 | 18 | # VGG 19 | def memoize(obj): 20 | cache = obj.cache = {} 21 | 22 | @functools.wraps(obj) 23 | def memoizer(*args, **kwargs): 24 | key = str(args) + str(kwargs) 25 | if key not in cache: 26 | cache[key] = obj(*args, **kwargs) 27 | return cache[key] 28 | 29 | return memoizer 30 | 31 | def load_data(absPath, limit=-1, size=(600, 600), preprocess_type='none', verbose=False): 32 | X, y = load_images(absPath, limit, size, preprocess_type=preprocess_type, verbose=verbose, load_result=True) 33 | X_cv, y_cv = load_images(absPath + '/cv', limit, size, preprocess_type=preprocess_type, verbose=verbose, load_result=True) 34 | 35 | return (X, y), (X_cv, y_cv) 36 | 37 | def get_image_list(absPath): 38 | filenames = [absPath + '/' + f for f in os.listdir(absPath) if len(re.findall('\.(jpe?g|png)$', f))] 39 | 40 | return filenames 41 | 42 | def load_images(absPath, limit=-1, size=(600, 600), preprocess_type='none', verbose=False, load_result=False): 43 | ims = [] 44 | y_ims = [] 45 | if type(absPath) == list: 46 | fullpaths = absPath 47 | else: 48 | fullpaths = get_image_list(absPath) 49 | for idx, fullpath in enumerate(fullpaths): 50 | if limit > 0 and idx >= limit: 51 | break 52 | 53 | if load_result == True: 54 | im, y_im = load_image(fullpath, size, preprocess_type, verbose, load_result) 55 | else: 56 | im = load_image(fullpath, size, preprocess_type, verbose, load_result) 57 | 58 | ims.append(im) 59 | if load_result == True: 60 | y_ims.append(y_im) 61 | 62 | if load_result == True: 63 | return np.array(ims), np.array(y_ims) 64 | else: 65 | return np.array(ims) 66 | 67 | def load_image(fullpath, size=(600, 600), preprocess_type='none', verbose=False, load_result=False): 68 | if verbose: 69 | print('Loading %s' % fullpath) 70 | # VGG needs BGR data 71 | im = misc.imread(fullpath, mode='RGB') # tf ordering 72 | im = preprocess(im.copy(), size, type=preprocess_type) # tf ordering 73 | if K.image_dim_ordering() == 'th': 74 | im = im.transpose(2, 0, 1) 75 | 76 | if load_result == True: 77 | y_fullpath = get_y_fullpath(fullpath) 78 | if verbose: 79 | print('Loading %s' % y_fullpath) 80 | 81 | y_im = misc.imread(y_fullpath, mode='RGB') # height, width, channels 82 | y_im = preprocess(y_im.copy(), size, type=preprocess_type) 83 | if K.image_dim_ordering() == 'th': 84 | y_im = y_im.transpose(2, 0, 1) 85 | 86 | return im, y_im 87 | 88 | return im 89 | 90 | def get_y_fullpath(x_fullpath): 91 | splitted_fullpath = x_fullpath.split('/') 92 | filename = splitted_fullpath.pop() 93 | y_fullpath = '/'.join(splitted_fullpath + ['labels', filename]) 94 | 95 | return y_fullpath 96 | 97 | def resize(im, size): 98 | if size == None: 99 | return im 100 | 101 | nb_dims = len(im.shape) 102 | if nb_dims == 3: 103 | if type(size) != tuple: 104 | size = float(size) / im.shape[0] # make a float 105 | 106 | return misc.imresize(im, size, interp='bilinear') 107 | elif nb_dims == 4: 108 | if type(size) != tuple: 109 | size = float(size) / im.shape[1] # make a float 110 | 111 | ims = [] 112 | for i in range(im.shape[0]): 113 | new_im = misc.imresize(im[i], size, interp='bilinear') 114 | ims.append(new_im) 115 | 116 | return np.array(ims) 117 | else: 118 | raise Exception('image should have 3 or 4 dimensions') 119 | 120 | # input im is assumed in RGB tf ordering 121 | # return a BGR tf image 122 | def preprocess(im, size=None, type='none'): 123 | # im is in a tf ordering 124 | im = resize(im, size) 125 | im = im.copy().astype(K.floatx()) 126 | if type == 'none': 127 | pass 128 | elif type == 'vgg19': 129 | nb_dims = len(im.shape) 130 | dim_ordering = K.image_dim_ordering() 131 | mean = load_mean() # load with K.image_dim_ordering() 132 | if dim_ordering == 'th': # 'th' dim_ordering: [channels, height, width] 133 | mean = mean.transpose(0, 2, 3, 1) 134 | perm = np.argsort([2, 1, 0]) # VGG needs BGR 135 | 136 | if nb_dims == 3: 137 | im = im[:, :, perm] 138 | im -= mean[0] 139 | elif nb_dims == 4: 140 | im = im[:, :, :, perm] 141 | im -= mean 142 | else: 143 | raise Exception('image should have 3 or 4 dimensions') 144 | elif type == 'st': 145 | perm = np.argsort([2, 1, 0]) 146 | im = im[:, :, perm] # th ordering, BGR 147 | else: 148 | raise Exception('Unknown preprocess type: %s' % type) 149 | 150 | return im 151 | 152 | def deprocess(im, type='none'): 153 | im = np.copy(im) 154 | 155 | if type =='none': 156 | pass 157 | elif type =='vgg19': 158 | nb_dims = len(im.shape) 159 | mean = load_mean() # load with K.image_dim_ordering() 160 | perm = np.argsort([2, 1, 0]) # VGG needed BGR 161 | if nb_dims == 3: 162 | im += mean[0] 163 | im = im[:, :, perm] 164 | elif nb_dims == 4: 165 | im += mean 166 | im = im[:, :, :, perm] 167 | else: 168 | raise Exception('image should have 3 or 4 dimensions') 169 | elif type =='st': 170 | perm = np.argsort([2, 1, 0]) 171 | 172 | if K.image_dim_ordering() == 'th': 173 | im = im.transpose((1, 2, 0)) 174 | 175 | im = im[:, :, perm] 176 | 177 | im = im.clip(0, 255) 178 | im = im.astype('uint8') 179 | 180 | return im 181 | 182 | def save_image(fullOutPath, im, deprocess_type='none'): 183 | im = deprocess(im, deprocess_type) 184 | imsave(fullOutPath, im) 185 | 186 | def load_mean(name='vgg19'): 187 | if name == 'vgg19': 188 | return VGG_19_mean(K.image_dim_ordering()) # BGR ordering 189 | else: 190 | raise Exception('Invalid mean name:' + name) 191 | 192 | def create_noise_tensor(height, width, channels): 193 | dim_ordering = K.image_dim_ordering() 194 | if dim_ordering == 'tf': 195 | return np.random.randn(1, height, width, channels).astype(K.floatx()) * 0.001 196 | elif dim_ordering == 'th': 197 | return np.random.randn(1, channels, height, width).astype(K.floatx()) * 0.001 198 | else: 199 | raise Exception('Invalid dim_ordering value:' + dim_ordering) 200 | 201 | def load_hdf5_im(fullpath): 202 | file = h5py.File(fullpath, 'r') 203 | dset = file.get('dataset_1') 204 | im = np.array(dset) 205 | file.close() 206 | 207 | return im 208 | 209 | def dump_as_hdf5(fullpath, data): 210 | with h5py.File(fullpath, 'w') as hf: 211 | hf.create_dataset('dataset_1', data=data) 212 | 213 | def plot_losses(losses, dir='', prefix='', suffix=''): 214 | plt.clf() 215 | if 'val_loss' in losses and len(losses['val_loss']): 216 | plt.subplot(2, 1, 1) 217 | plt.plot(losses['loss']) 218 | plt.title('Training loss') 219 | plt.xlabel('Iteration number') 220 | plt.ylabel('Loss value') 221 | 222 | plt.subplot(2, 1, 2) 223 | plt.plot(losses['val_loss']) 224 | plt.title('Cross validation loss') 225 | plt.xlabel('Iteration number') 226 | plt.ylabel('Loss value') 227 | 228 | plt.savefig(dir + '/' + prefix + '_val_loss' + suffix + '.png') 229 | plt.clf() 230 | else: 231 | plt.plot(losses['loss']) 232 | plt.title('Training loss') 233 | plt.xlabel('Iteration number') 234 | plt.ylabel('Loss value') 235 | plt.savefig(dir + '/' + prefix + '_loss' + suffix + '.png') 236 | plt.clf() 237 | -------------------------------------------------------------------------------- /utils/general.py: -------------------------------------------------------------------------------- 1 | import random 2 | import h5py, os, sys 3 | import numpy as np 4 | 5 | dir = os.path.dirname(os.path.realpath(__file__)) 6 | sys.path.append(dir + '/../..') 7 | 8 | from keras import backend as K 9 | if K._BACKEND == 'tensorflow': 10 | import tensorflow as tf 11 | import utils.freeze_graph as freeze_tools 12 | 13 | from keras.models import model_from_json 14 | from keras.utils.np_utils import convert_kernel 15 | from vgg19.model_headless import get_layer_data 16 | 17 | from utils.imutils import load_image, get_image_list, preprocess 18 | 19 | dir = os.path.dirname(os.path.realpath(__file__)) 20 | 21 | # You don't need any input layer in a sequential model which usually end up 22 | # with a model minus that one input layer 23 | def copySeqWeights(model, weightsFullPath, outputFilename, offset=1, limit=-1): 24 | nbLayer = len(model.layers) 25 | f = h5py.File(weightsFullPath, "r") 26 | # print(f.name, f.keys()) 27 | # print(f['layer_1'].keys(), f['layer_1']['param_0']) 28 | 29 | nbLayers = f.attrs['nb_layers'] 30 | print("Number of layers in the original weight: ", nbLayers) 31 | print("Number of layers in the model: ", len(model.layers)) 32 | for k in range(nbLayers): 33 | if k + offset >= nbLayer or (limit > 0 and k >= limit): 34 | break 35 | g = f['layer_{}'.format(k)] 36 | weights = [g.get('param_{}'.format(p))[()] for p in range(g.attrs['nb_params'])] 37 | print(model.layers[k+offset].name) 38 | if len(weights): 39 | if K._BACKEND == 'theano': 40 | # I have to do it for theano because theano flips the convolutional kernel 41 | # and i don't have access to the API of the conv ops 42 | weights[0] = np.round(weights[0][:, :, ::-1, ::-1], 4) 43 | if K.image_dim_ordering() == 'tf': 44 | weights[0] = np.transpose(weights[0], [2, 3, 1, 0]) 45 | # print(weights[0].shape) 46 | model.layers[k+offset].set_weights(weights) 47 | f.close() 48 | 49 | model.save_weights(outputFilename, overwrite=True) 50 | 51 | def import_model(absolute_model_dir, best=True, should_convert=False, custom_objects={}): 52 | archi_json = open(absolute_model_dir + '/archi.json').read() 53 | model = model_from_json(archi_json, custom_objects) 54 | 55 | if os.path.isfile(absolute_model_dir + '/best_weights.hdf5') and best: 56 | model.load_weights(absolute_model_dir + '/best_weights.hdf5') 57 | else: 58 | model.load_weights(absolute_model_dir + '/last_weights.hdf5') 59 | 60 | if should_convert == True: 61 | if K._BACKEND == 'tensorflow': 62 | ops = [] 63 | for layer in model.layers: 64 | if layer.__class__.__name__ in ['Convolution1D', 'Convolution2D']: 65 | original_w = K.get_value(layer.W) 66 | converted_w = convert_kernel(original_w) 67 | ops.append(tf.assign(layer.W, converted_w).op) 68 | K.get_session().run(ops) 69 | else: 70 | for layer in model.layers: 71 | if layer.__class__.__name__ in ['Convolution1D', 'Convolution2D']: 72 | original_w = K.get_value(layer.W) 73 | converted_w = convert_kernel(original_w) 74 | K.set_value(layer.W, converted_w) 75 | 76 | 77 | return model 78 | 79 | def export_model(model, absolute_model_dir, best_weights=None, saver=None, global_step=None): 80 | if not os.path.isdir(absolute_model_dir): 81 | os.makedirs(absolute_model_dir) 82 | 83 | model.save_weights(absolute_model_dir + "/last_weights.hdf5", overwrite=True) 84 | if K._BACKEND == 'tensorflow' and saver != None: 85 | sess = K.get_session() 86 | saver.save(sess, absolute_model_dir + '/tf-last_weights', global_step=global_step) 87 | 88 | if best_weights != None: 89 | model.set_weights(best_weights) 90 | model.save_weights(absolute_model_dir + "/best_weights.hdf5", overwrite=True) 91 | if K._BACKEND == 'tensorflow' and saver != None: 92 | saver.save(sess, absolute_model_dir + '/tf-best_weights', global_step=global_step) 93 | 94 | # Graph 95 | json = model.to_json() 96 | open(absolute_model_dir + "/archi.json", 'w').write(json) 97 | if K._BACKEND == 'tensorflow' and saver != None and global_step == None: 98 | graph_def = sess.graph.as_graph_def() 99 | tf.train.write_graph(graph_def, absolute_model_dir, 'tf-model_graph') 100 | 101 | freeze_graph(model, absolute_model_dir, best_weights) 102 | 103 | def freeze_graph(model, absolute_model_dir, best_weights=None): 104 | input_graph_path = absolute_model_dir + '/tf-model_graph' 105 | input_saver_def_path = "" 106 | input_binary = False 107 | if best_weights != None: 108 | input_checkpoint_path = absolute_model_dir + '/tf-best_weights' 109 | else: 110 | input_checkpoint_path = absolute_model_dir + '/tf-last_weights' 111 | output_node_names = model.output.name.split(':')[0] 112 | restore_op_name = "save/restore_all" 113 | filename_tensor_name = "save/Const:0" 114 | output_graph_path = absolute_model_dir + "/tf-frozen_model.pb" 115 | clear_devices = True 116 | 117 | freeze_tools.freeze_graph(input_graph_path, input_saver_def_path, 118 | input_binary, input_checkpoint_path, 119 | output_node_names, restore_op_name, 120 | filename_tensor_name, output_graph_path, 121 | clear_devices, "", verbose=False) 122 | 123 | def get_shape(x): 124 | if isinstance(x, (np.ndarray)): 125 | return x.shape 126 | elif K._BACKEND == 'theano': 127 | return K.shape(x) 128 | else: 129 | try: 130 | return K.int_shape(x) 131 | except Exception: 132 | return K.shape(x) 133 | 134 | def mask_data(data, selector): 135 | return [d for d, s in zip(data, selector) if s] 136 | 137 | def generate_data_from_image_list(image_list, size, style_fullpath_prefix, input_len=1, output_len=1, batch_size=4, transform_f=None, preprocess_type='none', verbose=False): 138 | if transform_f != None: 139 | file = h5py.File(style_fullpath_prefix + '_' + str(size[0]) + '.hdf5', 'r') 140 | y_style1 = np.array(file.get('conv_1_2')) 141 | y_style2 = np.array(file.get('conv_2_2')) 142 | y_style3 = np.array(file.get('conv_3_4')) 143 | y_style4 = np.array(file.get('conv_4_2')) 144 | 145 | # Init inputs/outputs 146 | inputs = [] 147 | outputs = [] 148 | for i in range(input_len): 149 | inputs.append([]) 150 | for i in range(output_len): 151 | outputs.append([]) 152 | nb_element = 0 153 | while 1: 154 | random.shuffle(image_list) 155 | for fullpath in image_list: 156 | nb_element += 1 157 | 158 | im = load_image(fullpath, size, preprocess_type=preprocess_type, verbose=verbose) 159 | 160 | if transform_f != None: 161 | f_input = [ preprocess(np.array([im]), type='vgg19') ] 162 | y_content = transform_f(f_input)[0][0] # First element in the list of result, first element of the output batch 163 | inputs[0].append(im) 164 | outputs[0].append(y_content) 165 | outputs[1].append(y_style1) 166 | outputs[2].append(y_style2) 167 | outputs[3].append(y_style3) 168 | outputs[4].append(y_style4) 169 | outputs[5].append(np.zeros_like(im)) 170 | else: 171 | for i in range(input_len): 172 | inputs[i].append(im) 173 | for i in range(output_len): 174 | outputs[i].append(im) 175 | 176 | if nb_element >= batch_size: 177 | nb_element = 0 178 | 179 | inputs_list = [] 180 | for i in range(input_len): 181 | inputs_list.append(np.array(inputs[i])) 182 | outputs_list = [] 183 | for i in range(output_len): 184 | outputs_list.append(np.array(outputs[i])) 185 | 186 | yield(inputs_list, outputs_list) 187 | 188 | # reset inputs/outputs 189 | for i in range(input_len): 190 | inputs[i] = [] 191 | for i in range(output_len): 192 | outputs[i] = [] 193 | 194 | 195 | 196 | -------------------------------------------------------------------------------- /utils/lossutils.py: -------------------------------------------------------------------------------- 1 | import os, re, h5py, math 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | from keras.utils.generic_utils import Progbar 6 | if K._BACKEND == 'theano': 7 | from theano import tensor as T 8 | else: 9 | import tensorflow as tf 10 | 11 | from utils.imutils import load_image 12 | from utils.general import get_shape 13 | from utils.optimizers import adam 14 | from scipy.optimize import fmin_l_bfgs_b 15 | 16 | gogh_inc_val = 0 17 | 18 | ######## 19 | # Losses 20 | ######## 21 | def grams(X): 22 | dim_ordering = K.image_dim_ordering() 23 | if dim_ordering == 'tf': 24 | X = K.permute_dimensions(X, (0, 3, 1, 2)) 25 | 26 | (samples, c, h, w) = get_shape(X) 27 | 28 | X_reshaped = K.reshape(X, (-1, c, h * w)) 29 | X_T = K.permute_dimensions(X_reshaped, (0, 2, 1)) 30 | if K._BACKEND == 'theano': 31 | X_gram = T.batched_dot(X_reshaped, X_T) 32 | else: 33 | X_gram = tf.batch_matmul(X_reshaped, X_T) 34 | X_gram /= c * h * w 35 | 36 | return X_gram 37 | 38 | def grams_output_shape(input_shape): 39 | dim_ordering = K.image_dim_ordering() 40 | if dim_ordering == 'tf': 41 | return (input_shape[0], input_shape[3], input_shape[3]) 42 | else: 43 | return (input_shape[0], input_shape[1], input_shape[1]) 44 | 45 | def frobenius_error(y_true, y_pred): 46 | loss = K.mean(K.square(y_pred - y_true)) 47 | 48 | return loss 49 | 50 | def norm_l2(x): 51 | norm = K.sqrt(K.mean(K.square(x))) 52 | return x / (norm + K.epsilon()) 53 | 54 | def load_y_styles(painting_fullpath, layers_name): 55 | y_styles = [] 56 | with h5py.File(painting_fullpath, 'r') as f: 57 | for name in layers_name: 58 | y_styles.append(f[name][()]) 59 | 60 | return y_styles 61 | 62 | 63 | 64 | ####### 65 | # Regularizer 66 | ####### 67 | def total_variation_error(y, beta=1): 68 | # Negative stop indices are not currently supported in tensorflow ... 69 | if K._BACKEND == 'theano': 70 | a = K.square(y[:, :, 1:, :-1] - y[:, :, :-1, :-1]) 71 | b = K.square(y[:, :, :-1, 1:] - y[:, :, :-1, :-1]) 72 | else: 73 | samples, c, h, w = K.int_shape(y) 74 | a = K.square(y[:, :, 1:, :w-1] - y[:, :, :h-1, :w-1]) 75 | b = K.square(y[:, :, :h-1, 1:] - y[:, :, :h-1, :w-1]) 76 | if beta == 2: 77 | loss = K.sum(a + b) / beta 78 | else: 79 | loss = K.sum(K.pow(a + b, beta/2.)) / beta 80 | 81 | return loss 82 | 83 | def total_variation_error_keras(y_true, y_pred): 84 | return total_variation_error(y_pred, 2) 85 | 86 | ########## 87 | # Training 88 | ########## 89 | def train_input(input_data, train_iteratee, optimizerName, config={}, max_iter=2000, callbacks=[]): 90 | losses = {'loss': [], 'val_loss': [], 'best_loss': 1e15} 91 | 92 | wait = 0 93 | best_input_data = None 94 | if optimizerName == 'adam': 95 | for i in range(max_iter): 96 | data = train_iteratee([input_data]) 97 | loss = data[0].item(0) 98 | grads_val = data[1] 99 | 100 | losses['loss'].append(loss) 101 | if i % 25 == 0: 102 | print('Iteration: %d/%d' % (i, max_iter) ) 103 | for idx, subloss in enumerate(data): 104 | if idx < 2: 105 | continue 106 | print(' loss %d: %f' % (idx - 1, subloss)) 107 | print(' loss: %f' % (loss)) 108 | 109 | if loss < losses['best_loss']: 110 | losses['best_loss'] = loss 111 | best_input_data = np.copy(input_data) 112 | wait = 0 113 | else: 114 | if wait >= 100 and i > max_iter / 2: 115 | break 116 | wait +=1 117 | 118 | for callback in callbacks: 119 | callback({ 120 | 'current_iter':i, 121 | 'losses': losses, 122 | 'input_data':input_data 123 | }) 124 | 125 | input_data, config = adam(input_data, grads_val, config) 126 | if K._BACKEND == 'tensorflow': 127 | input_data = input_data[0] 128 | 129 | else: 130 | global gogh_inc_val 131 | gogh_inc_val = 0 132 | def iter(x): 133 | global gogh_inc_val 134 | gogh_inc_val += 1 135 | x = np.reshape(x, input_data.shape) 136 | 137 | data = train_iteratee([x]) 138 | loss = data[0].item(0) 139 | grads_val = data[1] 140 | 141 | losses['loss'].append(loss) 142 | if gogh_inc_val % 25 == 0: 143 | print('Iteration: %d/%d' % (gogh_inc_val, max_iter) ) 144 | for idx, subloss in enumerate(data): 145 | if idx < 2: 146 | continue 147 | print(' loss %d: %f' % (idx - 1, subloss)) 148 | print(' loss: %f' % (loss)) 149 | 150 | if loss < losses['best_loss']: 151 | losses['best_loss'] = loss 152 | 153 | for callback in callbacks: 154 | callback({ 155 | 'current_iter':gogh_inc_val, 156 | 'losses': losses, 157 | 'input_data':input_data 158 | }) 159 | 160 | return loss, grads_val.reshape(-1) 161 | 162 | best_input_data, f ,d = fmin_l_bfgs_b(iter, input_data, factr=1e7, maxiter=max_iter) #factr: 1e7 default value 163 | best_input_data = np.reshape(best_input_data, input_data.shape) 164 | 165 | print("final loss:", losses['best_loss']) 166 | return best_input_data, losses 167 | 168 | def train_weights(input_dir, size, model, train_iteratee, cv_input_dir=None, max_iter=2000, batch_size=4, callbacks=[], load_result=False): 169 | losses = {'loss': [], 'val_loss': [], 'best_loss': 1e15} 170 | 171 | best_weights = model.get_weights() 172 | 173 | need_more_training = True 174 | current_iter = 0 175 | current_epoch = 1 176 | files = [input_dir + '/' + name for name in os.listdir(input_dir) if len(re.findall('\.(jpe?g|png)$', name))] 177 | batch_size = min(batch_size, len(files)) 178 | print('total_files %d' % len(files)) 179 | 180 | max_epoch = math.floor((batch_size * max_iter) / len(files)) + 1 181 | while need_more_training: 182 | print('Epoch %d/%d' % (current_epoch, max_epoch)) 183 | nb_elem = min((max_iter - current_iter) * batch_size, len(files)) 184 | progbar = Progbar(nb_elem) 185 | progbar_values = [] 186 | 187 | ims = [] 188 | y_ims = [] 189 | for idx, fullpath in enumerate(files): 190 | if load_result == True: 191 | im, y_im = load_image(fullpath, size=size, preprocess_type='st', verbose=False, load_result=load_result) # th ordering, BGR 192 | y_ims.append(y_im) 193 | else: 194 | im = load_image(fullpath, size=size, preprocess_type='st', verbose=False, load_result=load_result) # th ordering, BGR 195 | ims.append(im) 196 | 197 | if len(ims) >= batch_size or idx == len(files) - 1: 198 | current_iter += 1 199 | if load_result == True: 200 | data = train_iteratee([np.array(ims), np.array(y_ims), True]) 201 | else: 202 | data = train_iteratee([np.array(ims), True]) 203 | 204 | loss = data[0].item(0) 205 | losses['loss'].append(loss) 206 | progbar_values.append(('loss', loss)) 207 | for loss_idx, subloss in enumerate(data): 208 | if loss_idx < 1: 209 | continue 210 | progbar_values.append(('loss ' + str(loss_idx), subloss)) 211 | progbar.update(idx + 1, progbar_values) 212 | 213 | if loss < losses['best_loss']: 214 | losses['best_loss'] = loss 215 | best_weights = model.get_weights() 216 | 217 | for callback in callbacks: 218 | callback({ 219 | current_iter: current_iter, 220 | losses: losses, 221 | model: model, 222 | data: data 223 | }) 224 | 225 | ims = [] 226 | if current_iter >= max_iter: 227 | need_more_training = False 228 | break 229 | 230 | current_epoch += 1 231 | 232 | last_weights = model.get_weights() 233 | print("final best loss:", losses['best_loss']) 234 | return (best_weights, last_weights), losses 235 | -------------------------------------------------------------------------------- /gatys_paper.py: -------------------------------------------------------------------------------- 1 | import os, argparse, json, time 2 | import numpy as np 3 | 4 | from keras import backend as K 5 | 6 | from vgg19.model_headless import VGG_19_headless_5, get_layer_data 7 | 8 | from utils.imutils import (load_image, load_images, create_noise_tensor, 9 | save_image, plot_losses, get_image_list) 10 | from utils.lossutils import (frobenius_error, total_variation_error, 11 | grams, norm_l2, train_input) 12 | 13 | if K._BACKEND == "tensorflow": 14 | K.set_image_dim_ordering('tf') 15 | else: 16 | K.set_image_dim_ordering('th') 17 | 18 | dir = os.path.dirname(os.path.realpath(__file__)) 19 | vgg19Dir = dir + '/vgg19' 20 | dataDir = dir + '/data' 21 | paintingsDir = dataDir + '/paintings' 22 | 23 | parser = argparse.ArgumentParser( 24 | description='Neural artistic style. Generates an image by combining ' 25 | 'the content of an image and the style of another.', 26 | formatter_class=argparse.ArgumentDefaultsHelpFormatter 27 | ) 28 | parser.add_argument('--content', default=dataDir + '/train', type=str, help='Content image.') 29 | parser.add_argument('--style', default=dataDir + '/paintings/edvard_munch-the_scream.jpg', type=str, help='Style image.') 30 | parser.add_argument('--pooling_type', default='avg', type=str, choices=['max', 'avg'], help='VGG pooling type.') 31 | parser.add_argument('--image_size', default=256, type=int, help='Input image size.') 32 | parser.add_argument('--max_iter', default=500, type=int, help='Number of training iter.') 33 | parser.add_argument('--input_type', default='content', type=str, choices=['random', 'content'], help='How to initialize the input data') 34 | parser.add_argument('--print_inter_img', default=False, type=bool, help='Print intermediate images') 35 | parser.add_argument('--output_dir', default=dataDir + '/output/vgg19/gatys_%s' % int(time.time()), type=str, help='optional output dir') 36 | parser.add_argument('--no_dump_losses', default=False, type=bool, help='Dump a graph of the losses') 37 | parser.add_argument('--alpha', default=0, type=float, help='Style coefficient') 38 | parser.add_argument('--beta', default=0, type=float, help='Content coefficient') 39 | parser.add_argument('--gamma', default=0, type=float, help='total variation coefficient') 40 | args = parser.parse_args() 41 | 42 | output_dir = args.output_dir 43 | if not os.path.isdir(output_dir): 44 | os.makedirs(output_dir) 45 | 46 | optimizer = 'adam' 47 | if optimizer == 'lbfgs': 48 | K.set_floatx('float64') # scipy needs float64 to use lbfgs 49 | 50 | dim_ordering = K.image_dim_ordering() 51 | channels = 3 52 | width = args.image_size 53 | height = args.image_size 54 | size = (height, width) 55 | if dim_ordering == 'th': 56 | input_shape = (channels, width, height) 57 | else: 58 | input_shape = (width, height, channels) 59 | 60 | if os.path.isdir(args.content): 61 | print('dir: %s' % args.content) 62 | image_list = get_image_list(args.content) 63 | else: 64 | image_list = [args.content] 65 | X_train = load_images(image_list, size=(height, width), preprocess_type='vgg19', verbose=True) 66 | print("X_train shape:", X_train.shape) 67 | 68 | X_train_style = np.array([load_image(args.style, size=(height, width), preprocess_type='vgg19', verbose=True)]) 69 | print("X_train_style shape:", X_train_style.shape) 70 | 71 | print('Loading VGG headless 5') 72 | modelWeights = "%s/%s-%s-%s%s" % (vgg19Dir,'vgg-19', dim_ordering, K._BACKEND, '_headless_5_weights.hdf5') 73 | model = VGG_19_headless_5(input_shape, modelWeights, trainable=False, pooling_type=args.pooling_type) 74 | layer_dict, layers_names = get_layer_data(model, 'conv_') 75 | print('Layers found:' + ', '.join(layers_names)) 76 | 77 | input_layer = model.input 78 | 79 | # Layer chosen thanks to the pre_analysis script 80 | # conv_5_* layers doesn't hold enough information to rebuild the structure of the content/style 81 | style_layers = ['conv_1_2', 'conv_2_2', 'conv_3_4', 'conv_4_2'] 82 | content_layers = ['conv_2_2'] 83 | style_output_layers = [layer_dict[ls_name].output for ls_name in style_layers] 84 | content_output_layers = [layer_dict[lc_name].output for lc_name in content_layers] 85 | 86 | print('Creating training labels') 87 | predict_style = K.function([input_layer], style_output_layers) 88 | y_styles = predict_style([X_train_style]) 89 | predict_content = K.function([input_layer], content_output_layers) 90 | 91 | print('Preparing training loss functions') 92 | train_loss_styles = [] 93 | for idx, y_style in enumerate(y_styles): 94 | train_loss_styles.append( 95 | frobenius_error( 96 | grams(y_style), 97 | grams(style_output_layers[idx]) 98 | ) 99 | ) 100 | 101 | reg_TV = total_variation_error(input_layer, 2) 102 | 103 | # Random sampling is seems to produce good results when iterating over hyperparameters 104 | config = { 105 | 'style': args.style, 106 | 'pooling_type': args.pooling_type, 107 | 'max_iter': args.max_iter, 108 | 'input_type': args.input_type, 109 | 'optimizer': optimizer, 110 | 'style_layers': style_layers, 111 | 'content_layers': content_layers, 112 | 'learning_rate': 5e-1 113 | } 114 | random_search = True 115 | if args.alpha != 0 and args.beta != 0 and args.gamma != 0: 116 | random_search = False 117 | config['alpha'] = args.alpha 118 | config['beta'] = args.beta 119 | config['gamma'] = args.gamma 120 | with open(output_dir + '/config.json', 'w') as outfile: 121 | json.dump(config, outfile) 122 | 123 | print('Using optimizer: ' + optimizer) 124 | for file_idx in range(len(X_train)): 125 | print('Initializing input %s data' % args.input_type) 126 | if args.input_type == 'random': 127 | input_data = create_noise_tensor(height, width, channels) 128 | elif args.input_type == 'content': 129 | input_data = X_train[file_idx:file_idx+1, :, :, :].copy() 130 | else: 131 | raise Exception('Input type choices are random|content') 132 | 133 | current_iter = 1 134 | for idx, content_output in enumerate(content_output_layers): 135 | lc_name = content_layers[idx] 136 | y_contents = predict_content([X_train[file_idx:file_idx+1]]) 137 | train_content_loss = frobenius_error(y_contents[idx], content_output) 138 | print('Compiling VGG headless 5 for ' + lc_name + ' content reconstruction') 139 | # Those hyper parameters are selected thx to pre_analysis scripts 140 | # Made for avg pooling + content init 141 | if random_search is True: 142 | range_size = 10 143 | else: 144 | range_size = 1 # no search 145 | for i in range(range_size): 146 | # Random search 147 | if range_size == 1: 148 | alpha = args.alpha 149 | beta = args.beta 150 | gamma = args.gamma 151 | else: 152 | alpha = np.random.uniform(1, 1e2) 153 | beta = 1e0 154 | gamma = np.random.uniform(1e-6, 1e-4) 155 | print("alpha, beta, gamma:", alpha, beta, gamma) 156 | 157 | print('Computing train loss') 158 | tls = [alpha * train_loss_style / len(train_loss_styles) for style_idx, train_loss_style in enumerate(train_loss_styles)] 159 | tlc = beta * train_content_loss 160 | rtv = gamma * reg_TV 161 | train_loss = sum(tls) + tlc + rtv 162 | 163 | print('Computing gradients') 164 | grads = K.gradients(train_loss, input_layer) 165 | if optimizer == 'adam': 166 | grads = norm_l2(grads) 167 | inputs = [input_layer] 168 | outputs = [train_loss, grads, tlc] + tls 169 | 170 | print('Computing iteratee function') 171 | train_iteratee = K.function(inputs, outputs) 172 | filename_array = image_list[file_idx].split('/')[-1].split('.') 173 | 174 | suffix = "_content%s_alpha%f_beta%f_gamma%f" % (lc_name, alpha, beta, gamma) 175 | out_filename = filename_array[0] + suffix + '.' + filename_array[1] 176 | def lambda_dump_input(obj): 177 | current_iter = obj['current_iter'] 178 | input_data = obj['input_data'] 179 | if current_iter % 25 == 0 and args.print_inter_img == True: 180 | save_image(output_dir + '/' + filename_array[0] + suffix + + '_' + str(current_iter.zfill(5)) + '.' + filename_array[1], input_data[0], deprocess_type='vgg19') 181 | 182 | best_input_data, losses = train_input( 183 | input_data, 184 | train_iteratee, 185 | optimizer, 186 | {'learning_rate': config['learning_rate']}, 187 | max_iter=args.max_iter, 188 | callbacks=[lambda_dump_input] 189 | ) 190 | 191 | print('Dumping data') 192 | save_image(output_dir + '/' + out_filename, best_input_data[0], deprocess_type='vgg19') 193 | if args.no_dump_losses == False: 194 | plot_losses(losses, output_dir, filename_array[0], suffix) 195 | 196 | current_iter += 1 197 | --------------------------------------------------------------------------------