├── .DS_Store ├── .gitignore ├── README.md ├── Super Resolution for TF Talk-GPU-2k-8x-Presentation.ipynb ├── Super Resolution for TF Talk-GPU-2k-Testing-Resampling.ipynb ├── images ├── SR-network.png ├── SR_compare_original.png └── super_resoltion_example.png ├── vgg16.py └── vgg16_avg.py /.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samwit/Super-Resolution/2c1516ab6aa78602b2a68a6b3390317d71f52b01/.DS_Store -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__/* 2 | checkpoints/* 3 | data/* 4 | .ipynb_checkpoints/* 5 | weights/ -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Super Resolution using perceptual loss 2 | 3 | ### The aim is to test using perceptual loss for a Super Resolution model 4 | 5 | **The Network** 6 | ![](images/SR-network.png?raw=true) 7 | 8 | 9 | #### The Results 10 | 11 | Pics can be seen in the Jupyter notebook 12 | 13 | Overall the results are pretty good for such a small amount of data and training. Also optimization of the model (loss function mostly) should improve the results if trained on full imagenet set etc 14 | 15 | ![](images/super_resoltion_example.png?raw=true) 16 | ![](images/SR_compare_original.png?raw=true) 17 | 18 | 19 | ### Next steps to try 20 | 21 | - Using Imagenet training set 22 | - Using Resnet50 as the basic model for feature extraction 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /images/SR-network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samwit/Super-Resolution/2c1516ab6aa78602b2a68a6b3390317d71f52b01/images/SR-network.png -------------------------------------------------------------------------------- /images/SR_compare_original.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samwit/Super-Resolution/2c1516ab6aa78602b2a68a6b3390317d71f52b01/images/SR_compare_original.png -------------------------------------------------------------------------------- /images/super_resoltion_example.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/samwit/Super-Resolution/2c1516ab6aa78602b2a68a6b3390317d71f52b01/images/super_resoltion_example.png -------------------------------------------------------------------------------- /vgg16.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | from __future__ import print_function 3 | from __future__ import absolute_import 4 | 5 | import warnings 6 | 7 | from keras.models import Model 8 | from keras.layers import Flatten, Dense, Input, Dropout 9 | from keras.layers import Convolution2D, MaxPooling2D 10 | from keras.engine.topology import get_source_inputs 11 | from keras.utils.layer_utils import convert_all_kernels_in_model 12 | from keras.utils.data_utils import get_file 13 | from keras import backend as K 14 | from keras.applications.imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape 15 | 16 | 17 | PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/' 18 | TF_WEIGHTS = 'vgg16_weights_tf_dim_ordering_tf_kernels.h5' 19 | TF_WEIGHTS_NO_TOP = 'vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5' 20 | 21 | 22 | def VGG16(include_top=True, weights='imagenet', 23 | input_tensor=None, input_shape=None): 24 | if weights not in {'imagenet', None}: 25 | raise ValueError('The `weights` argument should be either ' 26 | '`None` (random initialization) or `imagenet` ' 27 | '(pre-training on ImageNet).') 28 | # Determine proper input shape 29 | input_shape = _obtain_input_shape(input_shape, 30 | default_size=224, 31 | min_size=48, 32 | dim_ordering=K.image_dim_ordering(), 33 | include_top=include_top) 34 | 35 | if input_tensor is None: 36 | img_input = Input(shape=input_shape) 37 | else: 38 | if not K.is_keras_tensor(input_tensor): 39 | img_input = Input(tensor=input_tensor, shape=input_shape) 40 | else: 41 | img_input = input_tensor 42 | # Block 1 43 | x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input) 44 | x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x) 45 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) 46 | 47 | # Block 2 48 | x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x) 49 | x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x) 50 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) 51 | 52 | # Block 3 53 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x) 54 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x) 55 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x) 56 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) 57 | 58 | # Block 4 59 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x) 60 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x) 61 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x) 62 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) 63 | 64 | # Block 5 65 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x) 66 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x) 67 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x) 68 | x = MaxPooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) 69 | 70 | if include_top: 71 | # Classification block 72 | x = Flatten(name='flatten')(x) 73 | x = Dense(4096, activation='relu', name='fc1')(x) 74 | x = Dropout(0.5)(x) 75 | x = Dense(4096, activation='relu', name='fc2')(x) 76 | x = Dropout(0.5)(x) 77 | x = Dense(1000, activation='softmax', name='predictions')(x) 78 | 79 | # Ensure that the model takes into account 80 | # any potential predecessors of `input_tensor`. 81 | if input_tensor is not None: 82 | inputs = get_source_inputs(input_tensor) 83 | else: 84 | inputs = img_input 85 | # Create model. 86 | model = Model(inputs, x, name='vgg16') 87 | 88 | # load weights 89 | if weights == 'imagenet': 90 | if K.image_dim_ordering() == 'tf': 91 | wname = TF_WEIGHTS if include_top else TF_WEIGHTS_NO_TOP 92 | weights_path = get_file(wname, PATH+wname, cache_subdir='models') 93 | model.load_weights(weights_path) 94 | return model 95 | -------------------------------------------------------------------------------- /vgg16_avg.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | from __future__ import absolute_import 3 | 4 | import warnings 5 | 6 | from keras.models import Model 7 | from keras.layers import Flatten, Dense, Input 8 | from keras.layers import Convolution2D, AveragePooling2D 9 | from keras.engine.topology import get_source_inputs 10 | from keras.utils.layer_utils import convert_all_kernels_in_model 11 | from keras.utils.data_utils import get_file 12 | from keras import backend as K 13 | from keras.applications.imagenet_utils import decode_predictions, preprocess_input, _obtain_input_shape 14 | 15 | 16 | TH_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels.h5' 17 | TF_WEIGHTS_PATH = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels.h5' 18 | TH_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_th_dim_ordering_th_kernels_notop.h5' 19 | TF_WEIGHTS_PATH_NO_TOP = 'https://github.com/fchollet/deep-learning-models/releases/download/v0.1/vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5' 20 | 21 | 22 | def VGG16_Avg(include_top=True, weights='imagenet', 23 | input_tensor=None, input_shape=None, 24 | classes=1000): 25 | if weights not in {'imagenet', None}: 26 | raise ValueError('The `weights` argument should be either ' 27 | '`None` (random initialization) or `imagenet` ' 28 | '(pre-training on ImageNet).') 29 | 30 | if weights == 'imagenet' and include_top and classes != 1000: 31 | raise ValueError('If using `weights` as imagenet with `include_top`' 32 | ' as true, `classes` should be 1000') 33 | # Determine proper input shape 34 | input_shape = _obtain_input_shape(input_shape, 35 | default_size=224, 36 | min_size=48, 37 | dim_ordering=K.image_dim_ordering(), 38 | include_top=include_top) 39 | 40 | if input_tensor is None: 41 | img_input = Input(shape=input_shape) 42 | else: 43 | if not K.is_keras_tensor(input_tensor): 44 | img_input = Input(tensor=input_tensor, shape=input_shape) 45 | else: 46 | img_input = input_tensor 47 | # Block 1 48 | x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv1')(img_input) 49 | x = Convolution2D(64, 3, 3, activation='relu', border_mode='same', name='block1_conv2')(x) 50 | x = AveragePooling2D((2, 2), strides=(2, 2), name='block1_pool')(x) 51 | 52 | # Block 2 53 | x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv1')(x) 54 | x = Convolution2D(128, 3, 3, activation='relu', border_mode='same', name='block2_conv2')(x) 55 | x = AveragePooling2D((2, 2), strides=(2, 2), name='block2_pool')(x) 56 | 57 | # Block 3 58 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv1')(x) 59 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv2')(x) 60 | x = Convolution2D(256, 3, 3, activation='relu', border_mode='same', name='block3_conv3')(x) 61 | x = AveragePooling2D((2, 2), strides=(2, 2), name='block3_pool')(x) 62 | 63 | # Block 4 64 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv1')(x) 65 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv2')(x) 66 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block4_conv3')(x) 67 | x = AveragePooling2D((2, 2), strides=(2, 2), name='block4_pool')(x) 68 | 69 | # Block 5 70 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv1')(x) 71 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv2')(x) 72 | x = Convolution2D(512, 3, 3, activation='relu', border_mode='same', name='block5_conv3')(x) 73 | x = AveragePooling2D((2, 2), strides=(2, 2), name='block5_pool')(x) 74 | 75 | if include_top: 76 | # Classification block 77 | x = Flatten(name='flatten')(x) 78 | x = Dense(4096, activation='relu', name='fc1')(x) 79 | x = Dense(4096, activation='relu', name='fc2')(x) 80 | x = Dense(classes, activation='softmax', name='predictions')(x) 81 | 82 | # Ensure that the model takes into account 83 | # any potential predecessors of `input_tensor`. 84 | if input_tensor is not None: 85 | inputs = get_source_inputs(input_tensor) 86 | else: 87 | inputs = img_input 88 | # Create model. 89 | model = Model(inputs, x, name='vgg16') 90 | 91 | # load weights 92 | if weights == 'imagenet': 93 | if K.image_dim_ordering() == 'th': 94 | if include_top: 95 | weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels.h5', 96 | TH_WEIGHTS_PATH, 97 | cache_subdir='models') 98 | else: 99 | weights_path = get_file('vgg16_weights_th_dim_ordering_th_kernels_notop.h5', 100 | TH_WEIGHTS_PATH_NO_TOP, 101 | cache_subdir='models') 102 | model.load_weights(weights_path) 103 | if K.backend() == 'tensorflow': 104 | warnings.warn('You are using the TensorFlow backend, yet you ' 105 | 'are using the Theano ' 106 | 'image dimension ordering convention ' 107 | '(`image_dim_ordering="th"`). ' 108 | 'For best performance, set ' 109 | '`image_dim_ordering="tf"` in ' 110 | 'your Keras config ' 111 | 'at ~/.keras/keras.json.') 112 | convert_all_kernels_in_model(model) 113 | else: 114 | if include_top: 115 | weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels.h5', 116 | TF_WEIGHTS_PATH, 117 | cache_subdir='models') 118 | else: 119 | weights_path = get_file('vgg16_weights_tf_dim_ordering_tf_kernels_notop.h5', 120 | TF_WEIGHTS_PATH_NO_TOP, 121 | cache_subdir='models') 122 | model.load_weights(weights_path) 123 | if K.backend() == 'theano': 124 | convert_all_kernels_in_model(model) 125 | return model 126 | --------------------------------------------------------------------------------