├── images ├── t1.jpg ├── t2.jpg ├── t3.jpg ├── t4.jpg ├── t5.jpg ├── t6.jpg ├── t7.jpg ├── t8.jpg ├── t9.jpg ├── t10.jpg ├── t11.jpg ├── t12.jpg ├── t13.jpg ├── t14.jpg ├── t15.jpg ├── t16.jpg ├── t17.jpg ├── t18.jpg ├── t19.jpg ├── t20.jpg ├── t21.jpg ├── t22.jpg ├── t23.jpg ├── t24.jpg ├── t25.jpg ├── t26.jpg ├── t27.jpg ├── t28.jpg ├── t29.jpg ├── t30.jpg ├── t31.jpg ├── t32.jpg ├── t33.jpg ├── t34.jpg ├── t35.jpg ├── t36.jpg ├── t37.jpg ├── t38.jpg ├── t39.jpg ├── t40.jpg ├── t42.jpg ├── t43.jpg ├── t44.jpg ├── t45.jpg ├── t46.jpg ├── t47.jpg ├── t48.jpg ├── t49.jpg ├── t50.jpg ├── t51.jpg ├── t52.jpg ├── t53.jpg ├── t54.jpg ├── t55.jpg ├── t56.jpg ├── t57.jpg ├── t58.jpg ├── t59.jpg ├── t60.jpg ├── t61.jpg ├── t62.jpg ├── t63.jpg ├── t64.jpg ├── t65.jpg ├── t66.jpg ├── tt1.jpg ├── tt10.jpg ├── tt12.jpg ├── tt13.jpg ├── tt14.jpg ├── tt15.jpg ├── tt16.jpg ├── tt17.jpg ├── tt18.jpg ├── tt19.jpg ├── tt2.jpg ├── tt20.jpg ├── tt21.jpg ├── tt22.jpg ├── tt23.jpg ├── tt24.jpg ├── tt25.jpg ├── tt26.jpg ├── tt27.jpg ├── tt3.jpg ├── tt4.jpg ├── tt5.jpg ├── tt6.jpg ├── tt7.jpg ├── tt8.jpg ├── tt9.jpg ├── baby_GT.jpg ├── bird_GT.jpg ├── head_GT.jpg ├── woman_GT.jpg └── butterfly_GT.jpg ├── result ├── bicubic.jpg ├── espcn.png └── original.jpg ├── logdir_3x └── train │ ├── model.ckpt-950000.meta │ ├── model.ckpt-950000.index │ ├── model.ckpt-950000.data-00000-of-00001 │ └── checkpoint ├── .gitignore ├── logdir_2x └── train │ ├── model.ckpt-2500000.index │ ├── model.ckpt-2500000.meta │ ├── model.ckpt-2500000.data-00000-of-00001 │ └── checkpoint ├── video_process ├── video_to_frame.sh ├── to_180.sh └── to_360p.sh ├── params.json ├── psnr.py ├── reader.py ├── README.md ├── generate.py ├── train.py ├── generate_mul.py ├── espcn.py └── prepare_data.py /images/t1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t1.jpg -------------------------------------------------------------------------------- /images/t2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t2.jpg -------------------------------------------------------------------------------- /images/t3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t3.jpg -------------------------------------------------------------------------------- /images/t4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t4.jpg -------------------------------------------------------------------------------- /images/t5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t5.jpg -------------------------------------------------------------------------------- /images/t6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t6.jpg -------------------------------------------------------------------------------- /images/t7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t7.jpg -------------------------------------------------------------------------------- /images/t8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t8.jpg -------------------------------------------------------------------------------- /images/t9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t9.jpg -------------------------------------------------------------------------------- /images/t10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t10.jpg -------------------------------------------------------------------------------- /images/t11.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t11.jpg -------------------------------------------------------------------------------- /images/t12.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t12.jpg -------------------------------------------------------------------------------- /images/t13.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t13.jpg -------------------------------------------------------------------------------- /images/t14.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t14.jpg -------------------------------------------------------------------------------- /images/t15.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t15.jpg -------------------------------------------------------------------------------- /images/t16.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t16.jpg -------------------------------------------------------------------------------- /images/t17.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t17.jpg -------------------------------------------------------------------------------- /images/t18.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t18.jpg -------------------------------------------------------------------------------- /images/t19.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t19.jpg -------------------------------------------------------------------------------- /images/t20.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t20.jpg -------------------------------------------------------------------------------- /images/t21.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t21.jpg -------------------------------------------------------------------------------- /images/t22.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t22.jpg -------------------------------------------------------------------------------- /images/t23.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t23.jpg -------------------------------------------------------------------------------- /images/t24.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t24.jpg -------------------------------------------------------------------------------- /images/t25.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t25.jpg -------------------------------------------------------------------------------- /images/t26.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t26.jpg -------------------------------------------------------------------------------- /images/t27.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t27.jpg -------------------------------------------------------------------------------- /images/t28.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t28.jpg -------------------------------------------------------------------------------- /images/t29.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t29.jpg -------------------------------------------------------------------------------- /images/t30.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t30.jpg -------------------------------------------------------------------------------- /images/t31.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t31.jpg -------------------------------------------------------------------------------- /images/t32.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t32.jpg -------------------------------------------------------------------------------- /images/t33.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t33.jpg -------------------------------------------------------------------------------- /images/t34.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t34.jpg -------------------------------------------------------------------------------- /images/t35.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t35.jpg -------------------------------------------------------------------------------- /images/t36.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t36.jpg -------------------------------------------------------------------------------- /images/t37.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t37.jpg -------------------------------------------------------------------------------- /images/t38.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t38.jpg -------------------------------------------------------------------------------- /images/t39.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t39.jpg -------------------------------------------------------------------------------- /images/t40.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t40.jpg -------------------------------------------------------------------------------- /images/t42.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t42.jpg -------------------------------------------------------------------------------- /images/t43.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t43.jpg -------------------------------------------------------------------------------- /images/t44.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t44.jpg -------------------------------------------------------------------------------- /images/t45.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t45.jpg -------------------------------------------------------------------------------- /images/t46.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t46.jpg -------------------------------------------------------------------------------- /images/t47.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t47.jpg -------------------------------------------------------------------------------- /images/t48.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t48.jpg -------------------------------------------------------------------------------- /images/t49.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t49.jpg -------------------------------------------------------------------------------- /images/t50.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t50.jpg -------------------------------------------------------------------------------- /images/t51.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t51.jpg -------------------------------------------------------------------------------- /images/t52.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t52.jpg -------------------------------------------------------------------------------- /images/t53.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t53.jpg -------------------------------------------------------------------------------- /images/t54.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t54.jpg -------------------------------------------------------------------------------- /images/t55.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t55.jpg -------------------------------------------------------------------------------- /images/t56.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t56.jpg -------------------------------------------------------------------------------- /images/t57.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t57.jpg -------------------------------------------------------------------------------- /images/t58.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t58.jpg -------------------------------------------------------------------------------- /images/t59.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t59.jpg -------------------------------------------------------------------------------- /images/t60.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t60.jpg -------------------------------------------------------------------------------- /images/t61.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t61.jpg -------------------------------------------------------------------------------- /images/t62.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t62.jpg -------------------------------------------------------------------------------- /images/t63.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t63.jpg -------------------------------------------------------------------------------- /images/t64.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t64.jpg -------------------------------------------------------------------------------- /images/t65.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t65.jpg -------------------------------------------------------------------------------- /images/t66.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/t66.jpg -------------------------------------------------------------------------------- /images/tt1.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt1.jpg -------------------------------------------------------------------------------- /images/tt10.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt10.jpg -------------------------------------------------------------------------------- /images/tt12.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt12.jpg -------------------------------------------------------------------------------- /images/tt13.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt13.jpg -------------------------------------------------------------------------------- /images/tt14.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt14.jpg -------------------------------------------------------------------------------- /images/tt15.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt15.jpg -------------------------------------------------------------------------------- /images/tt16.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt16.jpg -------------------------------------------------------------------------------- /images/tt17.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt17.jpg -------------------------------------------------------------------------------- /images/tt18.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt18.jpg -------------------------------------------------------------------------------- /images/tt19.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt19.jpg -------------------------------------------------------------------------------- /images/tt2.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt2.jpg -------------------------------------------------------------------------------- /images/tt20.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt20.jpg -------------------------------------------------------------------------------- /images/tt21.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt21.jpg -------------------------------------------------------------------------------- /images/tt22.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt22.jpg -------------------------------------------------------------------------------- /images/tt23.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt23.jpg -------------------------------------------------------------------------------- /images/tt24.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt24.jpg -------------------------------------------------------------------------------- /images/tt25.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt25.jpg -------------------------------------------------------------------------------- /images/tt26.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt26.jpg -------------------------------------------------------------------------------- /images/tt27.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt27.jpg -------------------------------------------------------------------------------- /images/tt3.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt3.jpg -------------------------------------------------------------------------------- /images/tt4.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt4.jpg -------------------------------------------------------------------------------- /images/tt5.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt5.jpg -------------------------------------------------------------------------------- /images/tt6.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt6.jpg -------------------------------------------------------------------------------- /images/tt7.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt7.jpg -------------------------------------------------------------------------------- /images/tt8.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt8.jpg -------------------------------------------------------------------------------- /images/tt9.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/tt9.jpg -------------------------------------------------------------------------------- /images/baby_GT.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/baby_GT.jpg -------------------------------------------------------------------------------- /images/bird_GT.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/bird_GT.jpg -------------------------------------------------------------------------------- /images/head_GT.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/head_GT.jpg -------------------------------------------------------------------------------- /result/bicubic.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/result/bicubic.jpg -------------------------------------------------------------------------------- /result/espcn.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/result/espcn.png -------------------------------------------------------------------------------- /images/woman_GT.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/woman_GT.jpg -------------------------------------------------------------------------------- /result/original.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/result/original.jpg -------------------------------------------------------------------------------- /images/butterfly_GT.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/images/butterfly_GT.jpg -------------------------------------------------------------------------------- /logdir_3x/train/model.ckpt-950000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_3x/train/model.ckpt-950000.meta -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # VIM temp files 7 | .*.sw[op] 8 | -------------------------------------------------------------------------------- /logdir_2x/train/model.ckpt-2500000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_2x/train/model.ckpt-2500000.index -------------------------------------------------------------------------------- /logdir_2x/train/model.ckpt-2500000.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_2x/train/model.ckpt-2500000.meta -------------------------------------------------------------------------------- /logdir_3x/train/model.ckpt-950000.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_3x/train/model.ckpt-950000.index -------------------------------------------------------------------------------- /logdir_3x/train/model.ckpt-950000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_3x/train/model.ckpt-950000.data-00000-of-00001 -------------------------------------------------------------------------------- /logdir_2x/train/model.ckpt-2500000.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/drakelevy/ESPCN-TensorFlow/HEAD/logdir_2x/train/model.ckpt-2500000.data-00000-of-00001 -------------------------------------------------------------------------------- /video_process/video_to_frame.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 2 ]; then 4 | echo "Illegal number of parameters" 5 | echo "usage: $0 " 6 | exit 1 7 | fi 8 | 9 | ffmpeg -i $1 -vf scale=640:360 $2/frame%05d.png 10 | 11 | -------------------------------------------------------------------------------- /video_process/to_180.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 2 ]; then 4 | echo "Illegal number of parameters" 5 | exit 1 6 | fi 7 | 8 | fileList=$(ls -1 $1) 9 | 10 | for f in $fileList 11 | do 12 | echo $f 13 | convert $1/$f -resize 320x180 $2/$f 14 | done 15 | 16 | -------------------------------------------------------------------------------- /video_process/to_360p.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ "$#" -ne 2 ]; then 4 | echo "Illegal number of parameters" 5 | exit 1 6 | fi 7 | 8 | fileList=$(ls -1 $1) 9 | 10 | for f in $fileList 11 | do 12 | echo $f 13 | convert $1/$f -resize 640x360 $2/$f 14 | done 15 | 16 | -------------------------------------------------------------------------------- /logdir_3x/train/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "model.ckpt-950000" 2 | all_model_checkpoint_paths: "model.ckpt-949620" 3 | all_model_checkpoint_paths: "model.ckpt-949715" 4 | all_model_checkpoint_paths: "model.ckpt-949810" 5 | all_model_checkpoint_paths: "model.ckpt-949905" 6 | all_model_checkpoint_paths: "model.ckpt-950000" 7 | -------------------------------------------------------------------------------- /logdir_2x/train/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "model.ckpt-2500000" 2 | all_model_checkpoint_paths: "model.ckpt-2499000" 3 | all_model_checkpoint_paths: "model.ckpt-2499250" 4 | all_model_checkpoint_paths: "model.ckpt-2499500" 5 | all_model_checkpoint_paths: "model.ckpt-2499750" 6 | all_model_checkpoint_paths: "model.ckpt-2500000" 7 | -------------------------------------------------------------------------------- /params.json: -------------------------------------------------------------------------------- 1 | { 2 | "training_num": 70, 3 | "validation_num": 13, 4 | "test_num": 13, 5 | "ratio": 2, 6 | "lr_stride": 9, 7 | "lr_size": 17, 8 | "edge": 8, 9 | "image_dir": "images/", 10 | "training_dir": "training_{}x/", 11 | "training_image_dir": "training_images_{}x/", 12 | "validation_dir": "validation_{}x/", 13 | "validation_image_dir": "validation_images_{}x/", 14 | "test_dir": "test_{}x/", 15 | "test_image_dir": "test_images_{}x/", 16 | "filters_size": [5, 3, 3], 17 | "channels": [64, 32] 18 | } 19 | -------------------------------------------------------------------------------- /psnr.py: -------------------------------------------------------------------------------- 1 | import numpy as np 2 | import math 3 | 4 | def psnr(hr_image, sr_image, hr_edge): 5 | #assume RGB image 6 | hr_image_data = np.array(hr_image) 7 | if hr_edge > 0: 8 | hr_image_data = hr_image_data[hr_edge:-hr_edge, hr_edge:-hr_edge].astype('float32') 9 | 10 | sr_image_data = np.array(sr_image).astype('float32') 11 | 12 | diff = sr_image_data - hr_image_data 13 | diff = diff.flatten('C') 14 | rmse = math.sqrt( np.mean(diff ** 2.) ) 15 | return 20*math.log10(255.0/rmse) 16 | -------------------------------------------------------------------------------- /reader.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import numpy as np 3 | import os 4 | import pdb 5 | 6 | def create_inputs(params): 7 | """ 8 | Loads prepared training files and appends them as np arrays to a list. 9 | This approach is better because a FIFOQueue with a reader can't utilize 10 | the GPU while this approach can. 11 | """ 12 | sess = tf.Session() 13 | 14 | lr_images, hr_labels = [], [] 15 | training_dir = params['training_dir'].format(params['ratio']) 16 | 17 | # Raise exception if user has not ran prepare_data.py yet 18 | if not os.path.isdir(training_dir): 19 | raise Exception("You must first run prepare_data.py before you can train") 20 | 21 | lr_shape = (params['lr_size'], params['lr_size'], 3) 22 | hr_shape = output_shape = (params['lr_size'] - params['edge'], params['lr_size'] - params['edge'], 3 * params['ratio']**2) 23 | for file in os.listdir(training_dir): 24 | train_file = open("{}/{}".format(training_dir, file), "rb") 25 | train_data = np.fromfile(train_file, dtype=np.uint8) 26 | 27 | lr_image = train_data[:17 * 17 * 3].reshape(lr_shape) 28 | lr_images.append(lr_image) 29 | 30 | hr_label = train_data[17 * 17 * 3:].reshape(hr_shape) 31 | hr_labels.append(hr_label) 32 | 33 | return lr_images, hr_labels 34 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ESPCN-TensorFlow 2 | TensorFlow implementation of the Efficient Sub-Pixel Convolutional Neural Network in TensorFlow (ESPCN). Network based on this [paper](https://arxiv.org/pdf/1609.05158.pdf) and code adapted from this [repo](https://github.com/JesseYang/Espcn). 3 |
4 | This network can achieve the real-time performance of the [FSRCNN](https://arxiv.org/abs/1608.00367) while also surpassing the quality of the [SRCNN](https://arxiv.org/abs/1501.00092). 5 | 6 | ## Prerequisites 7 | * Python 2.7 8 | * TensorFlow 9 | * Numpy 10 | * Scipy version > 0.18 11 | 12 | ## Usage 13 | Run `prepare_data.py` to format the training and validation data before training each new model 14 |
15 | For training: `python train.py` 16 |
17 | Can specify epochs, learning rate, batch size etc: 18 |
19 | `python train.py --epochs 10 --learning_rate 0.0001 --batch_size 32` 20 |
21 | 22 | For generating: `python generate.py` 23 |
24 | Must specify checkpoint, low-resolution image, and output path 25 |
26 | `python generate.py --checkpoint logdir_2x/train --lr_image images/butterfly_GT.png --out_path result/butterfly_HR` 27 | 28 | Check `params.json` for parameter values and to change the upscaling ratio (2x, 3x, ...) the model is operating on. 29 | 30 | ## Result 31 | 32 | Original butterfly image: 33 |
34 | ![orig](https://github.com/drakelevy/ESPCN-TensorFlow/blob/master/result/original.jpg) 35 |
36 | Bicubic interpolated image: 37 |
38 | ![bicubic](https://github.com/drakelevy/ESPCN-Tensorflow/blob/master/result/bicubic.jpg) 39 |
40 | Super-resolved image: 41 |
42 | ![espcn](https://github.com/drakelevy/ESPCN-Tensorflow/blob/master/result/espcn.png) 43 | 44 | ## References 45 | * [JesseYang/Espcn](https://github.com/JesseYang/Espcn) 46 | -------------------------------------------------------------------------------- /generate.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tensorflow as tf 3 | from scipy import ndimage 4 | from scipy import misc 5 | import numpy as np 6 | from prepare_data import * 7 | from psnr import psnr 8 | import json 9 | import pdb 10 | 11 | from espcn import ESPCN 12 | 13 | def get_arguments(): 14 | parser = argparse.ArgumentParser(description='EspcnNet generation script') 15 | parser.add_argument('--checkpoint', type=str, 16 | help='Which model checkpoint to generate from') 17 | parser.add_argument('--lr_image', type=str, 18 | help='The low-resolution image waiting for processed.') 19 | parser.add_argument('--hr_image', type=str, 20 | help='The high-resolution image which is used to calculate PSNR.') 21 | parser.add_argument('--out_path', type=str, 22 | help='The output path for the super-resolution image') 23 | return parser.parse_args() 24 | 25 | def check_params(args, params): 26 | if len(params['filters_size']) - len(params['channels']) != 1: 27 | print("The length of 'filters_size' must be greater then the length of 'channels' by 1.") 28 | return False 29 | return True 30 | 31 | def generate(): 32 | args = get_arguments() 33 | 34 | with open("./params.json", 'r') as f: 35 | params = json.load(f) 36 | 37 | if check_params(args, params) == False: 38 | return 39 | 40 | sess = tf.Session() 41 | 42 | net = ESPCN(filters_size=params['filters_size'], 43 | channels=params['channels'], 44 | ratio=params['ratio'], 45 | batch_size=1, 46 | lr_size=params['lr_size'], 47 | edge=params['edge']) 48 | 49 | loss, images, labels = net.build_model() 50 | 51 | lr_image = tf.placeholder(tf.uint8) 52 | lr_image_data = misc.imread(args.lr_image) 53 | lr_image_ycbcr_data = rgb2ycbcr(lr_image_data) 54 | lr_image_y_data = lr_image_ycbcr_data[:, :, 0:1] 55 | lr_image_cb_data = lr_image_ycbcr_data[:, :, 1:2] 56 | lr_image_cr_data = lr_image_ycbcr_data[:, :, 2:3] 57 | lr_image_batch = np.zeros((1,) + lr_image_y_data.shape) 58 | lr_image_batch[0] = lr_image_y_data 59 | 60 | sr_image = net.generate(lr_image) 61 | 62 | saver = tf.train.Saver() 63 | try: 64 | model_loaded = net.load(sess, saver, args.checkpoint) 65 | except: 66 | raise Exception("Failed to load model, does the ratio in params.json match the ratio you trained your checkpoint with?") 67 | 68 | if model_loaded: 69 | print("[*] Checkpoint load success!") 70 | else: 71 | print("[*] Checkpoint load failed/no checkpoint found") 72 | return 73 | 74 | sr_image_y_data = sess.run(sr_image, feed_dict={lr_image: lr_image_batch}) 75 | 76 | sr_image_y_data = shuffle(sr_image_y_data[0], params['ratio']) 77 | sr_image_ycbcr_data = misc.imresize(lr_image_ycbcr_data, 78 | params['ratio'] * np.array(lr_image_data.shape[0:2]), 79 | 'bicubic') 80 | 81 | edge = params['edge'] * params['ratio'] / 2 82 | 83 | sr_image_ycbcr_data = np.concatenate((sr_image_y_data, sr_image_ycbcr_data[edge:-edge,edge:-edge,1:3]), axis=2) 84 | sr_image_data = ycbcr2rgb(sr_image_ycbcr_data) 85 | 86 | misc.imsave(args.out_path + '.png', sr_image_data) 87 | 88 | if args.hr_image != None: 89 | hr_image_data = misc.imread(args.hr_image) 90 | model_psnr = psnr(hr_image_data, sr_image_data, edge) 91 | print('PSNR of the model: {:.2f}dB'.format(model_psnr)) 92 | 93 | sr_image_bicubic_data = misc.imresize(lr_image_data, 94 | params['ratio'] * np.array(lr_image_data.shape[0:2]), 95 | 'bicubic') 96 | misc.imsave(args.out_path + '_bicubic.png', sr_image_bicubic_data) 97 | bicubic_psnr = psnr(hr_image_data, sr_image_bicubic_data, 0) 98 | print('PSNR of Bicubic: {:.2f}dB'.format(bicubic_psnr)) 99 | 100 | 101 | if __name__ == '__main__': 102 | generate() -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import argparse 3 | from datetime import datetime 4 | import os 5 | import sys 6 | import time 7 | import json 8 | import time 9 | 10 | import tensorflow as tf 11 | from reader import create_inputs 12 | from espcn import ESPCN 13 | 14 | import pdb 15 | 16 | BATCH_SIZE = 32 17 | NUM_EPOCHS = 100 18 | LEARNING_RATE = 0.0001 19 | LOGDIR_ROOT = './logdir_{}x' 20 | 21 | def get_arguments(): 22 | parser = argparse.ArgumentParser(description='EspcnNet example network') 23 | parser.add_argument('--checkpoint', type=str, 24 | help='Which model checkpoint to load from', default=None) 25 | parser.add_argument('--batch_size', type=int, default=BATCH_SIZE, 26 | help='How many image files to process at once.') 27 | parser.add_argument('--epochs', type=int, default=NUM_EPOCHS, 28 | help='Number of epochs.') 29 | parser.add_argument('--learning_rate', type=float, default=LEARNING_RATE, 30 | help='Learning rate for training.') 31 | parser.add_argument('--logdir_root', type=str, default=LOGDIR_ROOT, 32 | help='Root directory to place the logging ' 33 | 'output and generated model. These are stored ' 34 | 'under the dated subdirectory of --logdir_root. ' 35 | 'Cannot use with --logdir.') 36 | 37 | return parser.parse_args() 38 | 39 | def check_params(args, params): 40 | if len(params['filters_size']) - len(params['channels']) != 1: 41 | print("The length of 'filters_size' must be greater then the length of 'channels' by 1.") 42 | return False 43 | return True 44 | 45 | def train(): 46 | args = get_arguments() 47 | 48 | with open("./params.json", 'r') as f: 49 | params = json.load(f) 50 | 51 | if check_params(args, params) == False: 52 | return 53 | 54 | logdir_root = args.logdir_root # ./logdir 55 | if logdir_root == LOGDIR_ROOT: 56 | logdir_root = logdir_root.format(params['ratio']) # ./logdir_{RATIO}x 57 | logdir = os.path.join(logdir_root, 'train') # ./logdir_{RATIO}x/train 58 | 59 | # Load training data as np arrays 60 | lr_images, hr_labels = create_inputs(params) 61 | 62 | net = ESPCN(filters_size=params['filters_size'], 63 | channels=params['channels'], 64 | ratio=params['ratio'], 65 | batch_size=args.batch_size, 66 | lr_size=params['lr_size'], 67 | edge=params['edge']) 68 | 69 | loss, images, labels = net.build_model() 70 | optimizer = tf.train.AdamOptimizer(learning_rate=args.learning_rate) 71 | trainable = tf.trainable_variables() 72 | optim = optimizer.minimize(loss, var_list=trainable) 73 | 74 | # set up logging for tensorboard 75 | writer = tf.summary.FileWriter(logdir) 76 | writer.add_graph(tf.get_default_graph()) 77 | summaries = tf.summary.merge_all() 78 | 79 | # set up session 80 | sess = tf.Session() 81 | 82 | # saver for storing/restoring checkpoints of the model 83 | saver = tf.train.Saver() 84 | 85 | init = tf.initialize_all_variables() 86 | sess.run(init) 87 | 88 | if net.load(sess, saver, logdir): 89 | print("[*] Checkpoint load success!") 90 | else: 91 | print("[*] Checkpoint load failed/no checkpoint found") 92 | 93 | try: 94 | steps, start_average, end_average = 0, 0, 0 95 | start_time = time.time() 96 | for ep in xrange(1, args.epochs + 1): 97 | batch_idxs = len(lr_images) // args.batch_size 98 | batch_average = 0 99 | for idx in xrange(0, batch_idxs): 100 | # On the fly batch generation instead of Queue to optimize GPU usage 101 | batch_images = lr_images[idx * args.batch_size : (idx + 1) * args.batch_size] 102 | batch_labels = hr_labels[idx * args.batch_size : (idx + 1) * args.batch_size] 103 | 104 | steps += 1 105 | summary, loss_value, _ = sess.run([summaries, loss, optim], feed_dict={images: batch_images, labels: batch_labels}) 106 | writer.add_summary(summary, steps) 107 | batch_average += loss_value 108 | 109 | # Compare loss of first 20% and last 20% 110 | batch_average = float(batch_average) / batch_idxs 111 | if ep < (args.epochs * 0.2): 112 | start_average += batch_average 113 | elif ep >= (args.epochs * 0.8): 114 | end_average += batch_average 115 | 116 | duration = time.time() - start_time 117 | print('Epoch: {}, step: {:d}, loss: {:.9f}, ({:.3f} sec/epoch)'.format(ep, steps, batch_average, duration)) 118 | start_time = time.time() 119 | net.save(sess, saver, logdir, steps) 120 | except KeyboardInterrupt: 121 | print() 122 | finally: 123 | start_average = float(start_average) / (args.epochs * 0.2) 124 | end_average = float(end_average) / (args.epochs * 0.2) 125 | print("Start Average: [%.6f], End Average: [%.6f], Improved: [%.2f%%]" \ 126 | % (start_average, end_average, 100 - (100*end_average/start_average))) 127 | 128 | if __name__ == '__main__': 129 | train() 130 | -------------------------------------------------------------------------------- /generate_mul.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import tensorflow as tf 3 | from scipy import ndimage 4 | from scipy import misc 5 | import numpy as np 6 | from prepare_data import * 7 | from psnr import psnr 8 | import json 9 | import pdb 10 | import os 11 | from time import time 12 | 13 | from espcn import ESPCN 14 | 15 | RATIO = 2 16 | 17 | def get_arguments(): 18 | parser = argparse.ArgumentParser(description='EspcnNet generation script') 19 | parser.add_argument('--checkpoint', type=str, 20 | help='Which model checkpoint to generate from') 21 | parser.add_argument('--ratio', type=int, default=RATIO, 22 | help='The ratio for up-sampling, should be the same with the model.') 23 | parser.add_argument('--lr_image_dir', type=str, 24 | help='The low-resolution image directory waiting for processed.') 25 | parser.add_argument('--hr_image_dir', type=str, 26 | help='The high-resolution image directory which is used to calculate PSNR.') 27 | parser.add_argument('--out_path_dir', type=str, 28 | help='The output directory for the super-resolution image') 29 | return parser.parse_args() 30 | 31 | def check_params(args, params): 32 | if len(params['filters_size']) - len(params['channels']) != 1: 33 | print("The length of 'filters_size' must be greater then the length of 'channels' by 1.") 34 | return False 35 | return True 36 | 37 | def generate(): 38 | args = get_arguments() 39 | 40 | with open("./params.json", 'r') as f: 41 | params = json.load(f) 42 | 43 | if check_params(args, params) == False: 44 | return 45 | 46 | sess = tf.Session() 47 | 48 | net = ESPCN(filters_size=params['filters_size'], 49 | channels=params['channels'], 50 | ratio=params['ratio'], 51 | batch_size=1, 52 | lr_size=params['lr_size'], 53 | edge=params['edge']) 54 | 55 | loss, images, labels = net.build_model() 56 | 57 | files = [f for f in os.listdir(args.lr_image_dir) if os.path.isfile(os.path.join(args.lr_image_dir, f))] 58 | 59 | saver = tf.train.Saver() 60 | if net.load(sess, saver, args.checkpoint): 61 | print("[*] Checkpoint load success!") 62 | else: 63 | print("[*] Checkpoint load failed/no checkpoint found") 64 | return 65 | 66 | frame_range = (87, 10000) 67 | 68 | for fileName in files: 69 | try: 70 | ts = time() 71 | frame_cnt = int(fileName[5:10]) 72 | if frame_cnt < frame_range[0] or frame_cnt > frame_range[1]: 73 | print 'Ignoring frame ' + str(frame_cnt) 74 | continue 75 | else: 76 | print 'start sr for frame ' + str(frame_cnt) 77 | 78 | input_file = os.path.join(args.lr_image_dir, fileName) 79 | output_file = os.path.join(args.out_path_dir, fileName) 80 | 81 | lr_image = tf.placeholder(tf.uint8) 82 | lr_image_data = misc.imread(input_file) # pip install pillow 83 | lr_image_ycbcr_data = rgb2ycbcr(lr_image_data) 84 | lr_image_y_data = lr_image_ycbcr_data[:, :, 0:1] 85 | lr_image_cb_data = lr_image_ycbcr_data[:, :, 1:2] 86 | lr_image_cr_data = lr_image_ycbcr_data[:, :, 2:3] 87 | lr_image_batch = np.zeros((1,) + lr_image_y_data.shape) 88 | lr_image_batch[0] = lr_image_y_data 89 | print 'preprocessed %d ms' % ((time()-ts)*1000) 90 | ts = time() 91 | 92 | sr_image = net.generate(lr_image) 93 | print 'network generated %d ms' % ((time()-ts)*1000) 94 | ts = time() 95 | 96 | 97 | sr_image_y_data = sess.run(sr_image, feed_dict={lr_image: lr_image_batch}) 98 | 99 | print 'run %d ms' % ((time()-ts)*1000) 100 | ts = time() 101 | 102 | sr_image_y_data = shuffle(sr_image_y_data[0], args.ratio) 103 | sr_image_ycbcr_data = misc.imresize(lr_image_ycbcr_data, 104 | params['ratio'] * np.array(lr_image_data.shape[0:2]), 105 | 'bicubic') 106 | 107 | edge = params['edge'] * params['ratio'] / 2 108 | 109 | sr_image_ycbcr_data = np.concatenate((sr_image_y_data, sr_image_ycbcr_data[edge:-edge,edge:-edge,1:3]), axis=2) 110 | print 'mixed %d ms' % ((time()-ts)*1000) 111 | ts = time() 112 | sr_image_data = ycbcr2rgb(sr_image_ycbcr_data) 113 | #sr_image_data = sr_image_ycbcr_data 114 | print 'converted %d ms' % ((time()-ts)*1000) 115 | ts = time() 116 | 117 | misc.imsave(output_file, sr_image_data) 118 | print output_file + ' generated %d ms' % ((time()-ts)*1000) 119 | ts = time() 120 | 121 | if args.hr_image_dir != None: 122 | hr_image_path = os.path.join(args.hr_image_dir, fileName) 123 | hr_image_data = misc.imread(hr_image_path) 124 | model_psnr = psnr(hr_image_data, sr_image_data, edge) 125 | print('PSNR of the model: {:.2f}dB'.format(model_psnr)) 126 | 127 | sr_image_bicubic_data = misc.imresize(lr_image_data, 128 | params['ratio'] * np.array(lr_image_data.shape[0:2]), 129 | 'bicubic') 130 | bicubic_path = os.path.join(args.out_path_dir, fileName + '_bicubic.png') 131 | misc.imsave(bicubic_path, sr_image_bicubic_data) 132 | bicubic_psnr = psnr(hr_image_data, sr_image_bicubic_data, 0) 133 | print('PSNR of Bicubic: {:.2f}dB'.format(bicubic_psnr)) 134 | except IndexError: 135 | print 'Index error caught' 136 | except IOError: 137 | print 'Cannot identify image file: ' + fileName 138 | except ValueError: 139 | print 'Cannot parse file name: ' + fileName 140 | 141 | 142 | 143 | if __name__ == '__main__': 144 | generate() 145 | -------------------------------------------------------------------------------- /espcn.py: -------------------------------------------------------------------------------- 1 | import tensorflow as tf 2 | import os 3 | import sys 4 | import pdb 5 | 6 | def create_variable(name, shape): 7 | '''Create a convolution filter variable with the specified name and shape, 8 | and initialize it using Xavier initialition.''' 9 | initializer = tf.contrib.layers.xavier_initializer_conv2d() 10 | variable = tf.Variable(initializer(shape=shape), name=name) 11 | return variable 12 | 13 | def create_bias_variable(name, shape): 14 | '''Create a bias variable with the specified name and shape and initialize 15 | it to zero.''' 16 | initializer = tf.constant_initializer(value=0.0, dtype=tf.float32) 17 | return tf.Variable(initializer(shape=shape), name) 18 | 19 | class ESPCN: 20 | def __init__(self, filters_size, channels, ratio, batch_size, lr_size, edge): 21 | self.filters_size = filters_size 22 | self.channels = channels 23 | self.ratio = ratio 24 | self.batch_size = batch_size 25 | self.lr_size = lr_size 26 | self.edge = edge 27 | self.variables = self.create_variables() 28 | 29 | def create_variables(self): 30 | var = dict() 31 | var['filters'] = list() 32 | # the input layer 33 | var['filters'].append( 34 | create_variable('filter', 35 | [self.filters_size[0], 36 | self.filters_size[0], 37 | 1, 38 | self.channels[0]])) 39 | # the hidden layers 40 | for idx in range(1, len(self.filters_size) - 1): 41 | var['filters'].append( 42 | create_variable('filter', 43 | [self.filters_size[idx], 44 | self.filters_size[idx], 45 | self.channels[idx - 1], 46 | self.channels[idx]])) 47 | # the output layer 48 | var['filters'].append( 49 | create_variable('filter', 50 | [self.filters_size[-1], 51 | self.filters_size[-1], 52 | self.channels[-1], 53 | self.ratio**2])) 54 | 55 | var['biases'] = list() 56 | for channel in self.channels: 57 | var['biases'].append(create_bias_variable('bias', [channel])) 58 | var['biases'].append(create_bias_variable('bias', [self.ratio**2])) 59 | 60 | 61 | image_shape = (self.batch_size, self.lr_size, self.lr_size, 3) 62 | var['images'] = tf.placeholder(tf.uint8, shape=image_shape, name='images') 63 | label_shape = (self.batch_size, self.lr_size - self.edge, self.lr_size - self.edge, 3 * self.ratio**2) 64 | var['labels'] = tf.placeholder(tf.uint8, shape=label_shape, name='labels') 65 | 66 | return var 67 | 68 | def build_model(self): 69 | images, labels = self.variables['images'], self.variables['labels'] 70 | input_images, input_labels = self.preprocess([images, labels]) 71 | output = self.create_network(input_images) 72 | reduced_loss = self.loss(output, input_labels) 73 | return reduced_loss, images, labels 74 | 75 | def save(self, sess, saver, logdir, step): 76 | # print('[*] Storing checkpoint to {} ...'.format(logdir), end="") 77 | sys.stdout.flush() 78 | 79 | if not os.path.exists(logdir): 80 | os.makedirs(logdir) 81 | 82 | checkpoint = os.path.join(logdir, "model.ckpt") 83 | saver.save(sess, checkpoint, global_step=step) 84 | # print('[*] Done saving checkpoint.') 85 | 86 | def load(self, sess, saver, logdir): 87 | print("[*] Reading checkpoints...") 88 | ckpt = tf.train.get_checkpoint_state(logdir) 89 | 90 | if ckpt and ckpt.model_checkpoint_path: 91 | ckpt_name = os.path.basename(ckpt.model_checkpoint_path) 92 | saver.restore(sess, os.path.join(logdir, ckpt_name)) 93 | return True 94 | else: 95 | return False 96 | 97 | def preprocess(self, input_data): 98 | # cast to float32 and normalize the data 99 | input_list = list() 100 | for ele in input_data: 101 | if ele is None: 102 | continue 103 | ele = tf.cast(ele, tf.float32) / 255.0 104 | input_list.append(ele) 105 | 106 | input_images, input_labels = input_list[0][:,:,:,0:1], None 107 | # Generate doesn't use input_labels 108 | ratioSquare = self.ratio * self.ratio 109 | if input_data[1] is not None: 110 | input_labels = input_list[1][:,:,:,0:ratioSquare] 111 | return input_images, input_labels 112 | 113 | def create_network(self, input_labels): 114 | '''The default structure of the network is: 115 | 116 | input (3 channels) ---> 5 * 5 conv (64 channels) ---> 3 * 3 conv (32 channels) ---> 3 * 3 conv (3*r^2 channels) 117 | 118 | Where `conv` is 2d convolutions with a non-linear activation (tanh) at the output. 119 | ''' 120 | current_layer = input_labels 121 | 122 | for idx in range(len(self.filters_size)): 123 | conv = tf.nn.conv2d(current_layer, self.variables['filters'][idx], [1, 1, 1, 1], padding='VALID') 124 | with_bias = tf.nn.bias_add(conv, self.variables['biases'][idx]) 125 | if idx == len(self.filters_size) - 1: 126 | current_layer = with_bias 127 | else: 128 | current_layer = tf.nn.tanh(with_bias) 129 | return current_layer 130 | 131 | def loss(self, output, input_labels): 132 | residual = output - input_labels 133 | loss = tf.square(residual) 134 | reduced_loss = tf.reduce_mean(loss) 135 | tf.summary.scalar('loss', reduced_loss) 136 | return reduced_loss 137 | 138 | def generate(self, lr_image): 139 | lr_image = self.preprocess([lr_image, None])[0] 140 | sr_image = self.create_network(lr_image) 141 | sr_image = sr_image * 255.0 142 | sr_image = tf.cast(sr_image, tf.int32) 143 | sr_image = tf.maximum(sr_image, 0) 144 | sr_image = tf.minimum(sr_image, 255) 145 | sr_image = tf.cast(sr_image, tf.uint8) 146 | return sr_image 147 | -------------------------------------------------------------------------------- /prepare_data.py: -------------------------------------------------------------------------------- 1 | from scipy import misc, ndimage 2 | import numpy as np 3 | import imghdr 4 | import shutil 5 | import os 6 | import json 7 | 8 | mat = np.array( 9 | [[ 65.481, 128.553, 24.966 ], 10 | [-37.797, -74.203, 112.0 ], 11 | [ 112.0, -93.786, -18.214]]) 12 | mat_inv = np.linalg.inv(mat) 13 | offset = np.array([16, 128, 128]) 14 | 15 | def rgb2ycbcr(rgb_img): 16 | ycbcr_img = np.zeros(rgb_img.shape, dtype=np.uint8) 17 | for x in range(rgb_img.shape[0]): 18 | for y in range(rgb_img.shape[1]): 19 | ycbcr_img[x, y, :] = np.round(np.dot(mat, rgb_img[x, y, :] * 1.0 / 255) + offset) 20 | return ycbcr_img 21 | 22 | def ycbcr2rgb(ycbcr_img): 23 | rgb_img = np.zeros(ycbcr_img.shape, dtype=np.uint8) 24 | for x in range(ycbcr_img.shape[0]): 25 | for y in range(ycbcr_img.shape[1]): 26 | [r, g, b] = ycbcr_img[x,y,:] 27 | rgb_img[x, y, :] = np.maximum(0, np.minimum(255, np.round(np.dot(mat_inv, ycbcr_img[x, y, :] - offset) * 255.0))) 28 | return rgb_img 29 | 30 | def my_anti_shuffle(input_image, ratio): 31 | shape = input_image.shape 32 | ori_height = int(shape[0]) 33 | ori_width = int(shape[1]) 34 | ori_channels = int(shape[2]) 35 | if ori_height % ratio != 0 or ori_width % ratio != 0: 36 | print("Error! Height and width must be divided by ratio!") 37 | return 38 | height = ori_height / ratio 39 | width = ori_width / ratio 40 | channels = ori_channels * ratio * ratio 41 | anti_shuffle = np.zeros((height, width, channels), dtype=np.uint8) 42 | for c in range(0, ori_channels): 43 | for x in range(0, ratio): 44 | for y in range(0, ratio): 45 | anti_shuffle[:,:,c * ratio * ratio + x * ratio + y] = input_image[x::ratio, y::ratio, c] 46 | return anti_shuffle 47 | 48 | def shuffle(input_image, ratio): 49 | shape = input_image.shape 50 | height = int(shape[0]) * ratio 51 | width = int(shape[1]) * ratio 52 | channels = int(shape[2]) / ratio / ratio 53 | shuffled = np.zeros((height, width, channels), dtype=np.uint8) 54 | for i in range(0, height): 55 | for j in range(0, width): 56 | for k in range(0, channels): 57 | shuffled[i,j,k] = input_image[i / ratio, j / ratio, k * ratio * ratio + (i % ratio) * ratio + (j % ratio)] 58 | return shuffled 59 | 60 | def prepare_images(params): 61 | ratio, training_num, lr_stride, lr_size = params['ratio'], params['training_num'], params['lr_stride'], params['lr_size'] 62 | hr_stride = lr_stride * ratio 63 | hr_size = lr_size * ratio 64 | 65 | # first clear old images and create new directories 66 | for ele in ['training', 'validation', 'test']: 67 | new_dir = params[ele + '_image_dir'].format(ratio) 68 | if os.path.isdir(new_dir): 69 | shutil.rmtree(new_dir) 70 | for sub_dir in ['/hr', 'lr']: 71 | os.makedirs(new_dir + sub_dir) 72 | 73 | image_num = 0 74 | folder = params['training_image_dir'].format(ratio) 75 | for root, dirnames, filenames in os.walk(params['image_dir']): 76 | for filename in filenames: 77 | path = os.path.join(root, filename) 78 | if imghdr.what(path) != 'jpeg': 79 | continue 80 | 81 | hr_image = misc.imread(path) 82 | height = hr_image.shape[0] 83 | new_height = height - height % ratio 84 | width = hr_image.shape[1] 85 | new_width = width - width % ratio 86 | hr_image = hr_image[0:new_height,0:new_width] 87 | blurred = ndimage.gaussian_filter(hr_image, sigma=(1, 1, 0)) 88 | lr_image = blurred[::ratio,::ratio,:] 89 | 90 | height = hr_image.shape[0] 91 | width = hr_image.shape[1] 92 | vertical_number = height / hr_stride - 1 93 | horizontal_number = width / hr_stride - 1 94 | image_num = image_num + 1 95 | if image_num % 10 == 0: 96 | print "Finished image: {}".format(image_num) 97 | if image_num > training_num and image_num <= training_num + params['validation_num']: 98 | folder = params['validation_image_dir'].format(ratio) 99 | elif image_num > training_num + params['validation_num']: 100 | folder = params['test_image_dir'].format(ratio) 101 | #misc.imsave(folder + 'hr_full/' + filename[0:-4] + '.png', hr_image) 102 | #misc.imsave(folder + 'lr_full/' + filename[0:-4] + '.png', lr_image) 103 | for x in range(0, horizontal_number): 104 | for y in range(0, vertical_number): 105 | hr_sub_image = hr_image[y * hr_stride : y * hr_stride + hr_size, x * hr_stride : x * hr_stride + hr_size] 106 | lr_sub_image = lr_image[y * lr_stride : y * lr_stride + lr_size, x * lr_stride : x * lr_stride + lr_size] 107 | misc.imsave("{}hr/{}_{}_{}.png".format(folder, filename[0:-4], y, x), hr_sub_image) 108 | misc.imsave("{}lr/{}_{}_{}.png".format(folder, filename[0:-4], y, x), lr_sub_image) 109 | if image_num >= training_num + params['validation_num'] + params['test_num']: 110 | break 111 | else: 112 | continue 113 | break 114 | 115 | def prepare_data(params): 116 | ratio = params['ratio'] 117 | params['hr_stride'] = params['lr_stride'] * ratio 118 | params['hr_size'] = params['lr_size'] * ratio 119 | 120 | for ele in ['training', 'validation', 'test']: 121 | new_dir = params[ele + '_dir'].format(ratio) 122 | if os.path.isdir(new_dir): 123 | shutil.rmtree(new_dir) 124 | os.makedirs(new_dir) 125 | 126 | ratio, lr_size, edge = params['ratio'], params['lr_size'], params['edge'] 127 | image_dirs = [d.format(ratio) for d in [params['training_image_dir'], params['validation_image_dir'], params['test_image_dir']]] 128 | data_dirs = [d.format(ratio) for d in [params['training_dir'], params['validation_dir'], params['test_dir']]] 129 | hr_start_idx = ratio * edge / 2 130 | hr_end_idx = hr_start_idx + (lr_size - edge) * ratio 131 | sub_hr_size = (lr_size - edge) * ratio 132 | for dir_idx, image_dir in enumerate(image_dirs): 133 | data_dir = data_dirs[dir_idx] 134 | print "Creating {}".format(data_dir) 135 | for root, dirnames, filenames in os.walk(image_dir + "/lr"): 136 | for filename in filenames: 137 | lr_path = os.path.join(root, filename) 138 | hr_path = image_dir + "/hr/" + filename 139 | lr_image = misc.imread(lr_path) 140 | hr_image = misc.imread(hr_path) 141 | # convert to Ycbcr color space 142 | lr_image_y = rgb2ycbcr(lr_image) 143 | hr_image_y = rgb2ycbcr(hr_image) 144 | lr_data = lr_image_y.reshape((lr_size * lr_size * 3)) 145 | sub_hr_image_y = hr_image_y[hr_start_idx:hr_end_idx:1,hr_start_idx:hr_end_idx:1] 146 | hr_data = my_anti_shuffle(sub_hr_image_y, ratio).reshape(sub_hr_size * sub_hr_size * 3) 147 | data = np.concatenate([lr_data, hr_data]) 148 | data.astype('uint8').tofile(data_dir + "/" + filename[0:-4]) 149 | 150 | def remove_images(params): 151 | # Don't need old image folders 152 | for ele in ['training', 'validation', 'test']: 153 | rm_dir = params[ele + '_image_dir'].format(params['ratio']) 154 | if os.path.isdir(rm_dir): 155 | shutil.rmtree(rm_dir) 156 | 157 | 158 | if __name__ == '__main__': 159 | with open("./params.json", 'r') as f: 160 | params = json.load(f) 161 | 162 | print "Preparing images with scaling ratio: {}".format(params['ratio']) 163 | print "If you want a different ratio change 'ratio' in params.json" 164 | print "Splitting images (1/3)" 165 | prepare_images(params) 166 | 167 | print "Preparing data, this may take a while (2/3)" 168 | prepare_data(params) 169 | 170 | print "Cleaning up split images (3/3)" 171 | remove_images(params) 172 | print "Done, you can now train the model!" 173 | --------------------------------------------------------------------------------